From 553d89bd1293919585a14966b420c1af7455c52c Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 5 Apr 2023 00:12:01 -0700 Subject: [PATCH 001/215] add support for the new gwfailover bit --- common/const_var.go | 4 ++ http/supported_groups_handler_test.go | 2 + util/firmware_bitmap_test.go | 62 +++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index d1ad66d..e7e02a5 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -169,6 +169,9 @@ var ( 14: { {1, 24}, }, + 15: { + {1, 26}, + }, } ) @@ -199,6 +202,7 @@ var ( "wanfailover": 23, "cellularconfig": 24, "telcovoice": 25, + "gwfailover": 26, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index a8e5bc4..ccefd16 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -105,6 +105,7 @@ func TestSupportedGroupsHandler(t *testing.T) { assert.NilError(t, err) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) // ==== step 2 add lan data ==== @@ -285,5 +286,6 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) } diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 7ac8457..e8e55bf 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -22,7 +22,6 @@ import ( "strings" "testing" - "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) @@ -158,6 +157,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -203,6 +203,7 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -248,6 +249,7 @@ func TestBitmapParsing(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -293,6 +295,7 @@ func TestParseVoiceService(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -306,14 +309,8 @@ func TestManualBitmap(t *testing.T) { } func TestParseSupportedDocsWithNewGroups(t *testing.T) { - // get the max metadata group_id - var maxMetadataGroupId int - for mgid := range common.SupportedDocsBitMaskMap { - if mgid > maxMetadataGroupId { - maxMetadataGroupId = mgid - } - } - xBitValue := (maxMetadataGroupId << 24) + 1 + cellularBitGroupId := 14 + xBitValue := (cellularBitGroupId << 24) + 1 ss := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809,234881025" rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) @@ -354,6 +351,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -397,6 +395,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { } parsedSupportedMap := GetSupportedMap(cpeBitmap) + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -439,6 +438,51 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { assert.Equal(t, parsedEnabled, enabled) } + parsedSupportedMap := GetSupportedMap(cpeBitmap) + expectedEnabled["gwfailover"] = false + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) +} + +func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" + + // build expected + expectedEnabled := map[string]bool{ + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + parsedSupportedMap := GetSupportedMap(cpeBitmap) assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } From 149f7ff73b827e88287b87dbc289e118c050e1a1 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 20 May 2023 11:39:08 -0700 Subject: [PATCH 002/215] enable sarama client logger --- main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/main.go b/main.go index 8ac1ca3..343859e 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "os/signal" "syscall" + "github.com/Shopify/sarama" "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" "github.com/rdkcentral/webconfig/common" @@ -90,6 +91,9 @@ func main() { } log.SetLevel(logLevel) + // setup sarama logger + sarama.Logger = log.StandardLogger() + // setup router router := server.GetRouter(false) From 8dd03a06dd989aabcb15a775e49972cd0a3e3025 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 13 Nov 2023 22:12:48 -0800 Subject: [PATCH 003/215] add support for parsing bitmap of subdoc clienttosteeringprofile --- common/const_var.go | 64 +++---- http/supported_groups_handler_test.go | 183 ++++++++++---------- util/firmware_bitmap_test.go | 240 ++++++++++++++++---------- 3 files changed, 276 insertions(+), 211 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index ddba5f8..1d7cf70 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -143,7 +143,8 @@ var ( {1, 12}, }, 6: { - {1, 13}, + {1, 13}, // mesh + {2, 31}, // clienttosteeringprofile }, 7: { {1, 14}, @@ -188,36 +189,37 @@ var ( var ( SubdocBitIndexMap = map[string]int{ - "portforwarding": 1, - "lan": 2, - "wan": 3, - "macbinding": 4, - "hotspot": 5, - "bridge": 6, - "privatessid": 7, - "homessid": 8, - "radio": 9, - "moca": 10, - "xdns": 11, - "advsecurity": 12, - "mesh": 13, - "aker": 14, - "telemetry": 15, - "statusreport": 16, - "trafficreport": 17, - "interfacereport": 18, - "radioreport": 19, - "telcovoip": 20, - "wanmanager": 21, - "voiceservice": 22, - "wanfailover": 23, - "cellularconfig": 24, - "telcovoice": 25, - "gwfailover": 26, - "gwrestore": 27, - "prioritizedmacs": 28, - "connectedbuilding": 29, - "lldqoscontrol": 30, + "portforwarding": 1, + "lan": 2, + "wan": 3, + "macbinding": 4, + "hotspot": 5, + "bridge": 6, + "privatessid": 7, + "homessid": 8, + "radio": 9, + "moca": 10, + "xdns": 11, + "advsecurity": 12, + "mesh": 13, + "aker": 14, + "telemetry": 15, + "statusreport": 16, + "trafficreport": 17, + "interfacereport": 18, + "radioreport": 19, + "telcovoip": 20, + "wanmanager": 21, + "voiceservice": 22, + "wanfailover": 23, + "cellularconfig": 24, + "telcovoice": 25, + "gwfailover": 26, + "gwrestore": 27, + "prioritizedmacs": 28, + "connectedbuilding": 29, + "lldqoscontrol": 30, + "clienttosteeringprofile": 31, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index fa5a641..80ae1b1 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -67,36 +67,37 @@ func TestSupportedGroupsHandler(t *testing.T) { // call GET /supported_groups expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - "wanfailover": false, - "cellularconfig": false, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, + "portforwarding": true, + "lan": true, + "wan": true, + "macbinding": true, + "hotspot": false, + "bridge": false, + "privatessid": true, + "homessid": true, + "radio": false, + "moca": true, + "xdns": true, + "advsecurity": true, + "mesh": true, + "aker": true, + "telemetry": true, + "statusreport": false, + "trafficreport": false, + "interfacereport": false, + "radioreport": false, + "telcovoip": false, + "telcovoice": false, + "wanmanager": false, + "voiceservice": false, + "wanfailover": false, + "cellularconfig": false, + "gwfailover": false, + "gwrestore": false, + "prioritizedmacs": false, + "connectedbuilding": false, + "lldqoscontrol": false, + "clienttosteeringprofile": false, } // call GET /supported_groups to verify response @@ -251,36 +252,37 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { // call GET /supported_groups expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": false, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": true, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "homessid": false, + "hotspot": false, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": false, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoip": false, + "telcovoice": true, + "telemetry": true, + "trafficreport": false, + "voiceservice": false, + "wan": true, + "wanfailover": true, + "wanmanager": true, + "xdns": true, + "gwfailover": false, + "gwrestore": false, + "prioritizedmacs": false, + "connectedbuilding": false, + "lldqoscontrol": false, + "clienttosteeringprofile": false, } // call GET /supported_groups to verify response @@ -332,36 +334,37 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin // call GET /supported_groups expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwfailover": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoip": false, + "telcovoice": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwfailover": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 6cae866..4475183 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -162,6 +162,8 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -212,6 +214,7 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -262,6 +265,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -313,6 +317,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -374,6 +379,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -423,6 +429,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -472,6 +479,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -481,36 +489,37 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { // build expected expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": false, + "connectedbuilding": false, + "lldqoscontrol": false, + "clienttosteeringprofile": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -529,36 +538,37 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { // build expected expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": false, - "lldqoscontrol": false, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": false, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": false, + "lldqoscontrol": false, + "clienttosteeringprofile": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -577,36 +587,86 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test // build expected expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": false, + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) +} + +func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + expectedEnabled := map[string]bool{ + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": true, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From f78ac32659f23d59c1aa392e64a1dd9f3d34925d Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 20 Dec 2023 23:21:53 -0800 Subject: [PATCH 004/215] add a flag to disable appending query params in supplementary GET calls --- config/sample_webconfig.conf | 2 + http/supplementary_handler.go | 18 ++-- http/supplementary_handler_test.go | 141 +++++++++++++++++++++++++++++ http/webconfig_server.go | 107 ++++++++++++---------- 4 files changed, 213 insertions(+), 55 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 2655e1f..f47c838 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -230,4 +230,6 @@ webconfig { // if valid_partners is empty, then all partners are accepted // if valid partners is not empty, all parters NOT in the list will be stored as "unknown" valid_partners = ["company", "vendor1", "vendor2"] + + supplementary_appending_enabled = true } diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 880fe57..c4488c5 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -51,15 +51,17 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r // append the extra query_params if any var queryParams string - rootdoc, err := s.GetRootDocument(mac) - if err != nil { - if !s.IsDbNotFound(err) { - Error(w, http.StatusInternalServerError, common.NewError(err)) - return + if s.SupplementaryAppendingEnabled() { + rootdoc, err := s.GetRootDocument(mac) + if err != nil { + if !s.IsDbNotFound(err) { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + } + if rootdoc != nil { + queryParams = rootdoc.QueryParams } - } - if rootdoc != nil { - queryParams = rootdoc.QueryParams } // partner handling diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 8e1e0a3..ff08613 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -619,3 +619,144 @@ func TestSupplementaryApiBadPartnerHeader(t *testing.T) { assert.NilError(t, err) assert.DeepEqual(t, coreProfile1, srcItf) } + +func TestSupplementaryAppendingFlag(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock server ==== + var ss string + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ss = r.URL.String() + w.Header().Set(common.HeaderReqUrl, r.URL.String()) + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 set up the query params ==== + bitmap1 := 32479 + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + etag := strconv.Itoa(int(time.Now().Unix())) + queryParams1 := "stormReadyWifi=true" + srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1) + bbytes, err := json.Marshal(srcDoc1) + assert.NilError(t, err) + + rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) + req, err := http.NewRequest("POST", rootdocUrl, bytes.NewReader(bbytes)) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 2 set append flag true ==== + appendEnabled := true + server.SetSupplementaryAppendingEnabled(appendEnabled) + assert.Assert(t, appendEnabled == server.SupplementaryAppendingEnabled()) + + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // TODO verify the resHeader + xReqUrl := res.Header.Get(common.HeaderReqUrl) + assert.Assert(t, strings.Contains(xReqUrl, queryParams1)) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok := mparts["telemetry"] + assert.Assert(t, ok) + + output := common.TR181Output{} + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes := []byte(output.Parameters[0].Value) + + var itf util.Dict + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok := itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok := profilesItf.([]interface{}) + assert.Assert(t, ok) + + profile1Itf := profilesJs[0] + + profile1, ok := profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok := profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok := coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + var srcItf map[string]interface{} + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 3 verify the query params ==== + assert.Assert(t, strings.Contains(ss, "&stormReadyWifi=true")) + + // okok := false + // assert.Assert(t, okok) + + // ==== step 4 set append flag false ==== + appendEnabled = false + server.SetSupplementaryAppendingEnabled(appendEnabled) + assert.Assert(t, appendEnabled == server.SupplementaryAppendingEnabled()) + + req, err = http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 5 verify the query params ==== + assert.Assert(t, !strings.Contains(ss, "&stormReadyWifi=true")) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 932b8bf..94eed50 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -48,15 +48,16 @@ const ( ) const ( - MetricsEnabledDefault = true - FactoryResetEnabledDefault = false - serverApiTokenAuthEnabledDefault = false - deviceApiTokenAuthEnabledDefault = true - tokenApiEnabledDefault = false - activeDriverDefault = "cassandra" - defaultJwksEnabled = false - defaultTraceparentParentID = "0000000000000001" - defaultTracestateVendorID = "webconfig" + MetricsEnabledDefault = true + FactoryResetEnabledDefault = false + serverApiTokenAuthEnabledDefault = false + deviceApiTokenAuthEnabledDefault = true + tokenApiEnabledDefault = false + activeDriverDefault = "cassandra" + defaultJwksEnabled = false + defaultTraceparentParentID = "0000000000000001" + defaultTracestateVendorID = "webconfig" + defaultSupplementaryAppendingEnabled = true ) var ( @@ -85,21 +86,22 @@ type WebconfigServer struct { *XconfConnector *MqttConnector *UpstreamConnector - tlsConfig *tls.Config - notLoggedHeaders []string - metricsEnabled bool - factoryResetEnabled bool - serverApiTokenAuthEnabled bool - deviceApiTokenAuthEnabled bool - tokenApiEnabled bool - kafkaEnabled bool - upstreamEnabled bool - appName string - validateMacEnabled bool - validPartners []string - jwksEnabled bool - traceparentParentID string - tracestateVendorID string + tlsConfig *tls.Config + notLoggedHeaders []string + metricsEnabled bool + factoryResetEnabled bool + serverApiTokenAuthEnabled bool + deviceApiTokenAuthEnabled bool + tokenApiEnabled bool + kafkaEnabled bool + upstreamEnabled bool + appName string + validateMacEnabled bool + validPartners []string + jwksEnabled bool + traceparentParentID string + tracestateVendorID string + supplementaryAppendingEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -243,35 +245,38 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) + return &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, WriteTimeout: time.Duration(conf.GetInt32("webconfig.server.write_timeout_in_secs", 3)) * time.Second, }, - DatabaseClient: dbclient, - TokenManager: security.NewTokenManager(conf), - JwksManager: jwksManager, - ServerConfig: sc, - WebpaConnector: NewWebpaConnector(conf, tlsConfig), - XconfConnector: NewXconfConnector(conf, tlsConfig), - MqttConnector: NewMqttConnector(conf, tlsConfig), - UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), - tlsConfig: tlsConfig, - notLoggedHeaders: notLoggedHeaders, - metricsEnabled: metricsEnabled, - factoryResetEnabled: factoryResetEnabled, - serverApiTokenAuthEnabled: serverApiTokenAuthEnabled, - deviceApiTokenAuthEnabled: deviceApiTokenAuthEnabled, - tokenApiEnabled: tokenApiEnabled, - kafkaEnabled: kafkaEnabled, - upstreamEnabled: upstreamEnabled, - appName: appName, - validateMacEnabled: validateMacEnabled, - validPartners: validPartners, - jwksEnabled: jwksEnabled, - traceparentParentID: traceparentParentID, - tracestateVendorID: tracestateVendorID, + DatabaseClient: dbclient, + TokenManager: security.NewTokenManager(conf), + JwksManager: jwksManager, + ServerConfig: sc, + WebpaConnector: NewWebpaConnector(conf, tlsConfig), + XconfConnector: NewXconfConnector(conf, tlsConfig), + MqttConnector: NewMqttConnector(conf, tlsConfig), + UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), + tlsConfig: tlsConfig, + notLoggedHeaders: notLoggedHeaders, + metricsEnabled: metricsEnabled, + factoryResetEnabled: factoryResetEnabled, + serverApiTokenAuthEnabled: serverApiTokenAuthEnabled, + deviceApiTokenAuthEnabled: deviceApiTokenAuthEnabled, + tokenApiEnabled: tokenApiEnabled, + kafkaEnabled: kafkaEnabled, + upstreamEnabled: upstreamEnabled, + appName: appName, + validateMacEnabled: validateMacEnabled, + validPartners: validPartners, + jwksEnabled: jwksEnabled, + traceparentParentID: traceparentParentID, + tracestateVendorID: tracestateVendorID, + supplementaryAppendingEnabled: supplementaryAppendingEnabled, } } @@ -556,6 +561,14 @@ func (s *WebconfigServer) SetTracestateVendorID(x string) { s.tracestateVendorID = x } +func (s *WebconfigServer) SupplementaryAppendingEnabled() bool { + return s.supplementaryAppendingEnabled +} + +func (s *WebconfigServer) SetSupplementaryAppendingEnabled(enabled bool) { + s.supplementaryAppendingEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { From 5d20f85c46c751712d19dc3e7d0430db39818caf Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 8 Jan 2024 11:24:41 -0800 Subject: [PATCH 005/215] Update gocql lib version --- README.md | 2 +- go.mod | 4 ++-- go.sum | 10 ++++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c1ea263..c489fb1 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Webconfig supports 2 types of transport between cloud and devices 2. mqtt ## Install go -This project is written and tested with Go **1.17**. +This project is written and tested with Go **1.21**. ## Build the binary ```shell diff --git a/go.mod b/go.mod index 62ba922..a38b9a1 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module github.com/rdkcentral/webconfig -go 1.19 +go 1.21 require ( github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/Shopify/sarama v1.38.1 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 - github.com/gocql/gocql v1.2.1 + github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 diff --git a/go.sum b/go.sum index ed18197..2174f5b 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4Oe github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= +github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -77,6 +78,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -91,8 +93,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gocql/gocql v1.2.1 h1:G/STxUzD6pGvRHzG0Fi7S04SXejMKBbRZb7pwre1edU= -github.com/gocql/gocql v1.2.1/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= +github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= +github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= @@ -200,6 +202,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -224,6 +227,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -270,6 +274,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -566,6 +571,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= From 274072822e700c64431846e45a840d25bdde52a4 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 22 Jan 2024 21:48:42 -0800 Subject: [PATCH 006/215] add factory reset NONE/NONE-REBOOT handling --- common/const_var.go | 1 + common/server_config.go | 3 +- db/service.go | 36 +++- db/service_test.go | 28 +++ go.mod | 20 +- go.sum | 62 ++++--- http/document_handler_test.go | 66 +++---- http/factory_reset_upstream_test.go | 272 ++++++++++++++++++++++++++++ http/http_client.go | 3 +- http/multipart.go | 120 +++++++++++- http/multipart_test.go | 86 ++++----- http/poke_handler_test.go | 30 +-- http/rootdocument_handler_test.go | 18 +- http/simple_handler_test.go | 10 +- http/validator_test.go | 44 ++--- http/webconfig_server.go | 9 +- kafka/consumer.go | 4 +- kafka/consumer_group_test.go | 2 +- kafka/kafka_consumer_group.go | 2 +- main.go | 2 +- security/token.go | 6 +- 21 files changed, 641 insertions(+), 183 deletions(-) create mode 100644 http/factory_reset_upstream_test.go diff --git a/common/const_var.go b/common/const_var.go index 1d7cf70..4370c7e 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -105,6 +105,7 @@ const ( HeaderSourceAppName = "X-Source-App-Name" HeaderTraceparent = "Traceparent" HeaderTracestate = "Tracestate" + HeaderContentLength = "Content-Length" ) // header X-System-Supported-Docs diff --git a/common/server_config.go b/common/server_config.go index b37da7f..44fa5a7 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -19,7 +19,6 @@ package common import ( "fmt" - "io/ioutil" "os" "github.com/go-akka/configuration" @@ -40,7 +39,7 @@ type ServerConfig struct { } func NewServerConfig(configFile string) (*ServerConfig, error) { - configBytes, err := ioutil.ReadFile(configFile) + configBytes, err := os.ReadFile(configFile) if err != nil { return nil, NewError(err) } diff --git a/db/service.go b/db/service.go index 3da868c..4c2a598 100644 --- a/db/service.go +++ b/db/service.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "sort" "strings" "time" @@ -247,8 +248,13 @@ func HashRootVersion(itf interface{}) string { // if len(mparts) == 0, then the murmur hash value is 0 buffer := bytes.NewBufferString("") - for _, version := range versionMap { - buffer.WriteString(version) + keys := []string{} + for k := range versionMap { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + buffer.WriteString(versionMap[k]) } return util.GetMurmur3Hash(buffer.Bytes()) } @@ -386,7 +392,7 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old return nil } -func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, fields log.Fields) error { +func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, toDelete bool, fields log.Fields) error { newRootVersion := upstreamRespEtag if d.RootVersion() != newRootVersion { err := c.SetRootDocumentVersion(cpeMac, newRootVersion) @@ -395,6 +401,11 @@ func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string } } + oldMap := map[string]struct{}{} + for k := range d.Items() { + oldMap[k] = struct{}{} + } + // need to set "state" to proper values like the download is complete for subdocId, newSubdoc := range newDoc.Items() { oldSubdoc := d.SubDocument(subdocId) @@ -402,6 +413,16 @@ func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string if err != nil { return common.NewError(err) } + delete(oldMap, subdocId) + } + + if toDelete { + for subdocId := range oldMap { + err := c.DeleteSubDocument(cpeMac, subdocId) + if err != nil { + return common.NewError(err) + } + } } return nil } @@ -450,3 +471,12 @@ func RebuildDeviceVersionMap(versions []string, cloudVersionMap map[string]strin } return m } + +func RefreshRootDocumentVersion(doc *common.Document) { + versionMap := doc.VersionMap() + rootVersion := HashRootVersion(versionMap) + rootDoc := doc.GetRootDocument() + if rootDoc != nil { + rootDoc.Version = rootVersion + } +} diff --git a/db/service_test.go b/db/service_test.go index 1afe700..c0226b0 100644 --- a/db/service_test.go +++ b/db/service_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -47,3 +48,30 @@ func TestHashRootVersion(t *testing.T) { root = HashRootVersion(doc.VersionMap()) assert.Assert(t, root != "0") } + +func TestUpdateRootVersion(t *testing.T) { + doc := common.NewDocument(nil) + tt := int(123) + + bbytes1 := util.RandomBytes(100, 150) + v1 := util.GetMurmur3Hash(bbytes1) + subdoc1 := common.NewSubDocument(bbytes1, &v1, nil, &tt, nil, nil) + doc.SetSubDocument("advsecurity", subdoc1) + + bbytes2 := util.RandomBytes(100, 150) + v2 := util.GetMurmur3Hash(bbytes2) + subdoc2 := common.NewSubDocument(bbytes2, &v2, nil, &tt, nil, nil) + doc.SetSubDocument("mesh", subdoc2) + + rootVersion := HashRootVersion(doc.VersionMap()) + assert.Assert(t, rootVersion != "0") + rootDoc := common.NewRootDocument(123, "fw_ver_123", "model_123", "partner_123", "", rootVersion, "") + doc.SetRootDocument(rootDoc) + + doc.DeleteSubDocument("mesh") + + RefreshRootDocumentVersion(doc) + + newRootVersion := doc.RootVersion() + assert.Assert(t, rootVersion != newRootVersion) +} diff --git a/go.mod b/go.mod index a38b9a1..bbdcee4 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/rdkcentral/webconfig go 1.21 require ( + github.com/IBM/sarama v1.42.1 github.com/MicahParks/keyfunc/v2 v2.1.0 - github.com/Shopify/sarama v1.38.1 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt/v5 v5.0.0 @@ -19,7 +19,7 @@ require ( github.com/vmihailenco/msgpack/v4 v4.3.12 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.1.0 + golang.org/x/sync v0.4.0 gotest.tools v2.2.0+incompatible ) @@ -28,8 +28,8 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.3.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect + github.com/eapache/go-resiliency v1.4.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -41,21 +41,21 @@ require ( github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.15.14 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect - golang.org/x/net v0.5.0 // indirect - golang.org/x/sys v0.4.0 // indirect + golang.org/x/crypto v0.17.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.15.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 2174f5b..f502ae6 100644 --- a/go.sum +++ b/go.sum @@ -33,12 +33,10 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= +github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= -github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= -github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= -github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= -github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -67,10 +65,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= -github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM= -github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= +github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= +github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -181,8 +179,8 @@ github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVET github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= -github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -195,8 +193,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= -github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -218,8 +216,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -266,6 +264,7 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -273,8 +272,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -287,6 +287,7 @@ github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgq github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -304,8 +305,10 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -336,6 +339,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -364,13 +368,15 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -387,8 +393,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -427,12 +434,15 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -440,6 +450,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -483,6 +494,7 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 5b87fcf..f879914 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -21,7 +21,7 @@ import ( "bytes" "encoding/hex" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "strings" @@ -51,7 +51,7 @@ func TestSubDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -61,7 +61,7 @@ func TestSubDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -76,7 +76,7 @@ func TestSubDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -85,7 +85,7 @@ func TestSubDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -110,7 +110,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -120,7 +120,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -140,7 +140,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -149,7 +149,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -167,7 +167,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req, err = http.NewRequest("DELETE", url, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -176,7 +176,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -185,7 +185,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -217,7 +217,7 @@ func TestPostWithDeviceId(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -229,7 +229,7 @@ func TestPostWithDeviceId(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -249,7 +249,7 @@ func TestPostWithDeviceId(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -260,7 +260,7 @@ func TestPostWithDeviceId(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -292,7 +292,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -304,7 +304,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) assert.Equal(t, reqHeaderVersion, resHeaderVersion) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, gwrestoreBytes) @@ -333,7 +333,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -348,7 +348,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, remotedebuggerBytes) @@ -367,7 +367,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -377,7 +377,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -392,7 +392,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -426,7 +426,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { ifnonematch := strings.Join(subdocVersions, ",") req.Header.Set(common.HeaderIfNoneMatch, ifnonematch) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -452,7 +452,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -464,7 +464,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) assert.Equal(t, reqHeaderVersion, resHeaderVersion) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, gwrestoreBytes) @@ -493,7 +493,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -508,7 +508,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, remotedebuggerBytes) @@ -527,7 +527,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -537,7 +537,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -552,7 +552,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -580,7 +580,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { ifnonematch := strings.Join(subdocVersions, ",") req.Header.Set(common.HeaderIfNoneMatch, ifnonematch) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -609,7 +609,7 @@ func TestBadHeaderExpiryHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) } diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go new file mode 100644 index 0000000..03ce19c --- /dev/null +++ b/http/factory_reset_upstream_test.go @@ -0,0 +1,272 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestFactoryResetWithoutUpstream(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(false) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // verify Document is now empty + fields := make(log.Fields) + _, err = server.GetDocument(cpeMac, fields) + assert.Assert(t, server.IsDbNotFound(err)) +} + +func TestFactoryResetWithUpstream(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + // lanVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== setup mock upstream server ==== + fields := make(log.Fields) + mockDoc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + + mockRootDoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + mockDoc.SetRootDocument(mockRootDoc) + + mockDoc.DeleteSubDocument("wan") + assert.NilError(t, err) + + mockBytes, err := mockDoc.Bytes() + assert.NilError(t, err) + + db.RefreshRootDocumentVersion(mockDoc) + refreshedRootVersion := mockDoc.RootVersion() + + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.Header().Set(common.HeaderContentLength, strconv.Itoa(len(mockBytes))) + ifNoneMatch := refreshedRootVersion + w.Header().Set(common.HeaderEtag, ifNoneMatch) + w.WriteHeader(http.StatusOK) + w.Write(mockBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(true) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, !ok) + + // verify Document is now empty + doc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + assert.Equal(t, doc.Length(), 1) +} diff --git a/http/http_client.go b/http/http_client.go index 833c333..dfc0459 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -25,7 +25,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "net/http" neturl "net/url" @@ -260,7 +259,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] } fields[fmt.Sprintf("%v_status", loggerName)] = res.StatusCode - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) if err != nil { fields[errorKey] = err.Error() if userAgent != "mget" { diff --git a/http/multipart.go b/http/multipart.go index 6c1c663..81b42bf 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -116,6 +116,16 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin respHeader := make(http.Header) userAgent := rHeader.Get("User-Agent") + // factory reset handling + ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + if ifNoneMatch == "NONE" || ifNoneMatch == "NONE-REBOOT" { + status, respHeader, rbytes, err := BuildFactoryResetResponse(s, rHeader, fields) + if err != nil { + return status, respHeader, rbytes, common.NewError(err) + } + return status, respHeader, rbytes, nil + } + document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, err := db.BuildGetDocument(c, rHeader, route, fields) if uconn == nil { if err != nil { @@ -263,7 +273,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalFilteredDocument, document, fields) + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalFilteredDocument, document, false, fields) if err != nil { return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) } @@ -280,3 +290,111 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusOK, upstreamRespHeader, finalFilteredBytes, nil } + +func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { + c := s.DatabaseClient + uconn := s.GetUpstreamConnector() + mac := rHeader.Get(common.HeaderDeviceId) + respHeader := make(http.Header) + + document, err := c.GetDocument(mac, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + rootDocument, err := c.GetRootDocument(mac) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + document.SetRootDocument(rootDocument) + oldDocBytes, err := document.Bytes() + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + + if uconn == nil { + err := c.DeleteDocument(mac) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + return http.StatusNotFound, respHeader, nil, nil + } + + // handle partner + partnerId := rHeader.Get(common.HeaderPartnerID) + if len(partnerId) == 0 && rootDocument != nil { + partnerId = rootDocument.PartnerId + } + + // ============================= + // upstream handling + // ============================= + upstreamHeader := make(http.Header) + upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) + upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) + if itf, ok := fields["audit_id"]; ok { + auditId := itf.(string) + if len(auditId) > 0 { + upstreamHeader.Set(common.HeaderAuditid, auditId) + } + } + if x := rHeader.Get(common.HeaderTransactionId); len(x) > 0 { + upstreamHeader.Set(common.HeaderTransactionId, x) + } + + if s.TokenManager != nil { + token := rHeader.Get("Authorization") + if len(token) > 0 { + upstreamHeader.Set("Authorization", token) + } else { + token = s.Generate(mac, 86400) + upstreamHeader.Set("Authorization", "Bearer "+token) + } + } + + // call /upstream to handle factory reset + ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + upstreamHeader.Set(common.HeaderIfNoneMatch, ifNoneMatch) + + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) + if err != nil { + var rherr common.RemoteHttpError + if errors.As(err, &rherr) { + return rherr.StatusCode, respHeader, nil, common.NewError(err) + } + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + + // ==== parse the upstreamRespBytes and store them ==== + finalMparts, err := util.ParseMultipartAsList(upstreamRespHeader, upstreamRespBytes) + if err != nil { + return http.StatusInternalServerError, respHeader, oldDocBytes, common.NewError(err) + } + upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) + + // filter by versionMap and filter by blockedIds + finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") + finalDocument := common.NewDocument(finalRootDocument) + finalDocument.SetSubDocuments(finalMparts) + for _, subdocId := range c.BlockedSubdocIds() { + finalDocument.DeleteSubDocument(subdocId) + } + + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } + + // 304 + if finalDocument.Length() == 0 { + return http.StatusNotModified, upstreamRespHeader, nil, nil + } + + finalBytes, err := finalDocument.Bytes() + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) + } + + return http.StatusOK, upstreamRespHeader, finalBytes, nil +} diff --git a/http/multipart_test.go b/http/multipart_test.go index e4ebd4c..0dc6be5 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -21,7 +21,7 @@ import ( "bytes" "encoding/hex" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strings" @@ -52,7 +52,7 @@ func TestMultipartConfigHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -62,7 +62,7 @@ func TestMultipartConfigHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -81,7 +81,7 @@ func TestMultipartConfigHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -91,7 +91,7 @@ func TestMultipartConfigHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -102,7 +102,7 @@ func TestMultipartConfigHandler(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -139,7 +139,7 @@ func TestMultipartConfigHandler(t *testing.T) { assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, etag) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -151,7 +151,7 @@ func TestMultipartConfigHandler(t *testing.T) { ifNoneMatch := fmt.Sprintf("foo,%v,bar", lanMpartVersion) req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -196,7 +196,7 @@ func TestCpeMiddleware(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -206,7 +206,7 @@ func TestCpeMiddleware(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -220,7 +220,7 @@ func TestCpeMiddleware(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router1).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusForbidden) @@ -231,7 +231,7 @@ func TestCpeMiddleware(t *testing.T) { assert.NilError(t, err) req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) res = ExecuteRequest(req, router1).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -253,7 +253,7 @@ func TestVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -263,7 +263,7 @@ func TestVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -279,7 +279,7 @@ func TestVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -289,7 +289,7 @@ func TestVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -300,7 +300,7 @@ func TestVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -325,7 +325,7 @@ func TestVersionFiltering(t *testing.T) { assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, etag) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -337,7 +337,7 @@ func TestVersionFiltering(t *testing.T) { ifNoneMatch := fmt.Sprintf("foo,%v,bar", lanMpartVersion) req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -397,7 +397,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -407,7 +407,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -423,7 +423,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -433,7 +433,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -444,7 +444,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -472,7 +472,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusServiceUnavailable) @@ -481,7 +481,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusServiceUnavailable) @@ -494,7 +494,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { w.Header().Set(k, r.Header.Get(k)) } w.WriteHeader(http.StatusOK) - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { _, err := w.Write(rbytes) assert.NilError(t, err) } @@ -509,7 +509,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -531,7 +531,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderIfNoneMatch, matchedIfNoneMatch) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -542,7 +542,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderIfNoneMatch, mismatchedIfNoneMatch) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -572,7 +572,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -582,7 +582,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -598,7 +598,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -608,7 +608,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -669,7 +669,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { w.Header().Set(k, r.Header.Get(k)) } w.WriteHeader(http.StatusOK) - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { _, err := w.Write(rbytes) assert.NilError(t, err) } @@ -749,7 +749,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -759,7 +759,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -776,7 +776,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -786,7 +786,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -797,7 +797,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -826,7 +826,7 @@ func TestMultipartConfigMismatch(t *testing.T) { header1 := "NONE," + lanMpartVersion req.Header.Set(common.HeaderIfNoneMatch, header1) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -837,7 +837,7 @@ func TestMultipartConfigMismatch(t *testing.T) { header2 := "NONE,123" req.Header.Set(common.HeaderIfNoneMatch, header2) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -848,7 +848,7 @@ func TestMultipartConfigMismatch(t *testing.T) { header3 := etag + ",123" req.Header.Set(common.HeaderIfNoneMatch, header3) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index bbd0884..29c3364 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -20,7 +20,7 @@ package http import ( "bytes" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strconv" @@ -62,7 +62,7 @@ func TestPokeHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusNoContent) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -89,7 +89,7 @@ func TestPokeHandlerWithCpe(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() } @@ -121,7 +121,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -130,7 +130,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -153,7 +153,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -162,7 +162,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -189,7 +189,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req, err = http.NewRequest("POST", mqttUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusAccepted) @@ -198,7 +198,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -210,7 +210,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -229,7 +229,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -238,7 +238,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes2) @@ -255,7 +255,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req, err = http.NewRequest("POST", mqttUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusAccepted) @@ -264,7 +264,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -301,7 +301,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusBadRequest) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() } diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 5debc71..7bb66c6 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -22,7 +22,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "testing" @@ -51,7 +51,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/json") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -97,7 +97,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -106,7 +106,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -124,7 +124,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -133,7 +133,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -147,7 +147,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set(common.HeaderPartnerID, partner1) req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.Equal(t, res.StatusCode, http.StatusOK) assert.NilError(t, err) res.Body.Close() @@ -182,7 +182,7 @@ func TestRootDocumentHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -223,7 +223,7 @@ func TestPostRootDocumentHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() diff --git a/http/simple_handler_test.go b/http/simple_handler_test.go index 0e4e093..28109fa 100644 --- a/http/simple_handler_test.go +++ b/http/simple_handler_test.go @@ -18,7 +18,7 @@ package http import ( - "io/ioutil" + "io" "net/http" "testing" @@ -36,7 +36,7 @@ func TestSimpleHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -45,7 +45,7 @@ func TestSimpleHandler(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, len(rbytes), 0) @@ -56,7 +56,7 @@ func TestSimpleHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, len(rbytes), 0) @@ -70,7 +70,7 @@ func TestNotifHandler(t *testing.T) { req, err := http.NewRequest("GET", "/notif", nil) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) res.Body.Close() diff --git a/http/validator_test.go b/http/validator_test.go index f368804..8d36dcf 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -22,7 +22,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" @@ -49,7 +49,7 @@ func TestValidatorDisabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -58,7 +58,7 @@ func TestValidatorDisabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -75,7 +75,7 @@ func TestValidatorDisabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -89,7 +89,7 @@ func TestValidatorDisabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -98,7 +98,7 @@ func TestValidatorDisabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -113,7 +113,7 @@ func TestValidatorDisabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -142,7 +142,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -152,7 +152,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -162,7 +162,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -172,7 +172,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -188,7 +188,7 @@ func TestValidatorEnabled(t *testing.T) { req, err = http.NewRequest("GET", rootdocUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -199,7 +199,7 @@ func TestValidatorEnabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -214,7 +214,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -224,7 +224,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -233,7 +233,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -248,7 +248,7 @@ func TestValidatorEnabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -275,7 +275,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -284,7 +284,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -304,7 +304,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -313,7 +313,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -331,7 +331,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 94eed50..0c2d146 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -22,14 +22,13 @@ import ( "crypto/tls" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "os" "strings" "time" "github.com/rdkcentral/webconfig/common" - owcommon "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" @@ -301,7 +300,7 @@ func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { if r.Method == "POST" { if r.Body != nil { - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { xw.SetBodyBytes(rbytes) } } @@ -625,7 +624,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } // extract traceparent from the header - traceparent := r.Header.Get(owcommon.HeaderTraceparent) + traceparent := r.Header.Get(common.HeaderTraceparent) if len(traceparent) == 55 { traceId = traceparent[3:35] outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] @@ -698,7 +697,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if r.Method == "POST" { if r.Body != nil { - bbytes, err := ioutil.ReadAll(r.Body) + bbytes, err := io.ReadAll(r.Body) if err != nil { fields["error"] = err log.WithFields(fields).Error("request starts") diff --git a/kafka/consumer.go b/kafka/consumer.go index b97268c..1135d5d 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" wchttp "github.com/rdkcentral/webconfig/http" @@ -170,7 +170,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram // NOTE: // Do not move the code below to a goroutine. // The `ConsumeClaim` itself is called within a goroutine, see: - // https://github.com/Shopify/sarama/blob/master/consumer_group.go#L27-L29 + // https://github.com/IBM/sarama/blob/master/consumer_group.go#L27-L29 rl := ratelimit.New(c.ratelimitMessagesPerSecond, ratelimit.WithoutSlack) // per second, no slack. for { diff --git a/kafka/consumer_group_test.go b/kafka/consumer_group_test.go index 5db165f..d720eb2 100644 --- a/kafka/consumer_group_test.go +++ b/kafka/consumer_group_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "gotest.tools/assert" ) diff --git a/kafka/kafka_consumer_group.go b/kafka/kafka_consumer_group.go index 298a3a2..c768af7 100644 --- a/kafka/kafka_consumer_group.go +++ b/kafka/kafka_consumer_group.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" wchttp "github.com/rdkcentral/webconfig/http" diff --git a/main.go b/main.go index 4c32552..11c8893 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,7 @@ import ( "os/signal" "syscall" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" wchttp "github.com/rdkcentral/webconfig/http" "github.com/rdkcentral/webconfig/kafka" diff --git a/security/token.go b/security/token.go index 52592e0..51695e2 100644 --- a/security/token.go +++ b/security/token.go @@ -22,7 +22,7 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io/ioutil" + "os" "strings" "time" @@ -110,7 +110,7 @@ func NewTokenManager(conf *configuration.Config) *TokenManager { } func loadDecodeKey(keyfile string) (*rsa.PublicKey, error) { - kbytes, err := ioutil.ReadFile(keyfile) + kbytes, err := os.ReadFile(keyfile) if err != nil { return nil, common.NewError(err) } @@ -122,7 +122,7 @@ func loadDecodeKey(keyfile string) (*rsa.PublicKey, error) { } func loadEncodeKey(keyfile string) (*rsa.PrivateKey, error) { - kbytes, err := ioutil.ReadFile(keyfile) + kbytes, err := os.ReadFile(keyfile) if err != nil { return nil, common.NewError(err) } From 521a46df5040dacde541d7f064caa97c8a7f7917 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 24 Jan 2024 22:27:00 -0800 Subject: [PATCH 007/215] fix a bug that factory reset NONE returns 500 when no data in db --- http/factory_reset_upstream_test.go | 19 +++++++++++++++++++ http/multipart.go | 3 +++ 2 files changed, 22 insertions(+) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 03ce19c..3f587a8 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -33,6 +33,25 @@ import ( "gotest.tools/assert" ) +func TestFactoryResetWithoutData(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetUpstreamEnabled(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== no data ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} + func TestFactoryResetWithoutUpstream(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) diff --git a/http/multipart.go b/http/multipart.go index 81b42bf..287d5aa 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -299,6 +299,9 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { + if s.IsDbNotFound(err) { + return http.StatusNotFound, respHeader, nil, nil + } return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } rootDocument, err := c.GetRootDocument(mac) From b5a355481fa2d93cd1a34fc635ced632d8a12de2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 7 Feb 2024 23:12:42 -0800 Subject: [PATCH 008/215] Fix a bug that 304 was returned when all subdocs were wiped out on factory reset header none/none-reboot --- http/multipart.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 287d5aa..6117b85 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -389,9 +389,8 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) } - // 304 if finalDocument.Length() == 0 { - return http.StatusNotModified, upstreamRespHeader, nil, nil + return http.StatusOK, upstreamRespHeader, nil, nil } finalBytes, err := finalDocument.Bytes() From 898df13dbd4dbddbd80ea0531de295c349b6d9b6 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 8 Feb 2024 16:27:12 -0800 Subject: [PATCH 009/215] return 200 empty on if-none-match:none and no data in db --- http/factory_reset_upstream_test.go | 2 +- http/multipart.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 3f587a8..1d0e7bf 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -49,7 +49,7 @@ func TestFactoryResetWithoutData(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusNotFound) + assert.Equal(t, res.StatusCode, http.StatusOK) } func TestFactoryResetWithoutUpstream(t *testing.T) { diff --git a/http/multipart.go b/http/multipart.go index 6117b85..595c0ab 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -300,7 +300,8 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { if s.IsDbNotFound(err) { - return http.StatusNotFound, respHeader, nil, nil + // return http.StatusNotFound, respHeader, nil, nil + return http.StatusOK, respHeader, nil, nil } return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } From 3d44fe21d5d352dcefcf48be2c3080a090aac30b Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 8 Feb 2024 23:31:56 -0800 Subject: [PATCH 010/215] fix a bug that root_document was not updated when if-none-match:NONE-REBOOT --- db/service.go | 39 +++++++++++++++++++++++++++++ http/factory_reset_upstream_test.go | 16 ++++++++++++ http/multipart.go | 23 +++++++++-------- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/db/service.go b/db/service.go index 4c2a598..582c94e 100644 --- a/db/service.go +++ b/db/service.go @@ -480,3 +480,42 @@ func RefreshRootDocumentVersion(doc *common.Document) { rootDoc.Version = rootVersion } } + +func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerId string, fields log.Fields) (*common.RootDocument, error) { + fieldsDict := make(util.Dict) + fieldsDict.Update(fields) + + // ==== deviceRootDocument should always be created from request header ==== + var bitmap int + var err error + supportedDocs := rHeader.Get(common.HeaderSupportedDocs) + if len(supportedDocs) > 0 { + bitmap, err = util.GetCpeBitmap(supportedDocs) + if err != nil { + log.WithFields(fields).Warn(common.NewError(err)) + } + } + + schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) + modelName := rHeader.Get(common.HeaderModelName) + firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + + // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers + deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") + + // ==== read the cloudRootDocument from db ==== + cloudRootDocument, err := c.GetRootDocument(mac) + if err != nil { + if !c.IsDbNotFound(err) { + return cloudRootDocument, common.NewError(err) + } + cloudRootDocument = deviceRootDocument.Clone() + } else { + cloudRootDocument.Update(deviceRootDocument) + } + + if err := c.SetRootDocument(mac, cloudRootDocument); err != nil { + return cloudRootDocument, common.NewError(err) + } + return cloudRootDocument, nil +} diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 1d0e7bf..429814a 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -41,15 +41,31 @@ func TestFactoryResetWithoutData(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() // ==== no data ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err := http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) + + rootDocument, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + + assert.Equal(t, rootDocument.Bitmap, 32479) } func TestFactoryResetWithoutUpstream(t *testing.T) { diff --git a/http/multipart.go b/http/multipart.go index 595c0ab..fefc94d 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -297,6 +297,18 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l mac := rHeader.Get(common.HeaderDeviceId) respHeader := make(http.Header) + fieldsDict := make(util.Dict) + fieldsDict.Update(fields) + partnerId := rHeader.Get(common.HeaderPartnerID) + if len(partnerId) == 0 { + partnerId = fieldsDict.GetString("partner") + } + + rootDocument, err := db.PreprocessRootDocument(c, rHeader, mac, partnerId, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + document, err := c.GetDocument(mac, fields) if err != nil { if s.IsDbNotFound(err) { @@ -305,10 +317,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - rootDocument, err := c.GetRootDocument(mac) - if err != nil { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) - } + document.SetRootDocument(rootDocument) oldDocBytes, err := document.Bytes() if err != nil { @@ -323,12 +332,6 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusNotFound, respHeader, nil, nil } - // handle partner - partnerId := rHeader.Get(common.HeaderPartnerID) - if len(partnerId) == 0 && rootDocument != nil { - partnerId = rootDocument.PartnerId - } - // ============================= // upstream handling // ============================= From a9933b51889f698ef1cf35a15fc769c46b88ee68 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 12 Feb 2024 12:38:55 -0800 Subject: [PATCH 011/215] change the none-reboot no subdoc response from 200 to 404 --- http/factory_reset_upstream_test.go | 2 +- http/multipart.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 429814a..e697661 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -60,7 +60,7 @@ func TestFactoryResetWithoutData(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusOK) + assert.Equal(t, res.StatusCode, http.StatusNotFound) rootDocument, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) diff --git a/http/multipart.go b/http/multipart.go index fefc94d..154beb6 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -312,8 +312,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { if s.IsDbNotFound(err) { - // return http.StatusNotFound, respHeader, nil, nil - return http.StatusOK, respHeader, nil, nil + return http.StatusNotFound, respHeader, nil, nil } return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } @@ -394,7 +393,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } if finalDocument.Length() == 0 { - return http.StatusOK, upstreamRespHeader, nil, nil + return http.StatusNotFound, upstreamRespHeader, nil, nil } finalBytes, err := finalDocument.Bytes() From aa019fb9fde387f1f8a9f7d3b6ba3202c08aa4c8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 15 Feb 2024 13:47:55 -0800 Subject: [PATCH 012/215] always call upstream on factory reset none-reboot --- common/document.go | 4 ++ http/factory_reset_upstream_test.go | 85 ++++++++++++++++++++++++++++- http/multipart.go | 11 ++-- 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/common/document.go b/common/document.go index f400945..61b054c 100644 --- a/common/document.go +++ b/common/document.go @@ -148,6 +148,10 @@ func (d *Document) FilterForGet(versionMap map[string]string) *Document { } func (d *Document) Bytes() ([]byte, error) { + if len(d.docmap) == 0 { + return nil, nil + } + // build the http stream mparts := []Multipart{} for subdocId, subdoc := range d.docmap { diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index e697661..f6bbe0c 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -25,6 +25,7 @@ import ( "net/http/httptest" "strconv" "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" @@ -40,6 +41,25 @@ func TestFactoryResetWithoutData(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() + // ==== setup upstream mock server ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + if k == common.HeaderContentLength { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + w.WriteHeader(http.StatusNotFound) + w.Write(nil) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + // ==== no data ==== supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" @@ -64,7 +84,6 @@ func TestFactoryResetWithoutData(t *testing.T) { rootDocument, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - assert.Equal(t, rootDocument.Bitmap, 32479) } @@ -305,3 +324,67 @@ func TestFactoryResetWithUpstream(t *testing.T) { assert.NilError(t, err) assert.Equal(t, doc.Length(), 1) } + +func TestFactoryResetUpstreamAddData(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetUpstreamEnabled(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup upstream mock server ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for k := range r.Header { + if k == common.HeaderContentLength { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + + // add a subdoc from upsream + mparts := []common.Multipart{ + { + Bytes: util.RandomBytes(100, 150), + Version: strconv.Itoa(int(time.Now().Unix())), + Name: "network", + State: common.PendingDownload, + }, + } + respBytes, err := common.WriteMultipartBytes(mparts) + assert.NilError(t, err) + w.WriteHeader(http.StatusOK) + w.Write(respBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== no data ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + rootDocument, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, rootDocument.Bitmap, 32479) +} diff --git a/http/multipart.go b/http/multipart.go index 154beb6..ea54b8c 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -311,13 +311,16 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { - if s.IsDbNotFound(err) { - return http.StatusNotFound, respHeader, nil, nil + if !s.IsDbNotFound(err) { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + if document == nil { + document = common.NewDocument(rootDocument) + } else { + document.SetRootDocument(rootDocument) } - document.SetRootDocument(rootDocument) oldDocBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) From 73fa2d5c46b4bc07b08657dc915462e8327d8006 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 26 Feb 2024 19:29:40 -0800 Subject: [PATCH 013/215] fix a bug that auto migration did not update xpc_group_config table when devices reporting same versions as the newly automigrated versions --- common/document.go | 3 ++ common/document_test.go | 46 +++++++++++++++++++++++++++++-- common/subdocument.go | 34 +++++++++++------------ db/cassandra/document_test.go | 18 ++++++++---- db/cassandra/state_update_test.go | 6 ++-- db/service.go | 14 ++++++++-- db/sqlite/document_test.go | 12 +++++--- db/sqlite/state_metrics_test.go | 3 +- http/multipart.go | 13 +++++---- 9 files changed, 108 insertions(+), 41 deletions(-) diff --git a/common/document.go b/common/document.go index 61b054c..d99f245 100644 --- a/common/document.go +++ b/common/document.go @@ -105,6 +105,9 @@ func (d *Document) GetRootDocument() *RootDocument { } func (d *Document) RootVersion() string { + if d.rootDocument == nil { + return "" + } return d.rootDocument.Version } diff --git a/common/document_test.go b/common/document_test.go index 26caeb7..bb222d9 100644 --- a/common/document_test.go +++ b/common/document_test.go @@ -18,6 +18,9 @@ package common import ( + crand "crypto/rand" + "math/rand" + "strconv" "testing" "time" @@ -25,7 +28,44 @@ import ( ) func TestDocument(t *testing.T) { - // TODO a place holder for now - ts := int(time.Now().UnixNano() / 1000000) - assert.Assert(t, ts > 0) + document := NewDocument(nil) + assert.Equal(t, len(document.RootVersion()), 0) + + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + document = NewDocument(rootdoc) + + subdocIds := []string{"red", "orange", "yellow", "green"} + mparts := []Multipart{} + versionMap := make(map[string]string) + for _, subdocId := range subdocIds { + blen := rand.Intn(10) + 10 + bbytes := make([]byte, blen) + crand.Read(bbytes) + version := strconv.Itoa(int(time.Now().Unix())) + mpart := Multipart{ + Bytes: bbytes, + Version: version, + Name: subdocId, + State: Deployed, + } + mparts = append(mparts, mpart) + versionMap[subdocId] = version + } + + document.SetSubDocuments(mparts) + assert.Equal(t, len(document.Items()), len(subdocIds)) + + filteredDocument := document.FilterForGet(versionMap) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 0) + + filteredDocument = document.FilterForGet(nil) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) } diff --git a/common/subdocument.go b/common/subdocument.go index 836eb37..336b22b 100644 --- a/common/subdocument.go +++ b/common/subdocument.go @@ -142,59 +142,59 @@ func (d *SubDocument) SetExpiry(expiry *int) { d.expiry = expiry } -func (d *SubDocument) Equals(tdoc *SubDocument) error { +func (d *SubDocument) Equals(tdoc *SubDocument) (bool, error) { if d.HasPayload() && tdoc.HasPayload() { if !bytes.Equal(d.Payload(), tdoc.Payload()) { - err := fmt.Errorf("d.Payload() != tdoc.Payload(), len(d.Payload())=%v, len(tdoc.Payload())=%v", len(d.payload), len(tdoc.payload)) - return NewError(err) + err := fmt.Errorf("d.Payload() != tdoc.Payload(), d.Payload()=%v, tdoc.Payload()=%v", d.payload, tdoc.payload) + return false, NewError(err) } } else { if d.HasPayload() != tdoc.HasPayload() { err := fmt.Errorf("d.HasPayload() != tdoc.HasPayload()") - return NewError(err) + return false, NewError(err) } } if d.Version() != nil && tdoc.Version() != nil { if *d.Version() != *tdoc.Version() { err := fmt.Errorf("*d.Version()[%v] != *tdoc.Version()[%v]", *d.Version(), *tdoc.Version()) - return NewError(err) + return false, NewError(err) } } else { if d.Version() != tdoc.Version() { err := fmt.Errorf("d.Version()[%v] != tdoc.Version()[%v]", d.Version(), tdoc.Version()) - return NewError(err) + return false, NewError(err) } } if d.UpdatedTime() != nil && tdoc.UpdatedTime() != nil { if *d.UpdatedTime() != *tdoc.UpdatedTime() { err := fmt.Errorf("*d.UpdatedTime()[%v] != *tdoc.UpdatedTime()[%v]", *d.UpdatedTime(), *tdoc.UpdatedTime()) - return NewError(err) + return false, NewError(err) } } else { if d.UpdatedTime() != tdoc.UpdatedTime() { err := fmt.Errorf("d.UpdatedTime()[%v] != tdoc.UpdatedTime()[%v]", d.UpdatedTime(), tdoc.UpdatedTime()) - return NewError(err) + return false, NewError(err) } } if d.State() != nil && tdoc.State() != nil { if *d.State() != *tdoc.State() { err := fmt.Errorf("*d.State()[%v] != *tdoc.State()[%v]", *d.State(), *tdoc.State()) - return NewError(err) + return false, NewError(err) } } else { if d.State() != tdoc.State() { err := fmt.Errorf("d.State()[%v] != tdoc.State()[%v]", d.State(), tdoc.State()) - return NewError(err) + return false, NewError(err) } } if d.ErrorCode() != nil && tdoc.ErrorCode() != nil { if *d.ErrorCode() != *tdoc.ErrorCode() { err := fmt.Errorf("*d.ErrorCode()[%v] != *tdoc.ErrorCode()[%v]", *d.ErrorCode(), *tdoc.ErrorCode()) - return NewError(err) + return false, NewError(err) } } else { var dErrorCode, tdocErrorCode int @@ -206,14 +206,14 @@ func (d *SubDocument) Equals(tdoc *SubDocument) error { } if dErrorCode != tdocErrorCode { err := fmt.Errorf("d.ErrorCode()[%v] != tdoc.ErrorCode()[%v]", d.ErrorCode(), tdoc.ErrorCode()) - return NewError(err) + return false, NewError(err) } } if d.ErrorDetails() != nil && tdoc.ErrorDetails() != nil { if *d.ErrorDetails() != *tdoc.ErrorDetails() { err := fmt.Errorf("*d.ErrorDetails()[%v] != *tdoc.ErrorDetails()[%v]", *d.ErrorDetails(), *tdoc.ErrorDetails()) - return NewError(err) + return false, NewError(err) } } else { var dErrorDetails, tdocErrorDetails string @@ -225,23 +225,23 @@ func (d *SubDocument) Equals(tdoc *SubDocument) error { } if dErrorDetails != tdocErrorDetails { err := fmt.Errorf("d.ErrorDetails()[%v] != tdoc.ErrorDetails()[%v]", d.ErrorDetails(), tdoc.ErrorDetails()) - return NewError(err) + return false, NewError(err) } } if d.Expiry() != nil && tdoc.Expiry() != nil { if *d.Expiry() != *tdoc.Expiry() { err := fmt.Errorf("*d.Expiry()[%v] != *tdoc.Expiry()[%v]", *d.Expiry(), *tdoc.Expiry()) - return NewError(err) + return false, NewError(err) } } else { if d.Expiry() != tdoc.Expiry() { err := fmt.Errorf("d.Expiry()[%v] != tdoc.Expiry()[%v]", d.Expiry(), tdoc.Expiry()) - return NewError(err) + return false, NewError(err) } } - return nil + return true, nil } func (d *SubDocument) NeedsUpdateForHttp304() bool { diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index 24bf0ac..b314690 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -57,7 +57,9 @@ func TestMocaSubDocument(t *testing.T) { fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) assert.NilError(t, err) - assert.Assert(t, srcSubdoc.Equals(fetchedSubdoc)) + ok, err := srcSubdoc.Equals(fetchedSubdoc) + assert.NilError(t, err) + assert.Assert(t, ok) } func TestPrivatessidSubDocument(t *testing.T) { @@ -82,7 +84,9 @@ func TestPrivatessidSubDocument(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - assert.Assert(t, srcDoc.Equals(fetchedDoc)) + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) } func TestMultiSubDocuments(t *testing.T) { @@ -114,8 +118,9 @@ func TestMultiSubDocuments(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) } doc, err := tdbclient.GetDocument(cpeMac) @@ -125,7 +130,9 @@ func TestMultiSubDocuments(t *testing.T) { for k, v := range srcmap { dv := doc.SubDocument(k) assert.Assert(t, dv != nil) - assert.Assert(t, v.Equals(dv)) + ok, err := v.Equals(dv) + assert.NilError(t, err) + assert.Assert(t, ok) } // ==== delete a document ==== @@ -178,8 +185,9 @@ func TestBlockedSubdocIds(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) } // add version1 and bitmap1 version1 := "indigo violet" diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 4b57677..0857770 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -48,8 +48,9 @@ func TestStateUpdate1(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // update to state failure template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` @@ -98,8 +99,9 @@ func TestStateUpdate2(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // update to state failure template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` diff --git a/db/service.go b/db/service.go index 582c94e..2cc326d 100644 --- a/db/service.go +++ b/db/service.go @@ -369,7 +369,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage return nil } -func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, fields log.Fields) error { +func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { var oldState int if oldSubdoc != nil && oldSubdoc.State() != nil { oldState = *oldSubdoc.State() @@ -384,6 +384,14 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old updatedTime := int(time.Now().UnixNano() / 1000000) newSubdoc.SetUpdatedTime(&updatedTime) newState := common.InDeployment + if oldVersion, ok := versionMap[subdocId]; ok { + if newSubdoc.Version() != nil { + if oldVersion == *newSubdoc.Version() { + newState = common.Deployed + } + } + } + newSubdoc.SetState(&newState) err = c.SetSubDocument(cpeMac, subdocId, newSubdoc, oldState, labels, fields) if err != nil { @@ -392,7 +400,7 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old return nil } -func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, toDelete bool, fields log.Fields) error { +func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, toDelete bool, versionMap map[string]string, fields log.Fields) error { newRootVersion := upstreamRespEtag if d.RootVersion() != newRootVersion { err := c.SetRootDocumentVersion(cpeMac, newRootVersion) @@ -409,7 +417,7 @@ func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string // need to set "state" to proper values like the download is complete for subdocId, newSubdoc := range newDoc.Items() { oldSubdoc := d.SubDocument(subdocId) - err := UpdateSubDocument(c, cpeMac, subdocId, &newSubdoc, oldSubdoc, fields) + err := UpdateSubDocument(c, cpeMac, subdocId, &newSubdoc, oldSubdoc, versionMap, fields) if err != nil { return common.NewError(err) } diff --git a/db/sqlite/document_test.go b/db/sqlite/document_test.go index a633ee3..ea6e15e 100644 --- a/db/sqlite/document_test.go +++ b/db/sqlite/document_test.go @@ -48,8 +48,9 @@ func TestSubDocumentDb(t *testing.T) { // read a SubDocument from db and verify identical targetSubDocument, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = sourceDoc.Equals(targetSubDocument) + ok, err := sourceDoc.Equals(targetSubDocument) assert.NilError(t, err) + assert.Assert(t, ok) // ==== update an existing doc with the same cpeMac and groupId ==== srcVersion2 := "red white blue" @@ -61,8 +62,9 @@ func TestSubDocumentDb(t *testing.T) { assert.NilError(t, err) expectedDoc := common.NewSubDocument(srcBytes, &srcVersion2, &srcState, &srcUpdatedTime, nil, nil) - err = targetSubDocument.Equals(expectedDoc) + ok, err = targetSubDocument.Equals(expectedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // ==== delete a doc ==== err = tdbclient.DeleteSubDocument(cpeMac, groupId) @@ -106,10 +108,12 @@ func TestDbReadDocument(t *testing.T) { assert.NilError(t, err) assert.Equal(t, Document.Length(), 2) - err = pdoc.Equals(Document.SubDocument("privatessid")) + ok, err := pdoc.Equals(Document.SubDocument("privatessid")) assert.NilError(t, err) - err = hdoc.Equals(Document.SubDocument("homessid")) + assert.Assert(t, ok) + ok, err = hdoc.Equals(Document.SubDocument("homessid")) assert.NilError(t, err) + assert.Assert(t, ok) // ==== delete all SubDocuments ==== err = tdbclient.DeleteDocument(cpeMac) diff --git a/db/sqlite/state_metrics_test.go b/db/sqlite/state_metrics_test.go index d8bcb02..bb2cb00 100644 --- a/db/sqlite/state_metrics_test.go +++ b/db/sqlite/state_metrics_test.go @@ -65,8 +65,9 @@ func TestStateMetrics(t *testing.T) { // read a SubDocument from db and verify identical doc1, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = sourceDoc.Equals(doc1) + ok, err := sourceDoc.Equals(doc1) assert.NilError(t, err) + assert.Assert(t, ok) // ==== update an doc with the same cpeMac and a changed state ==== state2 := common.InDeployment diff --git a/http/multipart.go b/http/multipart.go index ea54b8c..5051082 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -267,17 +267,18 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) - finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) - for _, subdocId := range c.BlockedSubdocIds() { - finalFilteredDocument.DeleteSubDocument(subdocId) - } // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalFilteredDocument, document, false, fields) + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) if err != nil { return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) } + finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) + for _, subdocId := range c.BlockedSubdocIds() { + finalFilteredDocument.DeleteSubDocument(subdocId) + } + // 304 if finalFilteredDocument.Length() == 0 { return http.StatusNotModified, upstreamRespHeader, nil, nil @@ -390,7 +391,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, fields) + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) if err != nil { return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) } From 51e6afe0d3f814dcc1132ab2fdc88a5184a3b020 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 1 Mar 2024 09:41:30 -0800 Subject: [PATCH 014/215] return updated root_version in POST /document call --- common/error.go | 33 +++++++++++++++++++++++++-------- http/document_handler.go | 13 +++++++++++-- http/main_test.go | 3 +-- http/webconfig_server.go | 6 ++---- security/token.go | 4 ++-- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/common/error.go b/common/error.go index 0bb87e5..e733968 100644 --- a/common/error.go +++ b/common/error.go @@ -22,11 +22,16 @@ import ( "fmt" "path/filepath" "runtime" + "slices" + "strings" +) + +const ( + defaultTracebackLevel = 3 ) var ( - NotOK = fmt.Errorf("!ok") - ProfileNotFound = fmt.Errorf("profile not found") + ErrNotOK = fmt.Errorf("!ok") ) type Http400Error struct { @@ -121,12 +126,24 @@ func UnwrapAll(wrappedErr error) error { return err } -func GetCaller() string { - _, file, line, _ := runtime.Caller(1) - filename := filepath.Base(file) - fulldir := filepath.Dir(file) - dir := filepath.Base(fulldir) - return fmt.Sprintf("%v/%v[%v]", dir, filename, line) +func GetCaller(args ...int) string { + items := []string{} + + tracebackLevel := defaultTracebackLevel + if len(args) > 0 { + tracebackLevel = args[0] + } + + for i := 0; i < tracebackLevel; i++ { + _, file, line, _ := runtime.Caller(i + 1) + filename := filepath.Base(file) + fulldir := filepath.Dir(file) + dir := filepath.Base(fulldir) + items = append(items, fmt.Sprintf("%v/%v[%v]", dir, filename, line)) + } + + slices.Reverse(items) + return strings.Join(items, " ") } type NoCapabilitiesError struct { diff --git a/http/document_handler.go b/http/document_handler.go index 2f38afd..f41f7ff 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -160,6 +160,8 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. metricsAgent = "default" } + rootVersionMap := make(map[string]string) + var newRootVersion string for _, deviceId := range deviceIds { fields["src_caller"] = common.GetCaller() @@ -189,15 +191,22 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. } doc.SetSubDocument(subdocId, subdoc) - newRootVersion := db.HashRootVersion(doc.VersionMap()) + newRootVersion = db.HashRootVersion(doc.VersionMap()) err = s.SetRootDocumentVersion(deviceId, newRootVersion) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) return } + rootVersionMap[deviceId] = newRootVersion + } + d := make(util.Dict) + if len(rootVersionMap) == 1 { + d["root_version"] = newRootVersion + } else { + d["root_version"] = rootVersionMap } - WriteOkResponse(w, nil) + WriteByMarshal(w, http.StatusOK, d) } func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *http.Request) { diff --git a/http/main_test.go b/http/main_test.go index 7ccd1f2..6da258a 100644 --- a/http/main_test.go +++ b/http/main_test.go @@ -45,9 +45,8 @@ func TestMain(m *testing.M) { panic(err) } + _ = GetTestDatabaseClient(sc) log.SetOutput(io.Discard) - returnCode := m.Run() - os.Exit(returnCode) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0c2d146..0d82f31 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -151,10 +151,6 @@ func GetTestDatabaseClient(sc *common.ServerConfig) db.DatabaseClient { err = fmt.Errorf("Unsupported database.active_driver %v is configured", activeDriver) panic(err) } - err = tdbclient.SetUp() - if err != nil { - panic(err) - } return tdbclient } @@ -767,6 +763,8 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { err1 := common.NewError(err) fields["response"] = ObfuscatedMap fields["response_text"] = err1.Error() + } else { + fields["response"] = itf } } else { fields["response"] = ObfuscatedMap diff --git a/security/token.go b/security/token.go index 51695e2..169cd0f 100644 --- a/security/token.go +++ b/security/token.go @@ -213,11 +213,11 @@ func ParseKidFromTokenHeader(tokenString string) (string, error) { rawKid, ok := headers["kid"] if !ok { - return kid, common.NewError(common.NotOK) + return kid, common.NewError(common.ErrNotOK) } kid, ok = rawKid.(string) if !ok { - return kid, common.NewError(common.NotOK) + return kid, common.NewError(common.ErrNotOK) } return kid, nil From 8eebeb8122f17720960d0978de2cc61561998e5c Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 7 Mar 2024 13:40:58 -0800 Subject: [PATCH 015/215] copy all incoming headers when calling upstream --- http/multipart.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 5051082..cb445a6 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -207,7 +207,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) + upstreamHeader := rHeader.Clone() upstreamHeader.Set("Content-type", common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { @@ -338,19 +338,17 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) + upstreamHeader := rHeader.Clone() upstreamHeader.Set("Content-type", common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) + if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) if len(auditId) > 0 { upstreamHeader.Set(common.HeaderAuditid, auditId) } } - if x := rHeader.Get(common.HeaderTransactionId); len(x) > 0 { - upstreamHeader.Set(common.HeaderTransactionId, x) - } if s.TokenManager != nil { token := rHeader.Get("Authorization") @@ -363,9 +361,6 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } // call /upstream to handle factory reset - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) - upstreamHeader.Set(common.HeaderIfNoneMatch, ifNoneMatch) - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) if err != nil { var rherr common.RemoteHttpError From 6e829bb8ca46c1b23ab63b7312f79ff3b128a45f Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 8 Mar 2024 22:22:39 -0800 Subject: [PATCH 016/215] fix a bug that some responses are not of json format when there are errors --- http/multipart.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index cb445a6..227364f 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -88,9 +88,6 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. } status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) - if err != nil && respBytes == nil { - respBytes = []byte(err.Error()) - } // REMINDER 404 use standard response if status == http.StatusNotFound { @@ -102,6 +99,11 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. w.Header().Set(k, respHeader.Get(k)) } + if err != nil && respBytes == nil { + Error(w, status, common.NewError(err)) + return + } + w.WriteHeader(status) _, _ = w.Write(respBytes) } From 19a57e97d2c02e08b49ef79c6f75ccd5f8e3b262 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 7 Mar 2024 13:40:58 -0800 Subject: [PATCH 017/215] copy all incoming headers when calling upstream --- http/multipart.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 5051082..cb445a6 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -207,7 +207,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) + upstreamHeader := rHeader.Clone() upstreamHeader.Set("Content-type", common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { @@ -338,19 +338,17 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) + upstreamHeader := rHeader.Clone() upstreamHeader.Set("Content-type", common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) + if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) if len(auditId) > 0 { upstreamHeader.Set(common.HeaderAuditid, auditId) } } - if x := rHeader.Get(common.HeaderTransactionId); len(x) > 0 { - upstreamHeader.Set(common.HeaderTransactionId, x) - } if s.TokenManager != nil { token := rHeader.Get("Authorization") @@ -363,9 +361,6 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } // call /upstream to handle factory reset - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) - upstreamHeader.Set(common.HeaderIfNoneMatch, ifNoneMatch) - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) if err != nil { var rherr common.RemoteHttpError From 5f4eedd82191418791326f05ccfc596f9661aaf6 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 16 Mar 2024 11:28:12 -0700 Subject: [PATCH 018/215] add support of using reference_document --- common/const_var.go | 1 + common/refsubdocument.go | 81 +++++++++++++++ common/refsubdocument_test.go | 40 ++++++++ db/cassandra/refsubdocument.go | 81 +++++++++++++++ db/cassandra/refsubdocument_test.go | 59 +++++++++++ db/cassandra/schema.go | 5 + db/database_client.go | 5 + db/service.go | 36 +++++++ db/sqlite/refsubdocument.go | 153 ++++++++++++++++++++++++++++ db/sqlite/refsubdocument_test.go | 59 +++++++++++ db/sqlite/schema.go | 5 + http/multipart.go | 23 +++++ http/refsubdocument_handler.go | 125 +++++++++++++++++++++++ http/refsubdocument_handler_test.go | 76 ++++++++++++++ http/router.go | 14 +++ http/validator.go | 37 +++++++ 16 files changed, 800 insertions(+) create mode 100644 common/refsubdocument.go create mode 100644 common/refsubdocument_test.go create mode 100644 db/cassandra/refsubdocument.go create mode 100644 db/cassandra/refsubdocument_test.go create mode 100644 db/sqlite/refsubdocument.go create mode 100644 db/sqlite/refsubdocument_test.go create mode 100644 http/refsubdocument_handler.go create mode 100644 http/refsubdocument_handler_test.go diff --git a/common/const_var.go b/common/const_var.go index 4370c7e..e024f97 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -106,6 +106,7 @@ const ( HeaderTraceparent = "Traceparent" HeaderTracestate = "Tracestate" HeaderContentLength = "Content-Length" + HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" ) // header X-System-Supported-Docs diff --git a/common/refsubdocument.go b/common/refsubdocument.go new file mode 100644 index 0000000..cad7ffc --- /dev/null +++ b/common/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "bytes" +) + +type RefSubDocument struct { + payload []byte + version *string +} + +func NewRefSubDocument(payload []byte, version *string) *RefSubDocument { + return &RefSubDocument{ + payload: payload, + version: version, + } +} + +func (d *RefSubDocument) Payload() []byte { + return d.payload +} + +func (d *RefSubDocument) SetPayload(payload []byte) { + d.payload = payload +} + +func (d *RefSubDocument) HasPayload() bool { + if d.payload != nil && len(d.payload) > 0 { + return true + } else { + return false + } +} + +func (d *RefSubDocument) Version() *string { + return d.version +} + +func (d *RefSubDocument) SetVersion(version *string) { + d.version = version +} + +func (d *RefSubDocument) Equals(tdoc *RefSubDocument) bool { + if d.HasPayload() && tdoc.HasPayload() { + if !bytes.Equal(d.Payload(), tdoc.Payload()) { + return false + } + } else { + if d.HasPayload() != tdoc.HasPayload() { + return false + } + } + + if d.Version() != nil && tdoc.Version() != nil { + if *d.Version() != *tdoc.Version() { + return false + } + } else { + if d.Version() != tdoc.Version() { + return false + } + } + return true +} diff --git a/common/refsubdocument_test.go b/common/refsubdocument_test.go new file mode 100644 index 0000000..416c123 --- /dev/null +++ b/common/refsubdocument_test.go @@ -0,0 +1,40 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestRefSubDocument(t *testing.T) { + bbytes1 := []byte("hello world") + version1 := "12345" + refsubdoc1 := NewRefSubDocument(bbytes1, &version1) + + bbytes2 := []byte("hello world") + version2 := "12345" + refsubdoc2 := NewRefSubDocument(bbytes2, &version2) + assert.Assert(t, refsubdoc1.Equals(refsubdoc2)) + + bbytes3 := []byte("foo bar") + version3 := "12345" + refsubdoc3 := NewRefSubDocument(bbytes3, &version3) + assert.Assert(t, !refsubdoc1.Equals(refsubdoc3)) +} diff --git a/db/cassandra/refsubdocument.go b/db/cassandra/refsubdocument.go new file mode 100644 index 0000000..1c9ca9f --- /dev/null +++ b/db/cassandra/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/gocql/gocql" +) + +func (c *CassandraClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + var payload []byte + var version string + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "SELECT payload,version FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Scan(&payload, &version); err != nil { + return nil, common.NewError(err) + } + + if len(payload) == 0 { + return nil, common.NewError(gocql.ErrNotFound) + } + + refsubdoc := common.NewRefSubDocument(payload, &version) + return refsubdoc, nil +} + +func (c *CassandraClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) (fnerr error) { + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil && len(refsubdoc.Payload()) > 0 { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + stmt := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + if err := c.Query(stmt, values...).Exec(); err != nil { + return common.NewError(err) + } + return nil +} + +func (c *CassandraClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "DELETE FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Exec(); err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/cassandra/refsubdocument_test.go b/db/cassandra/refsubdocument_test.go new file mode 100644 index 0000000..22e0ba8 --- /dev/null +++ b/db/cassandra/refsubdocument_test.go @@ -0,0 +1,59 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "crypto/rand" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + slen := util.RandomInt(100) + 16 + srcBytes := make([]byte, slen) + rand.Read(srcBytes) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index 4d0ac95..b013804 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -44,6 +44,11 @@ var ( route text, schema_version text, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } diff --git a/db/database_client.go b/db/database_client.go index b074e59..7f9a8bf 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -59,4 +59,9 @@ type DatabaseClient interface { FactoryReset(string) error FirmwareUpdate(string, int, *common.RootDocument) error AppendProfiles(string, []byte) ([]byte, error) + + // reference subdocument + GetRefSubDocument(string) (*common.RefSubDocument, error) + SetRefSubDocument(string, *common.RefSubDocument) error + DeleteRefSubDocument(string) error } diff --git a/db/service.go b/db/service.go index 2cc326d..c80e240 100644 --- a/db/service.go +++ b/db/service.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "slices" "sort" "strings" "time" @@ -31,6 +32,14 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + referenceIndicatorByteLength = 4 +) + +var ( + referenceIndicatorBytes = make([]byte, referenceIndicatorByteLength) +) + // TODO s.MultipartSupplementaryHandler(w, r) should be handled separately // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream @@ -527,3 +536,30 @@ func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerI } return cloudRootDocument, nil } + +func GetRefId(payload []byte) (string, bool) { + if len(payload) > referenceIndicatorByteLength { + prefixBytes := payload[:referenceIndicatorByteLength] + if slices.Equal(referenceIndicatorBytes, prefixBytes) { + suffixBytes := payload[referenceIndicatorByteLength:] + return string(suffixBytes), true + } + } + return "", false +} + +func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log.Fields) (*common.Document, error) { + newDocument := common.NewDocument(document.GetRootDocument()) + for subdocId, subDocument := range document.Items() { + payload := subDocument.Payload() + if refId, ok := GetRefId(payload); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + return nil, common.NewError(err) + } + subDocument.SetPayload(refsubdocument.Payload()) + } + newDocument.SetSubDocument(subdocId, &subDocument) + } + return newDocument, nil +} diff --git a/db/sqlite/refsubdocument.go b/db/sqlite/refsubdocument.go new file mode 100644 index 0000000..b93d4eb --- /dev/null +++ b/db/sqlite/refsubdocument.go @@ -0,0 +1,153 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "database/sql" + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + _ "github.com/mattn/go-sqlite3" +) + +func (c *SqliteClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + rows, err := c.Query("SELECT payload,version FROM reference_document WHERE ref_id=?", refId) + if err != nil { + return nil, common.NewError(err) + } + + var ns1 sql.NullString + var b1 []byte + + if !rows.Next() { + return nil, sql.ErrNoRows + } + err = rows.Scan(&b1, &ns1) + defer rows.Close() + if err != nil { + return nil, common.NewError(err) + } + + var s1 *string + if ns1.Valid { + s1 = &(ns1.String) + } + + refsubdoc := common.NewRefSubDocument(b1, s1) + return refsubdoc, nil +} + +func (c *SqliteClient) insertRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + qstr := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) updateRefSubDocument(refId string, doc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{} + values := []interface{}{} + if doc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, doc.Payload()) + } + if doc.Version() != nil { + columns = append(columns, "version") + values = append(values, doc.Version()) + } + values = append(values, refId) + qstr := fmt.Sprintf("UPDATE reference_document SET %v WHERE cpe_mac=? AND ref_id=?", db.GetSetColumnsStr(columns)) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + _, err := c.GetRefSubDocument(refId) + if err != nil { + if c.IsDbNotFound(err) { + err1 := c.insertRefSubDocument(refId, refsubdoc) + if err1 != nil { + return common.NewError(err1) + } + } else { + // unexpected error + return common.NewError(err) + } + } else { + // normal dbNotFound should not happen + err = c.updateRefSubDocument(refId, refsubdoc) + if err != nil { + return common.NewError(err) + } + } + + return nil +} + +func (c *SqliteClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt, err := c.Prepare("DELETE FROM reference_document WHERE ref_id=?") + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(refId) + if err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/sqlite/refsubdocument_test.go b/db/sqlite/refsubdocument_test.go new file mode 100644 index 0000000..01478f9 --- /dev/null +++ b/db/sqlite/refsubdocument_test.go @@ -0,0 +1,59 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "crypto/rand" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + slen := util.RandomInt(100) + 16 + srcBytes := make([]byte, slen) + rand.Read(srcBytes) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 8ee8d6e..3e4c132 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -50,6 +50,11 @@ var ( route text, schema_version, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } ) diff --git a/http/multipart.go b/http/multipart.go index 227364f..b2860af 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -147,6 +147,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document.DeleteSubDocument(subdocId) } + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } respBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -178,9 +182,20 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document = common.NewDocument(rootDocument) } + if userAgent == "mget" { + postUpstream = false + } + var respBytes []byte respStatus := http.StatusNotModified if document.Length() > 0 { + + if !postUpstream { + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + } respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -286,6 +301,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusNotModified, upstreamRespHeader, nil, nil } + finalFilteredDocument, err = db.LoadRefSubDocuments(c, finalFilteredDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalFilteredBytes, err := finalFilteredDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalFilteredBytes, common.NewError(err) @@ -397,6 +416,10 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusNotFound, upstreamRespHeader, nil, nil } + finalDocument, err = db.LoadRefSubDocuments(c, finalDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalBytes, err := finalDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go new file mode 100644 index 0000000..6413ee4 --- /dev/null +++ b/http/refsubdocument_handler.go @@ -0,0 +1,125 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "errors" + "net/http" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" +) + +func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + refsubdoc, err := s.GetRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + LogError(w, err) + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + + w.Header().Set("Content-Type", "application/msgpack") + if refsubdoc.Version() != nil { + w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) + } + w.WriteHeader(http.StatusOK) + w.Write(refsubdoc.Payload()) +} + +func (s *WebconfigServer) PostRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, bbytes, _, err := s.ValidateRefData(w, r, true) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + // handle version header + version := r.Header.Get(common.HeaderSubdocumentVersion) + if len(version) == 0 { + version = util.GetMurmur3Hash(bbytes) + } + + refsubdoc := common.NewRefSubDocument(bbytes, &version) + + err = s.SetRefSubDocument(refId, refsubdoc) + if err != nil { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + + WriteOkResponse(w, nil) +} + +func (s *WebconfigServer) DeleteRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + err = s.DeleteRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + WriteOkResponse(w, nil) +} diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go new file mode 100644 index 0000000..49171b0 --- /dev/null +++ b/http/refsubdocument_handler_test.go @@ -0,0 +1,76 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "testing" + + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + refId := uuid.New().String() + bbytes := util.RandomBytes(100, 150) + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes) + + // delete + req, err = http.NewRequest("DELETE", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get but expect 404 + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} diff --git a/http/router.go b/http/router.go index 7645c51..746bec0 100644 --- a/http/router.go +++ b/http/router.go @@ -127,5 +127,19 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { } sub4.HandleFunc("", s.DeleteDocumentHandler).Methods("DELETE") + sub5 := router.Path("/api/v1/reference/{ref}/document").Subrouter() + if testOnly { + sub5.Use(s.TestingMiddleware) + } else { + if s.ServerApiTokenAuthEnabled() { + sub5.Use(s.ApiMiddleware) + } else { + sub5.Use(s.NoAuthMiddleware) + } + } + sub5.HandleFunc("", s.GetRefSubDocumentHandler).Methods("GET") + sub5.HandleFunc("", s.PostRefSubDocumentHandler).Methods("POST") + sub5.HandleFunc("", s.DeleteRefSubDocumentHandler).Methods("DELETE") + return router } diff --git a/http/validator.go b/http/validator.go index 10683ec..d85921e 100644 --- a/http/validator.go +++ b/http/validator.go @@ -71,3 +71,40 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid } return mac, subdocId, bodyBytes, fields, nil } + +func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request, validateContent bool) (string, []byte, log.Fields, error) { + var fields log.Fields + + // check mac + params := mux.Vars(r) + refId := params["ref"] + + // check for safety, but it should not fail + xw, ok := w.(*XResponseWriter) + if !ok { + err := *common.NewHttp500Error("responsewriter cast error") + return refId, nil, nil, common.NewError(err) + } + fields = xw.Audit() + + if !validateContent { + return refId, nil, fields, nil + } + + // ==== validate content ==== + // check content-type + contentType := r.Header.Get("Content-type") + if contentType != "application/msgpack" { + // TODO (1) if we should validate this header + // (2) if unexpected, return 400 or 415 + err := *common.NewHttp400Error("content-type not msgpack") + return refId, nil, nil, common.NewError(err) + } + + bodyBytes := xw.BodyBytes() + if len(bodyBytes) == 0 { + err := *common.NewHttp400Error("empty body") + return refId, nil, nil, common.NewError(err) + } + return refId, bodyBytes, fields, nil +} From a56833234cc80a15321770ca5bfcd063973b2904 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 18 Mar 2024 23:25:56 -0700 Subject: [PATCH 019/215] add an optional feature to correct states --- common/subdocument.go | 16 ++ common/subdocument_test.go | 10 +- config/sample_webconfig.conf | 3 + db/cassandra/cassandra_client.go | 33 ++-- db/cassandra/cassandra_client_test.go | 8 + db/database_client.go | 4 + db/service.go | 31 +++ db/sqlite/sqlite_client.go | 22 ++- db/sqlite/sqlite_client_test.go | 43 +++++ http/document_handler_test.go | 2 +- http/multipart_test.go | 261 +++++++++++++++++++++++++- 11 files changed, 409 insertions(+), 24 deletions(-) create mode 100644 db/sqlite/sqlite_client_test.go diff --git a/common/subdocument.go b/common/subdocument.go index 336b22b..bd54947 100644 --- a/common/subdocument.go +++ b/common/subdocument.go @@ -102,6 +102,14 @@ func (d *SubDocument) SetVersion(version *string) { d.version = version } +// convenient function +func (d *SubDocument) GetVersion() string { + if d.version != nil { + return *d.version + } + return "" +} + func (d *SubDocument) State() *int { return d.state } @@ -110,6 +118,14 @@ func (d *SubDocument) SetState(state *int) { d.state = state } +// convenient function +func (d *SubDocument) GetState() int { + if d.state != nil { + return *d.state + } + return 0 +} + func (d *SubDocument) UpdatedTime() *int { return d.updatedTime } diff --git a/common/subdocument_test.go b/common/subdocument_test.go index 3b4bf1a..379ad1b 100644 --- a/common/subdocument_test.go +++ b/common/subdocument_test.go @@ -32,6 +32,12 @@ func TestSubDocumentString(t *testing.T) { errorCode := 103 errorDetails := "cannot parse" - doc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) - assert.Assert(t, doc != nil) + subdoc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) + assert.Assert(t, subdoc != nil) + + subdoc = &SubDocument{} + tgtVersion := subdoc.GetVersion() + assert.Equal(t, tgtVersion, "") + tgtState := subdoc.GetState() + assert.Equal(t, tgtState, 0) } diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index f47c838..894f513 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -232,4 +232,7 @@ webconfig { valid_partners = ["company", "vendor1", "vendor2"] supplementary_appending_enabled = true + + // correct subdoc states if versions match but not "deployed" + state_correction_enabled= false } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index d9ca1f7..7855b9f 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -48,10 +48,11 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool } /* @@ -159,15 +160,17 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, }, nil } @@ -216,6 +219,14 @@ func (c *CassandraClient) SetEncryptedSubdocIds(x []string) { c.encryptedSubdocIds = x } +func (c *CassandraClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 57c1734..355b701 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -38,6 +38,14 @@ func TestCassandraClient(t *testing.T) { tgtSubdocIds := tdbclient.EncryptedSubdocIds() assert.Assert(t, len(tgtSubdocIds) == 4) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/database_client.go b/db/database_client.go index 7f9a8bf..fa1065f 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -64,4 +64,8 @@ type DatabaseClient interface { GetRefSubDocument(string) (*common.RefSubDocument, error) SetRefSubDocument(string, *common.RefSubDocument) error DeleteRefSubDocument(string) error + + // enable state correction + StateCorrectionEnabled() bool + SetStateCorrectionEnabled(bool) } diff --git a/db/service.go b/db/service.go index c80e240..882d0c4 100644 --- a/db/service.go +++ b/db/service.go @@ -29,6 +29,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -147,6 +148,36 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field } } + if c.StateCorrectionEnabled() { + if document == nil { + document, err = c.GetDocument(mac, fields) + if err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + } + } + for subdocId, subdocument := range document.Items() { + cloudVersion := subdocument.GetVersion() + cloudState := subdocument.GetState() + if len(cloudVersion) == 0 { + continue + } + deviceVersion := deviceVersionMap[subdocId] + if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { + labels := prometheus.Labels{ + "model": modelName, + "fwversion": firmwareVersion, + } + // update state + newState := common.Deployed + subdocument.SetState(&newState) + if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + } + } + } + + } + switch rootCmpEnum { case common.RootDocumentEquals: // create an empty "document" diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index d623d07..9ab3b12 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -43,8 +43,9 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -58,15 +59,18 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + db, err := sql.Open("sqlite3", dbfile) if err != nil { return nil, common.NewError(err) } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, }, nil } @@ -136,6 +140,14 @@ func (c *SqliteClient) SetBlockedSubdocIds(x []string) { c.blockedSubdocIds = x } +func (c *SqliteClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go new file mode 100644 index 0000000..3a3f74f --- /dev/null +++ b/db/sqlite/sqlite_client_test.go @@ -0,0 +1,43 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "testing" + + "github.com/rdkcentral/webconfig/common" + "gotest.tools/assert" +) + +func TestSqliteClient(t *testing.T) { + configFile := "../../config/sample_webconfig.conf" + sc, err := common.GetTestServerConfig(configFile) + + assert.NilError(t, err) + dbc, err := GetTestSqliteClient(sc.Config, true) + assert.NilError(t, err) + assert.Assert(t, dbc != nil) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) +} diff --git a/http/document_handler_test.go b/http/document_handler_test.go index f879914..7d11287 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -419,7 +419,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { subdocVersions = append(subdocVersions, mpart.Version) // ==== step 5 get document again ==== - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,gwrestore,remotedebugger,lan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) diff --git a/http/multipart_test.go b/http/multipart_test.go index 0dc6be5..30919f8 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -133,7 +133,7 @@ func TestMultipartConfigHandler(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -144,7 +144,7 @@ func TestMultipartConfigHandler(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) - // ==== cal GET /config with if-none-match partial match ==== + // ==== call GET /config with if-none-match partial match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -819,7 +819,7 @@ func TestMultipartConfigMismatch(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -831,7 +831,7 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header2 := "NONE,123" @@ -842,7 +842,7 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header3 := etag + ",123" @@ -853,3 +853,254 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) } + +func TestStateCorrectionEnabled(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== group 3 mesh ==== + subdocId = "mesh" + meshBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, meshBytes) + + // ==== group 3 moca ==== + subdocId = "moca" + mocaBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, mocaBytes) + + // ==== GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 4) + etag := res.Header.Get(common.HeaderEtag) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + mpart, ok = mpartMap["mesh"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, meshBytes) + meshVersion := mpart.Version + + mpart, ok = mpartMap["moca"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, mocaBytes) + mocaVersion := mpart.Version + "x" + + // verify all states are in-deployment + lanSubdocument, err := server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.InDeployment) + + wanSubdocument, err := server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err := server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.InDeployment) + + mocaSubdocument, err := server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.InDeployment) + + // ==== setup special error conditions to test state correction scenario ==== + lanState := common.PendingDownload + lanSubdocument.SetState(&lanState) + err = server.SetSubDocument(cpeMac, "lan", lanSubdocument) + assert.NilError(t, err) + + wanState := common.InDeployment + wanSubdocument.SetState(&wanState) + err = server.SetSubDocument(cpeMac, "wan", wanSubdocument) + assert.NilError(t, err) + + meshState := common.Failure + meshSubdocument.SetState(&meshState) + err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) + assert.NilError(t, err) + + mocaState := common.PendingDownload + mocaSubdocument.SetState(&mocaState) + err = server.SetSubDocument(cpeMac, "moca", mocaSubdocument) + assert.NilError(t, err) + + // ==== call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(false) + + configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan,mesh,moca", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + header1 := fmt.Sprintf("%v,%v,%v,%v,%v", etag, lanVersion, wanVersion, meshVersion, mocaVersion) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify the states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Failure) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + + // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(true) + defer func() { + server.SetStateCorrectionEnabled(false) + }() + + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify all version-matched states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) +} From 395cba1c0e7e6c0db1d64c2f454323e50e15489b Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 19 Mar 2024 13:36:33 -0700 Subject: [PATCH 020/215] add support of rfc subdoc bitmap --- common/const_var.go | 2 ++ http/supported_groups_handler_test.go | 3 +++ util/firmware_bitmap_test.go | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/common/const_var.go b/common/const_var.go index e024f97..9206be1 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -153,6 +153,7 @@ var ( }, 8: { {1, 15}, + {2, 32}, }, 9: { {1, 16}, @@ -222,6 +223,7 @@ var ( "connectedbuilding": 29, "lldqoscontrol": 30, "clienttosteeringprofile": 31, + "rfc": 32, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 80ae1b1..51820cb 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -98,6 +98,7 @@ func TestSupportedGroupsHandler(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response @@ -283,6 +284,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response @@ -365,6 +367,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 4475183..841827d 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -163,6 +163,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -215,6 +216,8 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -266,6 +269,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -318,6 +322,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -380,6 +385,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -430,6 +436,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -480,6 +487,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -520,6 +528,7 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -569,6 +578,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -618,6 +628,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -667,6 +678,7 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From cb919c3b2dc26b4326ba1f290fa6d8e83777fc7a Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 18 Mar 2024 23:25:56 -0700 Subject: [PATCH 021/215] cherrypick and resolve a conflict --- common/subdocument.go | 16 ++ common/subdocument_test.go | 10 +- config/sample_webconfig.conf | 3 + db/cassandra/cassandra_client.go | 33 ++-- db/cassandra/cassandra_client_test.go | 8 + db/database_client.go | 4 + db/service.go | 31 +++ db/sqlite/sqlite_client.go | 22 ++- db/sqlite/sqlite_client_test.go | 43 +++++ http/document_handler_test.go | 2 +- http/multipart_test.go | 261 +++++++++++++++++++++++++- 11 files changed, 409 insertions(+), 24 deletions(-) create mode 100644 db/sqlite/sqlite_client_test.go diff --git a/common/subdocument.go b/common/subdocument.go index 336b22b..bd54947 100644 --- a/common/subdocument.go +++ b/common/subdocument.go @@ -102,6 +102,14 @@ func (d *SubDocument) SetVersion(version *string) { d.version = version } +// convenient function +func (d *SubDocument) GetVersion() string { + if d.version != nil { + return *d.version + } + return "" +} + func (d *SubDocument) State() *int { return d.state } @@ -110,6 +118,14 @@ func (d *SubDocument) SetState(state *int) { d.state = state } +// convenient function +func (d *SubDocument) GetState() int { + if d.state != nil { + return *d.state + } + return 0 +} + func (d *SubDocument) UpdatedTime() *int { return d.updatedTime } diff --git a/common/subdocument_test.go b/common/subdocument_test.go index 3b4bf1a..379ad1b 100644 --- a/common/subdocument_test.go +++ b/common/subdocument_test.go @@ -32,6 +32,12 @@ func TestSubDocumentString(t *testing.T) { errorCode := 103 errorDetails := "cannot parse" - doc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) - assert.Assert(t, doc != nil) + subdoc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) + assert.Assert(t, subdoc != nil) + + subdoc = &SubDocument{} + tgtVersion := subdoc.GetVersion() + assert.Equal(t, tgtVersion, "") + tgtState := subdoc.GetState() + assert.Equal(t, tgtState, 0) } diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index f47c838..894f513 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -232,4 +232,7 @@ webconfig { valid_partners = ["company", "vendor1", "vendor2"] supplementary_appending_enabled = true + + // correct subdoc states if versions match but not "deployed" + state_correction_enabled= false } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index d9ca1f7..7855b9f 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -48,10 +48,11 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool } /* @@ -159,15 +160,17 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, }, nil } @@ -216,6 +219,14 @@ func (c *CassandraClient) SetEncryptedSubdocIds(x []string) { c.encryptedSubdocIds = x } +func (c *CassandraClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 57c1734..355b701 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -38,6 +38,14 @@ func TestCassandraClient(t *testing.T) { tgtSubdocIds := tdbclient.EncryptedSubdocIds() assert.Assert(t, len(tgtSubdocIds) == 4) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/database_client.go b/db/database_client.go index b074e59..397c995 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -59,4 +59,8 @@ type DatabaseClient interface { FactoryReset(string) error FirmwareUpdate(string, int, *common.RootDocument) error AppendProfiles(string, []byte) ([]byte, error) + + // enable state correction + StateCorrectionEnabled() bool + SetStateCorrectionEnabled(bool) } diff --git a/db/service.go b/db/service.go index 2cc326d..ca77dc2 100644 --- a/db/service.go +++ b/db/service.go @@ -28,6 +28,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -138,6 +139,36 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field } } + if c.StateCorrectionEnabled() { + if document == nil { + document, err = c.GetDocument(mac, fields) + if err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + } + } + for subdocId, subdocument := range document.Items() { + cloudVersion := subdocument.GetVersion() + cloudState := subdocument.GetState() + if len(cloudVersion) == 0 { + continue + } + deviceVersion := deviceVersionMap[subdocId] + if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { + labels := prometheus.Labels{ + "model": modelName, + "fwversion": firmwareVersion, + } + // update state + newState := common.Deployed + subdocument.SetState(&newState) + if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + } + } + } + + } + switch rootCmpEnum { case common.RootDocumentEquals: // create an empty "document" diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index d623d07..9ab3b12 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -43,8 +43,9 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -58,15 +59,18 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + db, err := sql.Open("sqlite3", dbfile) if err != nil { return nil, common.NewError(err) } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, }, nil } @@ -136,6 +140,14 @@ func (c *SqliteClient) SetBlockedSubdocIds(x []string) { c.blockedSubdocIds = x } +func (c *SqliteClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go new file mode 100644 index 0000000..3a3f74f --- /dev/null +++ b/db/sqlite/sqlite_client_test.go @@ -0,0 +1,43 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "testing" + + "github.com/rdkcentral/webconfig/common" + "gotest.tools/assert" +) + +func TestSqliteClient(t *testing.T) { + configFile := "../../config/sample_webconfig.conf" + sc, err := common.GetTestServerConfig(configFile) + + assert.NilError(t, err) + dbc, err := GetTestSqliteClient(sc.Config, true) + assert.NilError(t, err) + assert.Assert(t, dbc != nil) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) +} diff --git a/http/document_handler_test.go b/http/document_handler_test.go index f879914..7d11287 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -419,7 +419,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { subdocVersions = append(subdocVersions, mpart.Version) // ==== step 5 get document again ==== - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,gwrestore,remotedebugger,lan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) diff --git a/http/multipart_test.go b/http/multipart_test.go index 0dc6be5..30919f8 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -133,7 +133,7 @@ func TestMultipartConfigHandler(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -144,7 +144,7 @@ func TestMultipartConfigHandler(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) - // ==== cal GET /config with if-none-match partial match ==== + // ==== call GET /config with if-none-match partial match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -819,7 +819,7 @@ func TestMultipartConfigMismatch(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -831,7 +831,7 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header2 := "NONE,123" @@ -842,7 +842,7 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header3 := etag + ",123" @@ -853,3 +853,254 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) } + +func TestStateCorrectionEnabled(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== group 3 mesh ==== + subdocId = "mesh" + meshBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, meshBytes) + + // ==== group 3 moca ==== + subdocId = "moca" + mocaBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, mocaBytes) + + // ==== GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 4) + etag := res.Header.Get(common.HeaderEtag) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + mpart, ok = mpartMap["mesh"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, meshBytes) + meshVersion := mpart.Version + + mpart, ok = mpartMap["moca"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, mocaBytes) + mocaVersion := mpart.Version + "x" + + // verify all states are in-deployment + lanSubdocument, err := server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.InDeployment) + + wanSubdocument, err := server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err := server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.InDeployment) + + mocaSubdocument, err := server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.InDeployment) + + // ==== setup special error conditions to test state correction scenario ==== + lanState := common.PendingDownload + lanSubdocument.SetState(&lanState) + err = server.SetSubDocument(cpeMac, "lan", lanSubdocument) + assert.NilError(t, err) + + wanState := common.InDeployment + wanSubdocument.SetState(&wanState) + err = server.SetSubDocument(cpeMac, "wan", wanSubdocument) + assert.NilError(t, err) + + meshState := common.Failure + meshSubdocument.SetState(&meshState) + err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) + assert.NilError(t, err) + + mocaState := common.PendingDownload + mocaSubdocument.SetState(&mocaState) + err = server.SetSubDocument(cpeMac, "moca", mocaSubdocument) + assert.NilError(t, err) + + // ==== call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(false) + + configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan,mesh,moca", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + header1 := fmt.Sprintf("%v,%v,%v,%v,%v", etag, lanVersion, wanVersion, meshVersion, mocaVersion) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify the states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Failure) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + + // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(true) + defer func() { + server.SetStateCorrectionEnabled(false) + }() + + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify all version-matched states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) +} From 26ce641b91ea1a51ccbc48c64679fec0587791e3 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 11 Apr 2024 23:02:04 -0700 Subject: [PATCH 022/215] Add support for defaultrfc bitmap --- common/const_var.go | 4 +- http/supported_groups_handler_test.go | 3 ++ util/firmware_bitmap_test.go | 62 +++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/common/const_var.go b/common/const_var.go index 9206be1..fdcda6f 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -154,6 +154,7 @@ var ( 8: { {1, 15}, {2, 32}, + {3, 33}, }, 9: { {1, 16}, @@ -223,7 +224,8 @@ var ( "connectedbuilding": 29, "lldqoscontrol": 30, "clienttosteeringprofile": 31, - "rfc": 32, + "defaultrfc": 32, + "rfc": 33, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 51820cb..d14dc56 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -99,6 +99,7 @@ func TestSupportedGroupsHandler(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response @@ -285,6 +286,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response @@ -368,6 +370,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "lldqoscontrol": true, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 841827d..f68affa 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -164,6 +164,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -217,6 +218,7 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -270,6 +272,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -323,6 +326,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -386,6 +390,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -437,6 +442,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -488,6 +494,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -529,6 +536,7 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -579,6 +587,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -629,6 +638,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "lldqoscontrol": true, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -679,6 +689,58 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "lldqoscontrol": true, "clienttosteeringprofile": true, "rfc": false, + "defaultrfc": false, + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) +} + +func TestParseSupportedDocsHeaderRfc(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" + + // build expected + expectedEnabled := map[string]bool{ + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": true, + "rfc": true, + "defaultrfc": true, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 84ff8a51ca8e47fe25255800b5f690c3b0a054a7 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 19 Apr 2024 15:44:34 -0700 Subject: [PATCH 023/215] add bitmaps for wifimotionsettings and xmspeedboost --- common/const_var.go | 10 +++++- http/supported_groups_handler_test.go | 12 +++++-- util/firmware_bitmap_test.go | 46 ++++++++++++++++++++------- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index fdcda6f..2fcf2af 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -128,7 +128,8 @@ var ( {4, 4}, {5, 5}, {6, 6}, - {7, 29}, + {7, 29}, // connectedbuilding + {8, 35}, // xmspeedboost }, 2: { {1, 7}, @@ -147,6 +148,11 @@ var ( 6: { {1, 13}, // mesh {2, 31}, // clienttosteeringprofile + {3, 36}, // meshsteeringprofiles + {4, 37}, // wifistatsconfig + {5, 38}, // mwoconfigs + {6, 39}, // interference + {7, 34}, // wifimotionsettings }, 7: { {1, 14}, @@ -226,6 +232,8 @@ var ( "clienttosteeringprofile": 31, "defaultrfc": 32, "rfc": 33, + "wifimotionsettings": 34, + "xmspeedboost": 35, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index d14dc56..13ffbe9 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -98,8 +98,10 @@ func TestSupportedGroupsHandler(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response @@ -285,8 +287,10 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response @@ -369,8 +373,10 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index f68affa..28c3159 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -163,8 +163,10 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -217,8 +219,10 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -271,8 +275,10 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -325,8 +331,10 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -389,8 +397,10 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -441,8 +451,10 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -493,8 +505,10 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -537,6 +551,8 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "clienttosteeringprofile": false, "rfc": false, "defaultrfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -586,8 +602,10 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -637,8 +655,10 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -688,8 +708,10 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -739,8 +761,10 @@ func TestParseSupportedDocsHeaderRfc(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, - "rfc": true, "defaultrfc": true, + "rfc": true, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 02765e2f8f053df76a2d910259d10786080f50de Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 19 Mar 2024 13:36:33 -0700 Subject: [PATCH 024/215] add support of rfc subdoc bitmap --- common/const_var.go | 2 ++ http/supported_groups_handler_test.go | 3 +++ util/firmware_bitmap_test.go | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/common/const_var.go b/common/const_var.go index 4370c7e..893bdf7 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -152,6 +152,7 @@ var ( }, 8: { {1, 15}, + {2, 32}, }, 9: { {1, 16}, @@ -221,6 +222,7 @@ var ( "connectedbuilding": 29, "lldqoscontrol": 30, "clienttosteeringprofile": 31, + "rfc": 32, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 80ae1b1..51820cb 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -98,6 +98,7 @@ func TestSupportedGroupsHandler(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response @@ -283,6 +284,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response @@ -365,6 +367,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 4475183..841827d 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -163,6 +163,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -215,6 +216,8 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -266,6 +269,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -318,6 +322,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -380,6 +385,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -430,6 +436,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -480,6 +487,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -520,6 +528,7 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -569,6 +578,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -618,6 +628,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -667,6 +678,7 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 437c523271b588d5864205a5aaae3b90a889a641 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 11 Apr 2024 23:02:04 -0700 Subject: [PATCH 025/215] Add support for defaultrfc bitmap --- common/const_var.go | 4 +- http/supported_groups_handler_test.go | 3 ++ util/firmware_bitmap_test.go | 62 +++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/common/const_var.go b/common/const_var.go index 893bdf7..a2fb8cc 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -153,6 +153,7 @@ var ( 8: { {1, 15}, {2, 32}, + {3, 33}, }, 9: { {1, 16}, @@ -222,7 +223,8 @@ var ( "connectedbuilding": 29, "lldqoscontrol": 30, "clienttosteeringprofile": 31, - "rfc": 32, + "defaultrfc": 32, + "rfc": 33, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 51820cb..d14dc56 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -99,6 +99,7 @@ func TestSupportedGroupsHandler(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response @@ -285,6 +286,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response @@ -368,6 +370,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "lldqoscontrol": true, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 841827d..f68affa 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -164,6 +164,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -217,6 +218,7 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -270,6 +272,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -323,6 +326,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -386,6 +390,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -437,6 +442,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -488,6 +494,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -529,6 +536,7 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -579,6 +587,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -629,6 +638,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "lldqoscontrol": true, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -679,6 +689,58 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "lldqoscontrol": true, "clienttosteeringprofile": true, "rfc": false, + "defaultrfc": false, + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) +} + +func TestParseSupportedDocsHeaderRfc(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" + + // build expected + expectedEnabled := map[string]bool{ + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": true, + "rfc": true, + "defaultrfc": true, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 8720d84d2304d74a78d7e3433e82b81111bcc1d9 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 19 Apr 2024 15:44:34 -0700 Subject: [PATCH 026/215] add bitmaps for wifimotionsettings and xmspeedboost --- common/const_var.go | 10 +++++- http/supported_groups_handler_test.go | 12 +++++-- util/firmware_bitmap_test.go | 46 ++++++++++++++++++++------- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index a2fb8cc..8efae88 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -127,7 +127,8 @@ var ( {4, 4}, {5, 5}, {6, 6}, - {7, 29}, + {7, 29}, // connectedbuilding + {8, 35}, // xmspeedboost }, 2: { {1, 7}, @@ -146,6 +147,11 @@ var ( 6: { {1, 13}, // mesh {2, 31}, // clienttosteeringprofile + {3, 36}, // meshsteeringprofiles + {4, 37}, // wifistatsconfig + {5, 38}, // mwoconfigs + {6, 39}, // interference + {7, 34}, // wifimotionsettings }, 7: { {1, 14}, @@ -225,6 +231,8 @@ var ( "clienttosteeringprofile": 31, "defaultrfc": 32, "rfc": 33, + "wifimotionsettings": 34, + "xmspeedboost": 35, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index d14dc56..13ffbe9 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -98,8 +98,10 @@ func TestSupportedGroupsHandler(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response @@ -285,8 +287,10 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response @@ -369,8 +373,10 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index f68affa..28c3159 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -163,8 +163,10 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -217,8 +219,10 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -271,8 +275,10 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -325,8 +331,10 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -389,8 +397,10 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -441,8 +451,10 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -493,8 +505,10 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -537,6 +551,8 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "clienttosteeringprofile": false, "rfc": false, "defaultrfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -586,8 +602,10 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -637,8 +655,10 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -688,8 +708,10 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -739,8 +761,10 @@ func TestParseSupportedDocsHeaderRfc(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, - "rfc": true, "defaultrfc": true, + "rfc": true, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 6b89ce52d8cd67f6f1df186fca13a5b3ed9b67b5 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 9 May 2024 23:20:02 -0700 Subject: [PATCH 027/215] do urlencode for xconf telemetry url and handle 400 response properly --- http/supplementary_handler.go | 7 ++---- http/supplementary_handler_test.go | 40 ++++++++++++++++++++++++++++++ util/string.go | 4 +++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index c4488c5..52e5660 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -77,11 +77,8 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - if rherr.StatusCode == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) - return - } - + Error(w, rherr.StatusCode, rherr) + return } Error(w, http.StatusInternalServerError, common.NewError(err)) return diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index ff08613..b7007e5 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -760,3 +760,43 @@ func TestSupplementaryAppendingFlag(t *testing.T) { // ==== step 5 verify the query params ==== assert.Assert(t, !strings.Contains(ss, "&stormReadyWifi=true")) } + +func TestSupplementaryApiBadRequest(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + validPartners := []string{"comcast", "comcast-dev", "cox", "rogers", "shaw", "sky-de", "sky-italia", "sky-italia-dev", "sky-roi", "sky-roi-dev", "sky-uk", "sky-uk-dev", "videotron"} + server.SetValidPartners(validPartners) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad Request")) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "R NOT") + req.Header.Set(common.HeaderAccountID, "ERROR: ld.so: object '/usr/lib/libwayland-egl.so.0' from LD_PRE") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) +} diff --git a/util/string.go b/util/string.go index a582544..439b0bb 100644 --- a/util/string.go +++ b/util/string.go @@ -103,6 +103,10 @@ func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId str if len(queryParams) > 0 && len(ret) > 0 { ret += "&" + queryParams } + + ret = url.QueryEscape(ret) + ret = strings.ReplaceAll(ret, "%3D", "=") + ret = strings.ReplaceAll(ret, "%26", "&") return ret } From b2e372a59ee10b5b1e4b4e60146ca63e216b74ff Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 9 May 2024 23:20:02 -0700 Subject: [PATCH 028/215] do urlencode for xconf telemetry url and handle 400 response properly --- http/supplementary_handler.go | 7 ++---- http/supplementary_handler_test.go | 40 ++++++++++++++++++++++++++++++ util/string.go | 4 +++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index c4488c5..52e5660 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -77,11 +77,8 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - if rherr.StatusCode == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) - return - } - + Error(w, rherr.StatusCode, rherr) + return } Error(w, http.StatusInternalServerError, common.NewError(err)) return diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index ff08613..b7007e5 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -760,3 +760,43 @@ func TestSupplementaryAppendingFlag(t *testing.T) { // ==== step 5 verify the query params ==== assert.Assert(t, !strings.Contains(ss, "&stormReadyWifi=true")) } + +func TestSupplementaryApiBadRequest(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + validPartners := []string{"comcast", "comcast-dev", "cox", "rogers", "shaw", "sky-de", "sky-italia", "sky-italia-dev", "sky-roi", "sky-roi-dev", "sky-uk", "sky-uk-dev", "videotron"} + server.SetValidPartners(validPartners) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad Request")) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "R NOT") + req.Header.Set(common.HeaderAccountID, "ERROR: ld.so: object '/usr/lib/libwayland-egl.so.0' from LD_PRE") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) +} diff --git a/util/string.go b/util/string.go index a582544..439b0bb 100644 --- a/util/string.go +++ b/util/string.go @@ -103,6 +103,10 @@ func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId str if len(queryParams) > 0 && len(ret) > 0 { ret += "&" + queryParams } + + ret = url.QueryEscape(ret) + ret = strings.ReplaceAll(ret, "%3D", "=") + ret = strings.ReplaceAll(ret, "%26", "&") return ret } From cc31901635c1cb29f02e979a6ce53f2b8d354a36 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Tue, 14 May 2024 16:44:31 -0700 Subject: [PATCH 029/215] Opentelemetry tracing option added Supports noop, stdouttracer and otlphttp tracer noop is the default, so no tracing will take place by default --- config/sample_webconfig.conf | 8 ++ go.mod | 38 +++++--- go.sum | 87 ++++++++++++----- http/otel.go | 178 +++++++++++++++++++++++++++++++++++ http/webconfig_server.go | 37 ++++++-- main.go | 1 + 6 files changed, 305 insertions(+), 44 deletions(-) create mode 100644 http/otel.go diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 894f513..a36a3d7 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -6,6 +6,14 @@ webconfig { panic_exit_enabled = false traceparent_parent_id = "0000000000000001" tracestate_vendor_id = "webconfig" + opentelemetry { + endpoint = "127.0.0.1:4318" + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop", + } // build info code_git_commit = "2ac7ff4" diff --git a/go.mod b/go.mod index bbdcee4..cfb4bb9 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt/v5 v5.0.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.0 github.com/mattn/go-sqlite3 v1.14.15 github.com/prometheus/client_golang v1.13.0 @@ -17,23 +17,32 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 + go.opentelemetry.io/otel v1.26.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 + go.opentelemetry.io/otel/sdk v1.26.0 + go.opentelemetry.io/otel/trace v1.26.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.4.0 + golang.org/x/sync v0.6.0 gotest.tools v2.2.0+incompatible ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -44,19 +53,24 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index f502ae6..8505ce4 100644 --- a/go.sum +++ b/go.sum @@ -52,16 +52,18 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -90,6 +92,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= @@ -122,8 +129,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -138,8 +146,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -151,14 +159,16 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -199,8 +209,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -254,8 +264,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -273,8 +283,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -293,6 +303,22 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 h1:0W5o9SzoR15ocYHEQfvfipzcNog1lBxOLfnex91Hk6s= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0/go.mod h1:zVZ8nz+VSggWmnh6tTsJqXQ7rU4xLwRtna1M4x5jq58= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= +go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= @@ -307,8 +333,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -375,8 +401,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -394,8 +420,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -438,8 +464,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -450,7 +476,10 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -521,8 +550,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -552,6 +581,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -564,6 +599,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -576,8 +613,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/otel.go b/http/otel.go new file mode 100644 index 0000000..6b3f838 --- /dev/null +++ b/http/otel.go @@ -0,0 +1,178 @@ +package http + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + "github.com/go-akka/configuration" + + log "github.com/sirupsen/logrus" +) + +// Tracing contains the core dependencies to make tracing possible across an application. +type otelTracing struct { + providerName string + tracerProvider trace.TracerProvider + propagator propagation.TextMapPropagator +} + +type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) + +var ( + ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") + ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, + "stdout": stdoutTraceProvider, + "noop": noopTraceProvider, + } +) + +// DefaultTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultTracerProvider = "noop" + +// newOtel creates a structure with components that apps can use to initialize OpenTelemetry +// tracing instrumentation code. +func newOtel(conf *configuration.Config) (otelTracing, error) { + if IsNoOpTracing(conf) { + log.Debug("open telemetry tracing disabled (noop)") + } else { + log.Debug("opentelemetry tracing enabled") + } + + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + var tracing = otelTracing{ + providerName: providerName, + } + tracerProvider, err := newTracerProvider(conf) + if err != nil { + return otelTracing{}, err + } + tracing.tracerProvider = tracerProvider + otel.SetTracerProvider(tracerProvider) + + // Set up propagator. + prop := newPropagator() + tracing.propagator = prop + otel.SetTextMapPropagator(prop) + + return tracing, nil +} + +// IsNoOpTracing returns true if the provider is set to "noop" +func IsNoOpTracing(conf *configuration.Config) bool { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + return strings.EqualFold(providerName, "noop") +} + +// TracerProvider returns the tracer provider component. By default, the noop +// tracer provider is returned. +func (t otelTracing) TracerProvider() trace.TracerProvider { + if t.tracerProvider == nil { + return noop.NewTracerProvider() + } + return t.tracerProvider +} + +// Propagator returns the component that helps propagate trace context across +// API boundaries. By default, a W3C Trace Context format propagator is returned. +func (t otelTracing) Propagator() propagation.TextMapPropagator { + if t.propagator == nil { + return propagation.TraceContext{} + } + return t.propagator +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +// newTracerProvider creates the TracerProvider based on config setting +// If no config setting, a noop tracerProvider will be returned. +func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + if len(providerName) == 0 { + providerName = defaultTracerProvider + } + // Handling camelcase of provider. + providerName = strings.ToLower(providerName) + providerConfig := providersConfig[providerName] + if providerConfig == nil { + return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) + } + + traceProvider, err := providerConfig(conf) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + return traceProvider, nil +} + +func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + ) + return tp, nil +} + +func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + // Send traces over HTTP + endpoint := conf.GetString("webconfig.opentelemetry.endpoint") + if endpoint == "" { + return nil, ErrTracerProviderBuildFailed + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + + appName := conf.GetString("webconfig.app_name") + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(appName), + ), + ), + ), nil +} + +func (s *WebconfigServer) OtelShutdown() { + sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0d82f31..bd1385b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -28,6 +28,8 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/propagation" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" @@ -101,6 +103,7 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool + otelTracer *otelTracing // For OpenTelemetry Tracing } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -239,6 +242,11 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + otelTracer, err := newOtel(conf) + if err != nil { + // Just log err and continue + log.Error("Could not initialize open telemetry for tracing, but continuing") + } supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) @@ -271,6 +279,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, + otelTracer: &otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } } @@ -308,8 +317,9 @@ func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { func (s *WebconfigServer) NoAuthMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw := s.logRequestStarts(w, r) + xw, ctx := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) + r = r.WithContext(ctx) next.ServeHTTP(xw, r) } return http.HandlerFunc(fn) @@ -318,8 +328,9 @@ func (s *WebconfigServer) NoAuthMiddleware(next http.Handler) http.Handler { // Token valid and mac must match func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw := s.logRequestStarts(w, r) + xw, ctx := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) + r = r.WithContext(ctx) isValid := false token := xw.Token() @@ -358,8 +369,9 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { // Token valid enough func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw := s.logRequestStarts(w, r) + xw, ctx := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) + r = r.WithContext(ctx) isValid := false token := xw.Token() @@ -596,7 +608,7 @@ func getFilteredHeader(r *http.Request, notLoggedHeaders []string) http.Header { return header } -func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Request) *XResponseWriter { +func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Request) (*XResponseWriter, context.Context) { remoteIp := r.RemoteAddr host := r.Host header := getFilteredHeader(r, s.notLoggedHeaders) @@ -626,11 +638,12 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] } - // extrac tracestate from the header + // extract tracestate from the header tracestate := r.Header.Get(common.HeaderTracestate) if len(tracestate) > 0 { outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) } + ctx := injectTrace(r, outTraceparent, outTracestate) // extract auditid from the header auditId := r.Header.Get("X-Auditid") @@ -697,7 +710,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if err != nil { fields["error"] = err log.WithFields(fields).Error("request starts") - return xwriter + return xwriter, ctx } xwriter.SetBodyBytes(bbytes) } @@ -708,7 +721,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques log.WithFields(tfields).Info("request starts") } - return xwriter + return xwriter, ctx } func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { @@ -843,3 +856,13 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { } return itf, "" } + +func injectTrace(r *http.Request, traceparent string, tracestate string) (context.Context) { + propagator := propagation.TraceContext{} + ctx := r.Context() + ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) + var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} + textMapCarrier.Set(common.HeaderTraceparent, traceparent) + textMapCarrier.Set(common.HeaderTracestate, tracestate) + return propagation.TraceContext{}.Extract(ctx, textMapCarrier) +} diff --git a/main.go b/main.go index 11c8893..d5684ef 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 315f2d890b1cc13bbe08213bff498cdb513cb625 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 1 Jun 2024 17:29:31 -0700 Subject: [PATCH 030/215] Fix a bug that poke api returns 500 if webpa returns 403 --- common/server_config.go | 1 + http/otel.go | 17 +++++++++++++++++ http/poke_handler.go | 4 +--- http/poke_handler_test.go | 36 +++++++++++++++++++++++++++++++++--- http/webconfig_server.go | 4 ++-- http/webpa_connector.go | 14 +++++++++++--- 6 files changed, 65 insertions(+), 11 deletions(-) diff --git a/common/server_config.go b/common/server_config.go index 44fa5a7..0873776 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -30,6 +30,7 @@ var ( "/app/webconfig/test_webconfig.conf", "../config/sample_webconfig.conf", "/app/webconfig/webconfig.conf", + "/app/webconfig/conf/webconfig.conf", } ) diff --git a/http/otel.go b/http/otel.go index 6b3f838..3028948 100644 --- a/http/otel.go +++ b/http/otel.go @@ -1,3 +1,20 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ package http import ( diff --git a/http/poke_handler.go b/http/poke_handler.go index 62c4092..ac50ba5 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -161,11 +161,9 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - status := http.StatusInternalServerError + status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 - } else if rherr.StatusCode > http.StatusInternalServerError { - status = rherr.StatusCode } // parse the core message diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 29c3364..ba63157 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -35,7 +35,8 @@ import ( ) var ( - mockedWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) ) func TestPokeHandler(t *testing.T) { @@ -47,7 +48,7 @@ func TestPokeHandler(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -286,7 +287,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -305,3 +306,32 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa403(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write(mockWebpaPoke403Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index bd1385b..c2dc929 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -857,8 +857,8 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func injectTrace(r *http.Request, traceparent string, tracestate string) (context.Context) { - propagator := propagation.TraceContext{} +func injectTrace(r *http.Request, traceparent string, tracestate string) context.Context { + propagator := propagation.TraceContext{} ctx := r.Context() ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 021f049..d7681f4 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -251,18 +251,26 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field } func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - var cont bool - + tfields := common.FilterLogFields(fields) + tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) copy(cbytes, bbytes) if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont = c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { + msg := fmt.Sprintf("finished success after 1 retry") + if i > 1 { + fmt.Sprintf("finished success after %v retries", i) + } + log.WithFields(tfields).Info(msg) break } + if i == c.retries { + log.WithFields(tfields).Infof("finished failure after %v retries", i) + } } <-c.queue } From af7d8c83ab467c985a84e7e86dc2f7cc8a17c536 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 1 Jun 2024 17:29:31 -0700 Subject: [PATCH 031/215] cherrypick a poke403 handling fix and resolve conflicts --- common/server_config.go | 1 + http/poke_handler.go | 4 +--- http/poke_handler_test.go | 36 +++++++++++++++++++++++++++++++++--- http/webpa_connector.go | 14 +++++++++++--- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/common/server_config.go b/common/server_config.go index 44fa5a7..0873776 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -30,6 +30,7 @@ var ( "/app/webconfig/test_webconfig.conf", "../config/sample_webconfig.conf", "/app/webconfig/webconfig.conf", + "/app/webconfig/conf/webconfig.conf", } ) diff --git a/http/poke_handler.go b/http/poke_handler.go index 62c4092..ac50ba5 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -161,11 +161,9 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - status := http.StatusInternalServerError + status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 - } else if rherr.StatusCode > http.StatusInternalServerError { - status = rherr.StatusCode } // parse the core message diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 29c3364..ba63157 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -35,7 +35,8 @@ import ( ) var ( - mockedWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) ) func TestPokeHandler(t *testing.T) { @@ -47,7 +48,7 @@ func TestPokeHandler(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -286,7 +287,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -305,3 +306,32 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa403(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write(mockWebpaPoke403Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 021f049..d7681f4 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -251,18 +251,26 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field } func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - var cont bool - + tfields := common.FilterLogFields(fields) + tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) copy(cbytes, bbytes) if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont = c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { + msg := fmt.Sprintf("finished success after 1 retry") + if i > 1 { + fmt.Sprintf("finished success after %v retries", i) + } + log.WithFields(tfields).Info(msg) break } + if i == c.retries { + log.WithFields(tfields).Infof("finished failure after %v retries", i) + } } <-c.queue } From 123411a0d2aa40b8d80d9f60d56808b37339af9f Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 13 Jun 2024 05:49:40 -0700 Subject: [PATCH 032/215] Otel Changes --- config/sample_webconfig.conf | 11 ++--- go.mod | 77 +++++++++++++++++----------------- go.sum | 80 ++++++++++++++++++++++++++++++++++++ http/http_client.go | 36 +++++++++------- http/otel.go | 37 +++++++++++++---- http/webconfig_server.go | 16 ++++++-- 6 files changed, 188 insertions(+), 69 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index a36a3d7..fbfc38b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -8,11 +8,12 @@ webconfig { tracestate_vendor_id = "webconfig" opentelemetry { endpoint = "127.0.0.1:4318" - // Allowed values: "noop", "stdout", "http" - // "noop" will generate no trace - // "stdout" will use stdoutTracer and output spans to stdout - // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector - provider = "noop", + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop" + env_name = "dev" } // build info diff --git a/go.mod b/go.mod index cfb4bb9..188e4df 100644 --- a/go.mod +++ b/go.mod @@ -3,48 +3,51 @@ module github.com/rdkcentral/webconfig go 1.21 require ( - github.com/IBM/sarama v1.42.1 + github.com/IBM/sarama v1.43.2 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 - github.com/golang-jwt/jwt/v5 v5.0.0 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.0 - github.com/mattn/go-sqlite3 v1.14.15 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 - github.com/sirupsen/logrus v1.9.0 - github.com/twmb/murmur3 v1.1.6 + github.com/gorilla/mux v1.8.1 + github.com/mattn/go-sqlite3 v1.14.22 + github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_model v0.6.1 + github.com/sirupsen/logrus v1.9.3 + github.com/twmb/murmur3 v1.1.8 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/otel v1.26.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 - go.opentelemetry.io/otel/sdk v1.26.0 - go.opentelemetry.io/otel/trace v1.26.0 - go.uber.org/automaxprocs v1.5.1 - go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.6.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 + go.uber.org/automaxprocs v1.5.3 + go.uber.org/ratelimit v0.3.1 + golang.org/x/sync v0.7.0 gotest.tools v2.2.0+incompatible ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.4.0 // indirect + github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -52,25 +55,25 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.54.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect - go.opentelemetry.io/otel/metric v1.26.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 8505ce4..9e2917d 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= +github.com/IBM/sarama v1.43.2 h1:HABeEqRUh32z8yzY2hGB/j8mHSzC/HA9zlEjqFNCzSw= +github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMdXFQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -44,6 +46,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -59,6 +63,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -69,6 +75,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30= +github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= @@ -77,6 +85,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= @@ -95,6 +105,8 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -103,6 +115,8 @@ github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJr github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -165,14 +179,20 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -205,6 +225,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -217,8 +239,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -228,6 +254,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -243,17 +271,23 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= +github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -261,6 +295,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -271,6 +307,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -287,6 +325,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= @@ -303,28 +343,50 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 h1:0W5o9SzoR15ocYHEQfvfipzcNog1lBxOLfnex91Hk6s= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0/go.mod h1:zVZ8nz+VSggWmnh6tTsJqXQ7rU4xLwRtna1M4x5jq58= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -335,6 +397,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -403,6 +467,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -422,6 +488,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -466,6 +534,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -480,6 +550,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -585,8 +657,12 @@ google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUE google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 h1:QW9+G6Fir4VcRXVH8x3LilNAb6cxBGLa6+GM4hRwexE= +google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3/go.mod h1:kdrSS/OiLkPrNUpzD4aHgCq2rVuC/YRxok32HXZ4vRE= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -601,6 +677,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -615,6 +693,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/http_client.go b/http/http_client.go index dfc0459..c6cc70b 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -32,11 +32,13 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -82,20 +84,26 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) userAgent := conf.GetString("webconfig.http_client.user_agent") + var transport http.RoundTripper = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: time.Duration(connectTimeout) * time.Second, + KeepAlive: time.Duration(keepaliveTimeout) * time.Second, + }).DialContext, + MaxIdleConns: 0, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: tlsConfig, + } + transport = otelhttp.NewTransport(transport, + otelhttp.WithPropagators(otelTracer.propagator), + otelhttp.WithTracerProvider(otelTracer.tracerProvider), + ) + return &HttpClient{ Client: &http.Client{ - Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: time.Duration(connectTimeout) * time.Second, - KeepAlive: time.Duration(keepaliveTimeout) * time.Second, - }).DialContext, - MaxIdleConns: 0, - MaxIdleConnsPerHost: maxIdleConnsPerHost, - IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientConfig: tlsConfig, - }, + Transport: transport, Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, diff --git a/http/otel.go b/http/otel.go index 6b3f838..920bebd 100644 --- a/http/otel.go +++ b/http/otel.go @@ -1,3 +1,20 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ package http import ( @@ -25,6 +42,7 @@ import ( // Tracing contains the core dependencies to make tracing possible across an application. type otelTracing struct { providerName string + envName string tracerProvider trace.TracerProvider propagator propagation.TextMapPropagator } @@ -39,6 +57,8 @@ var ( "stdout": stdoutTraceProvider, "noop": noopTraceProvider, } + + otelTracer otelTracing ) // DefaultTracerProvider is used when no provider is given. @@ -48,30 +68,28 @@ const defaultTracerProvider = "noop" // newOtel creates a structure with components that apps can use to initialize OpenTelemetry // tracing instrumentation code. -func newOtel(conf *configuration.Config) (otelTracing, error) { +func newOtel(conf *configuration.Config) (*otelTracing, error) { if IsNoOpTracing(conf) { log.Debug("open telemetry tracing disabled (noop)") } else { log.Debug("opentelemetry tracing enabled") } - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - var tracing = otelTracing{ - providerName: providerName, - } + otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") tracerProvider, err := newTracerProvider(conf) if err != nil { - return otelTracing{}, err + return &otelTracer, err } - tracing.tracerProvider = tracerProvider + otelTracer.tracerProvider = tracerProvider otel.SetTracerProvider(tracerProvider) // Set up propagator. prop := newPropagator() - tracing.propagator = prop + otelTracer.propagator = prop otel.SetTextMapPropagator(prop) - return tracing, nil + return &otelTracer, nil } // IsNoOpTracing returns true if the provider is set to "noop" @@ -165,6 +183,7 @@ func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), ), ), ), nil diff --git a/http/webconfig_server.go b/http/webconfig_server.go index bd1385b..a85c096 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -28,7 +28,9 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" @@ -279,7 +281,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, - otelTracer: &otelTracer, + otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } } @@ -371,8 +373,8 @@ func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { xw, ctx := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) - r = r.WithContext(ctx) + r = r.WithContext(ctx) isValid := false token := xw.Token() if len(token) > 0 { @@ -857,12 +859,18 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func injectTrace(r *http.Request, traceparent string, tracestate string) (context.Context) { - propagator := propagation.TraceContext{} +func injectTrace(r *http.Request, traceparent string, tracestate string) context.Context { ctx := r.Context() + + propagator := propagation.TraceContext{} ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} textMapCarrier.Set(common.HeaderTraceparent, traceparent) textMapCarrier.Set(common.HeaderTracestate, tracestate) + + span := trace.SpanFromContext(ctx) + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) + return propagation.TraceContext{}.Extract(ctx, textMapCarrier) } From 7fdcc9b808e13d9d152e1d8fe94a89149035a07c Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 13 Jun 2024 14:39:49 -0700 Subject: [PATCH 033/215] Otel changes second iteration Restrict opentelemetry to poke only Instrument the API using middleware instead of instrumenting the client library --- http/otel.go | 14 ++++++-- http/router.go | 1 + http/webconfig_server.go | 77 +++++++++++++++++++++++++++++++++++----- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/http/otel.go b/http/otel.go index 920bebd..1faa742 100644 --- a/http/otel.go +++ b/http/otel.go @@ -43,8 +43,10 @@ import ( type otelTracing struct { providerName string envName string + appName string tracerProvider trace.TracerProvider propagator propagation.TextMapPropagator + tracer trace.Tracer } type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) @@ -75,6 +77,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { log.Debug("opentelemetry tracing enabled") } + otelTracer.appName = conf.GetString("webconfig.app_name") otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") tracerProvider, err := newTracerProvider(conf) @@ -89,6 +92,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { otelTracer.propagator = prop otel.SetTextMapPropagator(prop) + otelTracer.tracer = otel.Tracer(otelTracer.appName) return &otelTracer, nil } @@ -158,6 +162,13 @@ func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, erro sdktrace.WithBatcher(exporter, // Default is 5s. Set to 1s for demonstrative purposes. sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), ) return tp, nil } @@ -176,13 +187,12 @@ func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) } - appName := conf.GetString("webconfig.app_name") return sdktrace.NewTracerProvider( sdktrace.WithBatcher(exporter), sdktrace.WithResource( resource.NewWithAttributes( semconv.SchemaURL, - semconv.ServiceNameKey.String(appName), + semconv.ServiceNameKey.String(otelTracer.appName), semconv.ServiceNamespaceKey.String(otelTracer.envName), ), ), diff --git a/http/router.go b/http/router.go index 746bec0..c6ddaf1 100644 --- a/http/router.go +++ b/http/router.go @@ -91,6 +91,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() + sub2.Use(spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { diff --git a/http/webconfig_server.go b/http/webconfig_server.go index a85c096..2069acb 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -20,6 +20,7 @@ package http import ( "context" "crypto/tls" + "encoding/hex" "encoding/json" "fmt" "io" @@ -29,7 +30,6 @@ import ( "time" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" "github.com/rdkcentral/webconfig/common" @@ -77,6 +77,7 @@ var ( "privatessid", "homessid", } + ws *WebconfigServer ) type WebconfigServer struct { @@ -252,7 +253,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - return &WebconfigServer{ + ws = &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -284,6 +285,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } + return ws } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -645,7 +647,8 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if len(tracestate) > 0 { outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) } - ctx := injectTrace(r, outTraceparent, outTracestate) + ctx := r.Context() + // ctx := injectTrace(r, outTraceparent, outTracestate) // extract auditid from the header auditId := r.Header.Get("X-Auditid") @@ -859,18 +862,74 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } +/* func injectTrace(r *http.Request, traceparent string, tracestate string) context.Context { ctx := r.Context() - propagator := propagation.TraceContext{} - ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} textMapCarrier.Set(common.HeaderTraceparent, traceparent) textMapCarrier.Set(common.HeaderTracestate, tracestate) - span := trace.SpanFromContext(ctx) - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) + propagator := propagation.TraceContext{} + return propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) + // return propagator.Extract(ctx, textMapCarrier) +} +*/ + +func spanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + spanContext := trace.SpanContextFromContext(ctx) + remote := spanContext.IsRemote() + sc := trace.SpanContext{}.WithRemote(remote) + + traceIDStr, traceFlagsStr := parseTraceparent(r) + if traceIDStr != "" { + traceID, _ := trace.TraceIDFromHex(traceIDStr) + sc = sc.WithTraceID(traceID) + } + if traceFlagsStr != "" { + traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) + sc = sc.WithTraceFlags(traceFlags) + } + tracestateStr := getTracestate(r) + if tracestateStr != "" { + tracestate, _ := trace.ParseTraceState(tracestateStr) + sc = sc.WithTraceState(tracestate) + } + ctx = trace.ContextWithSpanContext(ctx, sc) + ctx, span := otelTracer.tracer.Start(ctx, "mytest") + defer span.End() + + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) + + // Pass the context with the span to the next handler + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// extract traceparent from the header +func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { + inTraceparent := r.Header.Get(common.HeaderTraceparent) + if len(inTraceparent) == 55 { + traceID = inTraceparent[3:35] + traceFlags = inTraceparent[53:55] + } + return +} + +// extract tracestate from the header +func getTracestate(r *http.Request) string { + inTracestate := r.Header.Get(common.HeaderTracestate) + var outTracestate string + if len(inTracestate) > 0 { + outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, ws.TracestateVendorID(), ws.TraceparentParentID()) + } + return outTracestate +} - return propagation.TraceContext{}.Extract(ctx, textMapCarrier) +func hexStringToBytes(hexString string) []byte { + bytes, _ := hex.DecodeString(hexString) + return bytes } From e74912d0a4379be4e73dea04ddc2d143ca7dc690 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 13 Jun 2024 16:00:24 -0700 Subject: [PATCH 034/215] Code review feedback plus minor simplifications --- http/webconfig_server.go | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 2069acb..3a6082b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -321,9 +321,8 @@ func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { func (s *WebconfigServer) NoAuthMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw, ctx := s.logRequestStarts(w, r) + xw := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) - r = r.WithContext(ctx) next.ServeHTTP(xw, r) } return http.HandlerFunc(fn) @@ -332,9 +331,8 @@ func (s *WebconfigServer) NoAuthMiddleware(next http.Handler) http.Handler { // Token valid and mac must match func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw, ctx := s.logRequestStarts(w, r) + xw := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) - r = r.WithContext(ctx) isValid := false token := xw.Token() @@ -373,10 +371,9 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { // Token valid enough func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw, ctx := s.logRequestStarts(w, r) + xw := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) - r = r.WithContext(ctx) isValid := false token := xw.Token() if len(token) > 0 { @@ -612,7 +609,7 @@ func getFilteredHeader(r *http.Request, notLoggedHeaders []string) http.Header { return header } -func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Request) (*XResponseWriter, context.Context) { +func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Request) *XResponseWriter { remoteIp := r.RemoteAddr host := r.Host header := getFilteredHeader(r, s.notLoggedHeaders) @@ -647,9 +644,6 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if len(tracestate) > 0 { outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) } - ctx := r.Context() - // ctx := injectTrace(r, outTraceparent, outTracestate) - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { @@ -715,7 +709,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if err != nil { fields["error"] = err log.WithFields(fields).Error("request starts") - return xwriter, ctx + return xwriter } xwriter.SetBodyBytes(bbytes) } @@ -726,7 +720,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques log.WithFields(tfields).Info("request starts") } - return xwriter, ctx + return xwriter } func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { @@ -862,20 +856,6 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -/* -func injectTrace(r *http.Request, traceparent string, tracestate string) context.Context { - ctx := r.Context() - - var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} - textMapCarrier.Set(common.HeaderTraceparent, traceparent) - textMapCarrier.Set(common.HeaderTracestate, tracestate) - - propagator := propagation.TraceContext{} - return propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) - // return propagator.Extract(ctx, textMapCarrier) -} -*/ - func spanMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -898,7 +878,7 @@ func spanMiddleware(next http.Handler) http.Handler { sc = sc.WithTraceState(tracestate) } ctx = trace.ContextWithSpanContext(ctx, sc) - ctx, span := otelTracer.tracer.Start(ctx, "mytest") + ctx, span := otelTracer.tracer.Start(ctx, "oswebconfig_poke_handler") defer span.End() attr := attribute.String("env", otelTracer.envName) From b0f9841116097eb2b76783fdddf0e24ea47e93ee Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 15 Jun 2024 12:00:33 -0700 Subject: [PATCH 035/215] handle corrupted encrypted blobs --- db/cassandra/document.go | 7 +- http/document_handler.go | 1 + http/http_client.go | 7 +- http/multipart_test.go | 164 +++++++++++++++++++++++++++++ http/otel.go | 10 +- http/router.go | 2 +- http/supplementary_handler_test.go | 3 - http/webconfig_server.go | 12 +-- 8 files changed, 184 insertions(+), 22 deletions(-) diff --git a/db/cassandra/document.go b/db/cassandra/document.go index 2eaf58c..df580b8 100644 --- a/db/cassandra/document.go +++ b/db/cassandra/document.go @@ -29,7 +29,6 @@ import ( log "github.com/sirupsen/logrus" ) -// NOTE this func (c *CassandraClient) GetSubDocument(cpeMac string, groupId string) (*common.SubDocument, error) { var err error var payload []byte @@ -283,7 +282,11 @@ func (c *CassandraClient) GetDocument(cpeMac string, xargs ...interface{}) (fndo if c.IsEncryptedGroup(groupId) { payload, err = c.DecryptBytes(payload) if err != nil { - return nil, common.NewError(err) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "subdoc" + tfields["subdoc_id"] = groupId + log.WithFields(tfields).Warn(err) + continue } } diff --git a/http/document_handler.go b/http/document_handler.go index f41f7ff..3f3e834 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -301,6 +301,7 @@ func (s *WebconfigServer) DeleteDocumentHandler(w http.ResponseWriter, r *http.R if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } diff --git a/http/http_client.go b/http/http_client.go index c6cc70b..790d907 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -32,13 +32,12 @@ import ( "strings" "time" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" ) const ( @@ -104,7 +103,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl return &HttpClient{ Client: &http.Client{ Transport: transport, - Timeout: time.Duration(readTimeout) * time.Second, + Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, retryInMsecs: retryInMsecs, diff --git a/http/multipart_test.go b/http/multipart_test.go index 30919f8..2aed59a 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -28,6 +28,7 @@ import ( "testing" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "github.com/vmihailenco/msgpack/v4" @@ -1104,3 +1105,166 @@ func TestStateCorrectionEnabled(t *testing.T) { assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) } + +func TestCorruptedEncryptedDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + tdbclient, ok := server.DatabaseClient.(*cassandra.CassandraClient) + if !ok { + t.Skip("Only test in cassandra env") + } + + cpeMac := util.GenerateRandomCpeMac() + encSubdocIds := []string{} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds := tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, !tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 1 setup lan subdoc ==== + // post + subdocId := "lan" + lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + lanBytes := util.RandomBytes(100, 150) + req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + _ = rbytes + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", lanUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 2 setup wan subdoc ==== + // post + subdocId = "wan" + wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + wanBytes := util.RandomBytes(100, 150) + req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", wanUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 3 setup privatessid subdoc ==== + // post + subdocId = "privatessid" + privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + privatessidBytes := util.RandomBytes(100, 150) + req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", privatessidUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 read the document ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 3) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + mpart, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + + // ==== step 5 set privatessid as an encrypted subdoc ==== + encSubdocIds = []string{"privatessid"} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds = tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 6 read the document expect no error but 1 less subdoc ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + _, ok = mpartMap["privatessid"] + assert.Assert(t, !ok) +} diff --git a/http/otel.go b/http/otel.go index 1faa742..e0a806e 100644 --- a/http/otel.go +++ b/http/otel.go @@ -27,10 +27,10 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" @@ -45,7 +45,7 @@ type otelTracing struct { envName string appName string tracerProvider trace.TracerProvider - propagator propagation.TextMapPropagator + propagator propagation.TextMapPropagator tracer trace.Tracer } @@ -54,10 +54,10 @@ type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, var ( ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") - providersConfig = map[string]providerConstructor{ - "http": httpTraceProvider, + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, "stdout": stdoutTraceProvider, - "noop": noopTraceProvider, + "noop": noopTraceProvider, } otelTracer otelTracing diff --git a/http/router.go b/http/router.go index c6ddaf1..bc7b0d6 100644 --- a/http/router.go +++ b/http/router.go @@ -91,7 +91,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() - sub2.Use(spanMiddleware) + sub2.Use(s.spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index b7007e5..942b0f8 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -732,9 +732,6 @@ func TestSupplementaryAppendingFlag(t *testing.T) { // ==== step 3 verify the query params ==== assert.Assert(t, strings.Contains(ss, "&stormReadyWifi=true")) - // okok := false - // assert.Assert(t, okok) - // ==== step 4 set append flag false ==== appendEnabled = false server.SetSupplementaryAppendingEnabled(appendEnabled) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 3a6082b..e9800bb 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -77,7 +77,6 @@ var ( "privatessid", "homessid", } - ws *WebconfigServer ) type WebconfigServer struct { @@ -253,7 +252,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - ws = &WebconfigServer{ + return &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -285,7 +284,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } - return ws } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -856,7 +854,7 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func spanMiddleware(next http.Handler) http.Handler { +func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() spanContext := trace.SpanContextFromContext(ctx) @@ -872,7 +870,7 @@ func spanMiddleware(next http.Handler) http.Handler { traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) sc = sc.WithTraceFlags(traceFlags) } - tracestateStr := getTracestate(r) + tracestateStr := s.getTracestate(r) if tracestateStr != "" { tracestate, _ := trace.ParseTraceState(tracestateStr) sc = sc.WithTraceState(tracestate) @@ -900,11 +898,11 @@ func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { } // extract tracestate from the header -func getTracestate(r *http.Request) string { +func (s *WebconfigServer) getTracestate(r *http.Request) string { inTracestate := r.Header.Get(common.HeaderTracestate) var outTracestate string if len(inTracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, ws.TracestateVendorID(), ws.TraceparentParentID()) + outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, s.TracestateVendorID(), s.TraceparentParentID()) } return outTracestate } From 6062b63266350f338e838ee08e34fa07141b95cb Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Mon, 17 Jun 2024 11:52:07 -0700 Subject: [PATCH 036/215] Cherrypick of otel changes --- common/const_var.go | 1 + common/refsubdocument.go | 81 +++++++++++ common/refsubdocument_test.go | 40 ++++++ common/server_config.go | 1 + config/sample_webconfig.conf | 9 ++ db/cassandra/refsubdocument.go | 81 +++++++++++ db/cassandra/refsubdocument_test.go | 59 ++++++++ db/cassandra/schema.go | 5 + db/database_client.go | 5 + db/service.go | 36 +++++ db/sqlite/refsubdocument.go | 153 ++++++++++++++++++++ db/sqlite/refsubdocument_test.go | 59 ++++++++ db/sqlite/schema.go | 5 + go.mod | 40 ++++-- go.sum | 89 ++++++++---- http/http_client.go | 36 +++-- http/multipart.go | 23 ++++ http/otel.go | 207 ++++++++++++++++++++++++++++ http/poke_handler.go | 4 +- http/poke_handler_test.go | 36 ++++- http/refsubdocument_handler.go | 125 +++++++++++++++++ http/refsubdocument_handler_test.go | 76 ++++++++++ http/router.go | 15 ++ http/validator.go | 37 +++++ http/webconfig_server.go | 76 +++++++++- http/webpa_connector.go | 14 +- main.go | 1 + 27 files changed, 1251 insertions(+), 63 deletions(-) create mode 100644 common/refsubdocument.go create mode 100644 common/refsubdocument_test.go create mode 100644 db/cassandra/refsubdocument.go create mode 100644 db/cassandra/refsubdocument_test.go create mode 100644 db/sqlite/refsubdocument.go create mode 100644 db/sqlite/refsubdocument_test.go create mode 100644 http/otel.go create mode 100644 http/refsubdocument_handler.go create mode 100644 http/refsubdocument_handler_test.go diff --git a/common/const_var.go b/common/const_var.go index 8efae88..2fcf2af 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -106,6 +106,7 @@ const ( HeaderTraceparent = "Traceparent" HeaderTracestate = "Tracestate" HeaderContentLength = "Content-Length" + HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" ) // header X-System-Supported-Docs diff --git a/common/refsubdocument.go b/common/refsubdocument.go new file mode 100644 index 0000000..cad7ffc --- /dev/null +++ b/common/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "bytes" +) + +type RefSubDocument struct { + payload []byte + version *string +} + +func NewRefSubDocument(payload []byte, version *string) *RefSubDocument { + return &RefSubDocument{ + payload: payload, + version: version, + } +} + +func (d *RefSubDocument) Payload() []byte { + return d.payload +} + +func (d *RefSubDocument) SetPayload(payload []byte) { + d.payload = payload +} + +func (d *RefSubDocument) HasPayload() bool { + if d.payload != nil && len(d.payload) > 0 { + return true + } else { + return false + } +} + +func (d *RefSubDocument) Version() *string { + return d.version +} + +func (d *RefSubDocument) SetVersion(version *string) { + d.version = version +} + +func (d *RefSubDocument) Equals(tdoc *RefSubDocument) bool { + if d.HasPayload() && tdoc.HasPayload() { + if !bytes.Equal(d.Payload(), tdoc.Payload()) { + return false + } + } else { + if d.HasPayload() != tdoc.HasPayload() { + return false + } + } + + if d.Version() != nil && tdoc.Version() != nil { + if *d.Version() != *tdoc.Version() { + return false + } + } else { + if d.Version() != tdoc.Version() { + return false + } + } + return true +} diff --git a/common/refsubdocument_test.go b/common/refsubdocument_test.go new file mode 100644 index 0000000..416c123 --- /dev/null +++ b/common/refsubdocument_test.go @@ -0,0 +1,40 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestRefSubDocument(t *testing.T) { + bbytes1 := []byte("hello world") + version1 := "12345" + refsubdoc1 := NewRefSubDocument(bbytes1, &version1) + + bbytes2 := []byte("hello world") + version2 := "12345" + refsubdoc2 := NewRefSubDocument(bbytes2, &version2) + assert.Assert(t, refsubdoc1.Equals(refsubdoc2)) + + bbytes3 := []byte("foo bar") + version3 := "12345" + refsubdoc3 := NewRefSubDocument(bbytes3, &version3) + assert.Assert(t, !refsubdoc1.Equals(refsubdoc3)) +} diff --git a/common/server_config.go b/common/server_config.go index 44fa5a7..0873776 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -30,6 +30,7 @@ var ( "/app/webconfig/test_webconfig.conf", "../config/sample_webconfig.conf", "/app/webconfig/webconfig.conf", + "/app/webconfig/conf/webconfig.conf", } ) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 894f513..fbfc38b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -6,6 +6,15 @@ webconfig { panic_exit_enabled = false traceparent_parent_id = "0000000000000001" tracestate_vendor_id = "webconfig" + opentelemetry { + endpoint = "127.0.0.1:4318" + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop" + env_name = "dev" + } // build info code_git_commit = "2ac7ff4" diff --git a/db/cassandra/refsubdocument.go b/db/cassandra/refsubdocument.go new file mode 100644 index 0000000..1c9ca9f --- /dev/null +++ b/db/cassandra/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/gocql/gocql" +) + +func (c *CassandraClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + var payload []byte + var version string + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "SELECT payload,version FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Scan(&payload, &version); err != nil { + return nil, common.NewError(err) + } + + if len(payload) == 0 { + return nil, common.NewError(gocql.ErrNotFound) + } + + refsubdoc := common.NewRefSubDocument(payload, &version) + return refsubdoc, nil +} + +func (c *CassandraClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) (fnerr error) { + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil && len(refsubdoc.Payload()) > 0 { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + stmt := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + if err := c.Query(stmt, values...).Exec(); err != nil { + return common.NewError(err) + } + return nil +} + +func (c *CassandraClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "DELETE FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Exec(); err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/cassandra/refsubdocument_test.go b/db/cassandra/refsubdocument_test.go new file mode 100644 index 0000000..22e0ba8 --- /dev/null +++ b/db/cassandra/refsubdocument_test.go @@ -0,0 +1,59 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "crypto/rand" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + slen := util.RandomInt(100) + 16 + srcBytes := make([]byte, slen) + rand.Read(srcBytes) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index 4d0ac95..b013804 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -44,6 +44,11 @@ var ( route text, schema_version text, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } diff --git a/db/database_client.go b/db/database_client.go index 397c995..fa1065f 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -60,6 +60,11 @@ type DatabaseClient interface { FirmwareUpdate(string, int, *common.RootDocument) error AppendProfiles(string, []byte) ([]byte, error) + // reference subdocument + GetRefSubDocument(string) (*common.RefSubDocument, error) + SetRefSubDocument(string, *common.RefSubDocument) error + DeleteRefSubDocument(string) error + // enable state correction StateCorrectionEnabled() bool SetStateCorrectionEnabled(bool) diff --git a/db/service.go b/db/service.go index ca77dc2..882d0c4 100644 --- a/db/service.go +++ b/db/service.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "slices" "sort" "strings" "time" @@ -32,6 +33,14 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + referenceIndicatorByteLength = 4 +) + +var ( + referenceIndicatorBytes = make([]byte, referenceIndicatorByteLength) +) + // TODO s.MultipartSupplementaryHandler(w, r) should be handled separately // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream @@ -558,3 +567,30 @@ func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerI } return cloudRootDocument, nil } + +func GetRefId(payload []byte) (string, bool) { + if len(payload) > referenceIndicatorByteLength { + prefixBytes := payload[:referenceIndicatorByteLength] + if slices.Equal(referenceIndicatorBytes, prefixBytes) { + suffixBytes := payload[referenceIndicatorByteLength:] + return string(suffixBytes), true + } + } + return "", false +} + +func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log.Fields) (*common.Document, error) { + newDocument := common.NewDocument(document.GetRootDocument()) + for subdocId, subDocument := range document.Items() { + payload := subDocument.Payload() + if refId, ok := GetRefId(payload); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + return nil, common.NewError(err) + } + subDocument.SetPayload(refsubdocument.Payload()) + } + newDocument.SetSubDocument(subdocId, &subDocument) + } + return newDocument, nil +} diff --git a/db/sqlite/refsubdocument.go b/db/sqlite/refsubdocument.go new file mode 100644 index 0000000..b93d4eb --- /dev/null +++ b/db/sqlite/refsubdocument.go @@ -0,0 +1,153 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "database/sql" + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + _ "github.com/mattn/go-sqlite3" +) + +func (c *SqliteClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + rows, err := c.Query("SELECT payload,version FROM reference_document WHERE ref_id=?", refId) + if err != nil { + return nil, common.NewError(err) + } + + var ns1 sql.NullString + var b1 []byte + + if !rows.Next() { + return nil, sql.ErrNoRows + } + err = rows.Scan(&b1, &ns1) + defer rows.Close() + if err != nil { + return nil, common.NewError(err) + } + + var s1 *string + if ns1.Valid { + s1 = &(ns1.String) + } + + refsubdoc := common.NewRefSubDocument(b1, s1) + return refsubdoc, nil +} + +func (c *SqliteClient) insertRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + qstr := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) updateRefSubDocument(refId string, doc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{} + values := []interface{}{} + if doc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, doc.Payload()) + } + if doc.Version() != nil { + columns = append(columns, "version") + values = append(values, doc.Version()) + } + values = append(values, refId) + qstr := fmt.Sprintf("UPDATE reference_document SET %v WHERE cpe_mac=? AND ref_id=?", db.GetSetColumnsStr(columns)) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + _, err := c.GetRefSubDocument(refId) + if err != nil { + if c.IsDbNotFound(err) { + err1 := c.insertRefSubDocument(refId, refsubdoc) + if err1 != nil { + return common.NewError(err1) + } + } else { + // unexpected error + return common.NewError(err) + } + } else { + // normal dbNotFound should not happen + err = c.updateRefSubDocument(refId, refsubdoc) + if err != nil { + return common.NewError(err) + } + } + + return nil +} + +func (c *SqliteClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt, err := c.Prepare("DELETE FROM reference_document WHERE ref_id=?") + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(refId) + if err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/sqlite/refsubdocument_test.go b/db/sqlite/refsubdocument_test.go new file mode 100644 index 0000000..01478f9 --- /dev/null +++ b/db/sqlite/refsubdocument_test.go @@ -0,0 +1,59 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "crypto/rand" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + slen := util.RandomInt(100) + 16 + srcBytes := make([]byte, slen) + rand.Read(srcBytes) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 8ee8d6e..3e4c132 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -50,6 +50,11 @@ var ( route text, schema_version, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } ) diff --git a/go.mod b/go.mod index bbdcee4..2c15107 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt/v5 v5.0.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.0 github.com/mattn/go-sqlite3 v1.14.15 github.com/prometheus/client_golang v1.13.0 @@ -17,23 +17,34 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.4.0 + golang.org/x/sync v0.6.0 gotest.tools v2.2.0+incompatible ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -44,19 +55,24 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index f502ae6..b6af8c0 100644 --- a/go.sum +++ b/go.sum @@ -52,16 +52,18 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -75,6 +77,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= @@ -90,6 +94,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= @@ -122,8 +131,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -138,8 +148,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -151,14 +161,16 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -199,8 +211,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -254,8 +266,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -273,8 +285,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -293,6 +305,24 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= @@ -307,8 +337,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -375,8 +405,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -394,8 +424,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -438,8 +468,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -450,7 +480,10 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -521,8 +554,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -552,6 +585,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -564,6 +601,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -576,8 +615,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/http_client.go b/http/http_client.go index dfc0459..c6cc70b 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -32,11 +32,13 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -82,20 +84,26 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) userAgent := conf.GetString("webconfig.http_client.user_agent") + var transport http.RoundTripper = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: time.Duration(connectTimeout) * time.Second, + KeepAlive: time.Duration(keepaliveTimeout) * time.Second, + }).DialContext, + MaxIdleConns: 0, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: tlsConfig, + } + transport = otelhttp.NewTransport(transport, + otelhttp.WithPropagators(otelTracer.propagator), + otelhttp.WithTracerProvider(otelTracer.tracerProvider), + ) + return &HttpClient{ Client: &http.Client{ - Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: time.Duration(connectTimeout) * time.Second, - KeepAlive: time.Duration(keepaliveTimeout) * time.Second, - }).DialContext, - MaxIdleConns: 0, - MaxIdleConnsPerHost: maxIdleConnsPerHost, - IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientConfig: tlsConfig, - }, + Transport: transport, Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, diff --git a/http/multipart.go b/http/multipart.go index 227364f..b2860af 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -147,6 +147,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document.DeleteSubDocument(subdocId) } + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } respBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -178,9 +182,20 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document = common.NewDocument(rootDocument) } + if userAgent == "mget" { + postUpstream = false + } + var respBytes []byte respStatus := http.StatusNotModified if document.Length() > 0 { + + if !postUpstream { + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + } respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -286,6 +301,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusNotModified, upstreamRespHeader, nil, nil } + finalFilteredDocument, err = db.LoadRefSubDocuments(c, finalFilteredDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalFilteredBytes, err := finalFilteredDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalFilteredBytes, common.NewError(err) @@ -397,6 +416,10 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusNotFound, upstreamRespHeader, nil, nil } + finalDocument, err = db.LoadRefSubDocuments(c, finalDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalBytes, err := finalDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) diff --git a/http/otel.go b/http/otel.go new file mode 100644 index 0000000..1faa742 --- /dev/null +++ b/http/otel.go @@ -0,0 +1,207 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + "github.com/go-akka/configuration" + + log "github.com/sirupsen/logrus" +) + +// Tracing contains the core dependencies to make tracing possible across an application. +type otelTracing struct { + providerName string + envName string + appName string + tracerProvider trace.TracerProvider + propagator propagation.TextMapPropagator + tracer trace.Tracer +} + +type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) + +var ( + ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") + ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, + "stdout": stdoutTraceProvider, + "noop": noopTraceProvider, + } + + otelTracer otelTracing +) + +// DefaultTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultTracerProvider = "noop" + +// newOtel creates a structure with components that apps can use to initialize OpenTelemetry +// tracing instrumentation code. +func newOtel(conf *configuration.Config) (*otelTracing, error) { + if IsNoOpTracing(conf) { + log.Debug("open telemetry tracing disabled (noop)") + } else { + log.Debug("opentelemetry tracing enabled") + } + + otelTracer.appName = conf.GetString("webconfig.app_name") + otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") + tracerProvider, err := newTracerProvider(conf) + if err != nil { + return &otelTracer, err + } + otelTracer.tracerProvider = tracerProvider + otel.SetTracerProvider(tracerProvider) + + // Set up propagator. + prop := newPropagator() + otelTracer.propagator = prop + otel.SetTextMapPropagator(prop) + + otelTracer.tracer = otel.Tracer(otelTracer.appName) + return &otelTracer, nil +} + +// IsNoOpTracing returns true if the provider is set to "noop" +func IsNoOpTracing(conf *configuration.Config) bool { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + return strings.EqualFold(providerName, "noop") +} + +// TracerProvider returns the tracer provider component. By default, the noop +// tracer provider is returned. +func (t otelTracing) TracerProvider() trace.TracerProvider { + if t.tracerProvider == nil { + return noop.NewTracerProvider() + } + return t.tracerProvider +} + +// Propagator returns the component that helps propagate trace context across +// API boundaries. By default, a W3C Trace Context format propagator is returned. +func (t otelTracing) Propagator() propagation.TextMapPropagator { + if t.propagator == nil { + return propagation.TraceContext{} + } + return t.propagator +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +// newTracerProvider creates the TracerProvider based on config setting +// If no config setting, a noop tracerProvider will be returned. +func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + if len(providerName) == 0 { + providerName = defaultTracerProvider + } + // Handling camelcase of provider. + providerName = strings.ToLower(providerName) + providerConfig := providersConfig[providerName] + if providerConfig == nil { + return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) + } + + traceProvider, err := providerConfig(conf) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + return traceProvider, nil +} + +func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), + ) + return tp, nil +} + +func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + // Send traces over HTTP + endpoint := conf.GetString("webconfig.opentelemetry.endpoint") + if endpoint == "" { + return nil, ErrTracerProviderBuildFailed + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), + ), nil +} + +func (s *WebconfigServer) OtelShutdown() { + sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} diff --git a/http/poke_handler.go b/http/poke_handler.go index 62c4092..ac50ba5 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -161,11 +161,9 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - status := http.StatusInternalServerError + status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 - } else if rherr.StatusCode > http.StatusInternalServerError { - status = rherr.StatusCode } // parse the core message diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 29c3364..ba63157 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -35,7 +35,8 @@ import ( ) var ( - mockedWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) ) func TestPokeHandler(t *testing.T) { @@ -47,7 +48,7 @@ func TestPokeHandler(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -286,7 +287,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -305,3 +306,32 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa403(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write(mockWebpaPoke403Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go new file mode 100644 index 0000000..6413ee4 --- /dev/null +++ b/http/refsubdocument_handler.go @@ -0,0 +1,125 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "errors" + "net/http" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" +) + +func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + refsubdoc, err := s.GetRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + LogError(w, err) + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + + w.Header().Set("Content-Type", "application/msgpack") + if refsubdoc.Version() != nil { + w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) + } + w.WriteHeader(http.StatusOK) + w.Write(refsubdoc.Payload()) +} + +func (s *WebconfigServer) PostRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, bbytes, _, err := s.ValidateRefData(w, r, true) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + // handle version header + version := r.Header.Get(common.HeaderSubdocumentVersion) + if len(version) == 0 { + version = util.GetMurmur3Hash(bbytes) + } + + refsubdoc := common.NewRefSubDocument(bbytes, &version) + + err = s.SetRefSubDocument(refId, refsubdoc) + if err != nil { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + + WriteOkResponse(w, nil) +} + +func (s *WebconfigServer) DeleteRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + err = s.DeleteRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + WriteOkResponse(w, nil) +} diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go new file mode 100644 index 0000000..49171b0 --- /dev/null +++ b/http/refsubdocument_handler_test.go @@ -0,0 +1,76 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "testing" + + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + refId := uuid.New().String() + bbytes := util.RandomBytes(100, 150) + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes) + + // delete + req, err = http.NewRequest("DELETE", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get but expect 404 + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} diff --git a/http/router.go b/http/router.go index 7645c51..c6ddaf1 100644 --- a/http/router.go +++ b/http/router.go @@ -91,6 +91,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() + sub2.Use(spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -127,5 +128,19 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { } sub4.HandleFunc("", s.DeleteDocumentHandler).Methods("DELETE") + sub5 := router.Path("/api/v1/reference/{ref}/document").Subrouter() + if testOnly { + sub5.Use(s.TestingMiddleware) + } else { + if s.ServerApiTokenAuthEnabled() { + sub5.Use(s.ApiMiddleware) + } else { + sub5.Use(s.NoAuthMiddleware) + } + } + sub5.HandleFunc("", s.GetRefSubDocumentHandler).Methods("GET") + sub5.HandleFunc("", s.PostRefSubDocumentHandler).Methods("POST") + sub5.HandleFunc("", s.DeleteRefSubDocumentHandler).Methods("DELETE") + return router } diff --git a/http/validator.go b/http/validator.go index 10683ec..d85921e 100644 --- a/http/validator.go +++ b/http/validator.go @@ -71,3 +71,40 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid } return mac, subdocId, bodyBytes, fields, nil } + +func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request, validateContent bool) (string, []byte, log.Fields, error) { + var fields log.Fields + + // check mac + params := mux.Vars(r) + refId := params["ref"] + + // check for safety, but it should not fail + xw, ok := w.(*XResponseWriter) + if !ok { + err := *common.NewHttp500Error("responsewriter cast error") + return refId, nil, nil, common.NewError(err) + } + fields = xw.Audit() + + if !validateContent { + return refId, nil, fields, nil + } + + // ==== validate content ==== + // check content-type + contentType := r.Header.Get("Content-type") + if contentType != "application/msgpack" { + // TODO (1) if we should validate this header + // (2) if unexpected, return 400 or 415 + err := *common.NewHttp400Error("content-type not msgpack") + return refId, nil, nil, common.NewError(err) + } + + bodyBytes := xw.BodyBytes() + if len(bodyBytes) == 0 { + err := *common.NewHttp400Error("empty body") + return refId, nil, nil, common.NewError(err) + } + return refId, bodyBytes, fields, nil +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0d82f31..3a6082b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -20,6 +20,7 @@ package http import ( "context" "crypto/tls" + "encoding/hex" "encoding/json" "fmt" "io" @@ -28,6 +29,9 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" @@ -73,6 +77,7 @@ var ( "privatessid", "homessid", } + ws *WebconfigServer ) type WebconfigServer struct { @@ -101,6 +106,7 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool + otelTracer *otelTracing // For OpenTelemetry Tracing } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -239,10 +245,15 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + otelTracer, err := newOtel(conf) + if err != nil { + // Just log err and continue + log.Error("Could not initialize open telemetry for tracing, but continuing") + } supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - return &WebconfigServer{ + ws = &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -271,8 +282,10 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, + otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } + return ws } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -626,12 +639,11 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] } - // extrac tracestate from the header + // extract tracestate from the header tracestate := r.Header.Get(common.HeaderTracestate) if len(tracestate) > 0 { outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) } - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { @@ -843,3 +855,61 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { } return itf, "" } + +func spanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + spanContext := trace.SpanContextFromContext(ctx) + remote := spanContext.IsRemote() + sc := trace.SpanContext{}.WithRemote(remote) + + traceIDStr, traceFlagsStr := parseTraceparent(r) + if traceIDStr != "" { + traceID, _ := trace.TraceIDFromHex(traceIDStr) + sc = sc.WithTraceID(traceID) + } + if traceFlagsStr != "" { + traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) + sc = sc.WithTraceFlags(traceFlags) + } + tracestateStr := getTracestate(r) + if tracestateStr != "" { + tracestate, _ := trace.ParseTraceState(tracestateStr) + sc = sc.WithTraceState(tracestate) + } + ctx = trace.ContextWithSpanContext(ctx, sc) + ctx, span := otelTracer.tracer.Start(ctx, "oswebconfig_poke_handler") + defer span.End() + + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) + + // Pass the context with the span to the next handler + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// extract traceparent from the header +func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { + inTraceparent := r.Header.Get(common.HeaderTraceparent) + if len(inTraceparent) == 55 { + traceID = inTraceparent[3:35] + traceFlags = inTraceparent[53:55] + } + return +} + +// extract tracestate from the header +func getTracestate(r *http.Request) string { + inTracestate := r.Header.Get(common.HeaderTracestate) + var outTracestate string + if len(inTracestate) > 0 { + outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, ws.TracestateVendorID(), ws.TraceparentParentID()) + } + return outTracestate +} + +func hexStringToBytes(hexString string) []byte { + bytes, _ := hex.DecodeString(hexString) + return bytes +} diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 021f049..d7681f4 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -251,18 +251,26 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field } func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - var cont bool - + tfields := common.FilterLogFields(fields) + tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) copy(cbytes, bbytes) if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont = c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { + msg := fmt.Sprintf("finished success after 1 retry") + if i > 1 { + fmt.Sprintf("finished success after %v retries", i) + } + log.WithFields(tfields).Info(msg) break } + if i == c.retries { + log.WithFields(tfields).Infof("finished failure after %v retries", i) + } } <-c.queue } diff --git a/main.go b/main.go index 11c8893..d5684ef 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 468a7725a00489007dcb77ba532fcb3b10ac3b0e Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Mon, 17 Jun 2024 14:32:49 -0700 Subject: [PATCH 037/215] Feedback: Use the API instead of "oswebconfig_poke_handler" as the span name --- http/webconfig_server.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index e9800bb..7a5bf7b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -861,7 +861,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { remote := spanContext.IsRemote() sc := trace.SpanContext{}.WithRemote(remote) - traceIDStr, traceFlagsStr := parseTraceparent(r) + traceIDStr, traceFlagsStr := s.parseTraceparent(r) if traceIDStr != "" { traceID, _ := trace.TraceIDFromHex(traceIDStr) sc = sc.WithTraceID(traceID) @@ -876,7 +876,14 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { sc = sc.WithTraceState(tracestate) } ctx = trace.ContextWithSpanContext(ctx, sc) - ctx, span := otelTracer.tracer.Start(ctx, "oswebconfig_poke_handler") + + // Feedback: Better to use the "path"/API rather than a hard coded name + spanName := "oswebconfig_poke_handler" + pathTemplate, _ := mux.CurrentRoute(r).GetPathTemplate() + if pathTemplate != "" { + spanName = pathTemplate + } + ctx, span := otelTracer.tracer.Start(ctx, spanName) defer span.End() attr := attribute.String("env", otelTracer.envName) @@ -888,7 +895,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { } // extract traceparent from the header -func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { +func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) if len(inTraceparent) == 55 { traceID = inTraceparent[3:35] From d3c83391ad24a74d4e36e64e1c85985cf5471834 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Tue, 18 Jun 2024 10:26:17 -0700 Subject: [PATCH 038/215] Ticket to be created Fine grained control for generating traces for post/patch from oswebconfig No traces for GET calls from oswebconfig --- config/sample_webconfig.conf | 5 +++++ http/http_client.go | 13 ++++++++----- http/mqtt_connector.go | 3 ++- http/upstream_connector.go | 3 ++- http/webpa_connector.go | 5 +++-- http/xconf_connector.go | 3 ++- 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index fbfc38b..c485ae4 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -14,6 +14,11 @@ webconfig { // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" env_name = "dev" + // trace_post and trace_patch are flags that enable/disable instrumentation on child calls from oswebconfig + // trace_post is for requests to mqtt/upstream + trace_post = true + // trace_patch is for requests to webpa + trace_patch = true } // build info diff --git a/http/http_client.go b/http/http_client.go index 790d907..7bce843 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -63,7 +63,8 @@ type HttpClient struct { userAgent string } -func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config) *HttpClient { +// genTrace is a bool that indicates enabling otel or not +func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config, genTrace bool) *HttpClient { confKey := fmt.Sprintf("webconfig.%v.connect_timeout_in_secs", serviceName) connectTimeout := int(conf.GetInt32(confKey, defaultConnectTimeout)) @@ -95,10 +96,12 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: tlsConfig, } - transport = otelhttp.NewTransport(transport, - otelhttp.WithPropagators(otelTracer.propagator), - otelhttp.WithTracerProvider(otelTracer.tracerProvider), - ) + if genTrace { + transport = otelhttp.NewTransport(transport, + otelhttp.WithPropagators(otelTracer.propagator), + otelhttp.WithTracerProvider(otelTracer.tracerProvider), + ) + } return &HttpClient{ Client: &http.Client{ diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 07c086b..716aee6 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -45,8 +45,9 @@ func NewMqttConnector(conf *configuration.Config, tlsConfig *tls.Config) *MqttCo confKey := fmt.Sprintf("webconfig.%v.host", serviceName) host := conf.GetString(confKey, mqttHostDefault) + genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_post", false) return &MqttConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig, genTrace), host: host, serviceName: serviceName, } diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 165aac6..4a460bc 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -47,8 +47,9 @@ func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *Up confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) + genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_post", false) return &UpstreamConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig, genTrace), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, diff --git a/http/webpa_connector.go b/http/webpa_connector.go index d7681f4..b5258eb 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -130,9 +130,10 @@ func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *Webpa confKey = fmt.Sprintf("webconfig.%v.retry_in_msecs", webpaServiceName) retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) - syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig) + genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_patch", false) + syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig, genTrace) syncClient.SetStatusHandler(520, syncHandle520) - asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig) + asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig, genTrace) asyncClient.SetStatusHandler(520, asyncHandle520) confKey = fmt.Sprintf("webconfig.%v.api_version", webpaServiceName) diff --git a/http/xconf_connector.go b/http/xconf_connector.go index d40550a..e6cb630 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -44,7 +44,8 @@ func NewXconfConnector(conf *configuration.Config, tlsConfig *tls.Config) *Xconf host := conf.GetString(confKey, xconfHostDefault) return &XconfConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig), + // last param indicates no traces to be generated + HttpClient: NewHttpClient(conf, serviceName, tlsConfig, false), host: host, serviceName: serviceName, } From ef2702a4534f56edfb3892e2e501f517d42f60d6 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 19 Jun 2024 15:57:17 -0700 Subject: [PATCH 039/215] Narrower Tracing changes: Add a child span only for webpa poke patch instead of instrumenting the http client --- config/sample_webconfig.conf | 5 ----- http/http_client.go | 10 +--------- http/mqtt_connector.go | 3 +-- http/poke_handler.go | 2 +- http/upstream_connector.go | 3 +-- http/webconfig_server.go | 26 ++++++++++++++++++++------ http/webpa_connector.go | 10 +++++++--- http/xconf_connector.go | 3 +-- 8 files changed, 32 insertions(+), 30 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index c485ae4..fbfc38b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -14,11 +14,6 @@ webconfig { // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" env_name = "dev" - // trace_post and trace_patch are flags that enable/disable instrumentation on child calls from oswebconfig - // trace_post is for requests to mqtt/upstream - trace_post = true - // trace_patch is for requests to webpa - trace_patch = true } // build info diff --git a/http/http_client.go b/http/http_client.go index 7bce843..e7e66bc 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -37,7 +37,6 @@ import ( "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) const ( @@ -63,8 +62,7 @@ type HttpClient struct { userAgent string } -// genTrace is a bool that indicates enabling otel or not -func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config, genTrace bool) *HttpClient { +func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config) *HttpClient { confKey := fmt.Sprintf("webconfig.%v.connect_timeout_in_secs", serviceName) connectTimeout := int(conf.GetInt32(confKey, defaultConnectTimeout)) @@ -96,12 +94,6 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: tlsConfig, } - if genTrace { - transport = otelhttp.NewTransport(transport, - otelhttp.WithPropagators(otelTracer.propagator), - otelhttp.WithTracerProvider(otelTracer.tracerProvider), - ) - } return &HttpClient{ Client: &http.Client{ diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 716aee6..07c086b 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -45,9 +45,8 @@ func NewMqttConnector(conf *configuration.Config, tlsConfig *tls.Config) *MqttCo confKey := fmt.Sprintf("webconfig.%v.host", serviceName) host := conf.GetString(confKey, mqttHostDefault) - genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_post", false) return &MqttConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig, genTrace), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, } diff --git a/http/poke_handler.go b/http/poke_handler.go index ac50ba5..2192780 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -156,7 +156,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(mac, token, pokeStr, fields) + transactionId, err := s.Poke(r.Context(), mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 4a460bc..165aac6 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -47,9 +47,8 @@ func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *Up confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) - genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_post", false) return &UpstreamConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig, genTrace), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 7a5bf7b..8a5fdc3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -106,6 +106,7 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing + webpaPokeSpanName string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -252,7 +253,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - return &WebconfigServer{ + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -284,6 +285,10 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } + // Init the child poke span name + ws.webpaPokeSpanName = ws.WebpaConnector.PokeSpanName() + + return ws } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -590,7 +595,10 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(ctx context.Context, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { + ctx, span := newSpan(ctx, c.webpaPokeSpanName) + defer span.End() + body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) if err != nil { @@ -883,12 +891,9 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { if pathTemplate != "" { spanName = pathTemplate } - ctx, span := otelTracer.tracer.Start(ctx, spanName) + ctx, span := newSpan(ctx, spanName) defer span.End() - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) - // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) }) @@ -914,6 +919,15 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } +func newSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, spanName) + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) + + return ctx, span +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index b5258eb..4af41b4 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -130,10 +130,9 @@ func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *Webpa confKey = fmt.Sprintf("webconfig.%v.retry_in_msecs", webpaServiceName) retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) - genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_patch", false) - syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig, genTrace) + syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig) syncClient.SetStatusHandler(520, syncHandle520) - asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig, genTrace) + asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig) asyncClient.SetStatusHandler(520, asyncHandle520) confKey = fmt.Sprintf("webconfig.%v.api_version", webpaServiceName) @@ -169,6 +168,11 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } +func (c *WebpaConnector) PokeSpanName() string { + // By convention, span name won't have the host, but only the base template + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion) +} + func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") diff --git a/http/xconf_connector.go b/http/xconf_connector.go index e6cb630..d40550a 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -44,8 +44,7 @@ func NewXconfConnector(conf *configuration.Config, tlsConfig *tls.Config) *Xconf host := conf.GetString(confKey, xconfHostDefault) return &XconfConnector{ - // last param indicates no traces to be generated - HttpClient: NewHttpClient(conf, serviceName, tlsConfig, false), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, } From e015833323b49eb3c101909ed1dab4f918dc360b Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 19 Jun 2024 21:39:51 -0700 Subject: [PATCH 040/215] span name err fixed and span name improved --- http/webpa_connector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 4af41b4..0f5a1f8 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -170,7 +170,7 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { func (c *WebpaConnector) PokeSpanName() string { // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion) + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") + " PATCH" } func (c *WebpaConnector) NewQueue(capacity int) error { From 9c1a0e0ddb26fae2178753025d542f52700bcc46 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 20 Jun 2024 16:37:40 -0700 Subject: [PATCH 041/215] otel: Add method and status code explicitly as attributese --- http/poke_handler.go | 14 +++++++++++++- http/router.go | 2 +- http/webconfig_server.go | 32 ++++++++++++++++++++++++-------- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 2192780..83cea43 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -156,10 +156,21 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(r.Context(), mac, token, pokeStr, fields) + statusCode := http.StatusOK + statusCodePtr := &statusCode + ctx, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + + // TODO: endSpan should reflect the real status of the webpa patch call + // not the transformed custom status e.g 404 from webpa patch gets converted + // to 521. Clumsy way of doing this... + defer endSpanWithStatusCode(span, statusCodePtr) + + transactionId, err := s.Poke(ctx, mac, token, pokeStr, fields) + if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { + statusCode = rherr.StatusCode // webpa error handling status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { @@ -186,6 +197,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } + statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index bc7b0d6..f3e523f 100644 --- a/http/router.go +++ b/http/router.go @@ -91,7 +91,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() - sub2.Use(s.spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -101,6 +100,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } + sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 8a5fdc3..0dce2c2 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -596,9 +596,6 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { } func (c *WebconfigServer) Poke(ctx context.Context, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { - ctx, span := newSpan(ctx, c.webpaPokeSpanName) - defer span.End() - body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) if err != nil { @@ -891,8 +888,8 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { if pathTemplate != "" { spanName = pathTemplate } - ctx, span := newSpan(ctx, spanName) - defer span.End() + ctx, span := newSpan(ctx, spanName, "POST") + defer endSpanWithResponseWriter(span, w) // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -919,15 +916,34 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { +func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { var span trace.Span ctx, span = otelTracer.tracer.Start(ctx, spanName) - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) + envAttr := attribute.String("env", otelTracer.envName) + span.SetAttributes(envAttr) + + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) return ctx, span } +func endSpanWithStatusCode(span trace.Span, statusCodePtr *int) { + if statusCodePtr != nil { + statusAttr := attribute.Int("http.status_code", *statusCodePtr) + span.SetAttributes(statusAttr) + } + span.End() +} + +func endSpanWithResponseWriter(span trace.Span, w http.ResponseWriter) { + if xw, ok := w.(*XResponseWriter); ok { + statusAttr := attribute.Int("http.status_code", xw.Status()) + span.SetAttributes(statusAttr) + } + span.End() +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes From c90b48d37f444c10e6bfa203f5f6d7d107653645 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 20 Jun 2024 19:26:31 -0700 Subject: [PATCH 042/215] Code review feedback --- http/poke_handler.go | 17 +++++++++++------ http/webconfig_server.go | 18 +++++------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 83cea43..356874f 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,10 +25,12 @@ import ( "sort" "strings" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -157,13 +159,16 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } statusCode := http.StatusOK - statusCodePtr := &statusCode ctx, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") - // TODO: endSpan should reflect the real status of the webpa patch call - // not the transformed custom status e.g 404 from webpa patch gets converted - // to 521. Clumsy way of doing this... - defer endSpanWithStatusCode(span, statusCodePtr) + // endSpan should reflect the real status of the webpa patch call + // not the transformed custom status + // e.g 404 from webpa patch is converted to 521, but we want to show 404 + defer func() { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + span.End() + }() transactionId, err := s.Poke(ctx, mac, token, pokeStr, fields) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0dce2c2..d354735 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,6 +29,9 @@ import ( "strings" "time" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -38,9 +41,6 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -889,7 +889,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { spanName = pathTemplate } ctx, span := newSpan(ctx, spanName, "POST") - defer endSpanWithResponseWriter(span, w) + defer endSpan(span, w) // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -928,15 +928,7 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte return ctx, span } -func endSpanWithStatusCode(span trace.Span, statusCodePtr *int) { - if statusCodePtr != nil { - statusAttr := attribute.Int("http.status_code", *statusCodePtr) - span.SetAttributes(statusAttr) - } - span.End() -} - -func endSpanWithResponseWriter(span trace.Span, w http.ResponseWriter) { +func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { statusAttr := attribute.Int("http.status_code", xw.Status()) span.SetAttributes(statusAttr) From b50ac1128155c7211acac4d25759aea08016bf1b Mon Sep 17 00:00:00 2001 From: RV Subramanyan Date: Fri, 21 Jun 2024 10:22:56 -0700 Subject: [PATCH 043/215] Revert "Otel: Explicity add method and status_code as attributes" --- http/poke_handler.go | 21 ++------------------- http/router.go | 2 +- http/webconfig_server.go | 30 +++++++++++------------------- 3 files changed, 14 insertions(+), 39 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 356874f..2192780 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,12 +25,10 @@ import ( "sort" "strings" - "github.com/gorilla/mux" - "go.opentelemetry.io/otel/attribute" - "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" + "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -158,24 +156,10 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - statusCode := http.StatusOK - ctx, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") - - // endSpan should reflect the real status of the webpa patch call - // not the transformed custom status - // e.g 404 from webpa patch is converted to 521, but we want to show 404 - defer func() { - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - span.End() - }() - - transactionId, err := s.Poke(ctx, mac, token, pokeStr, fields) - + transactionId, err := s.Poke(r.Context(), mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - statusCode = rherr.StatusCode // webpa error handling status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { @@ -202,7 +186,6 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } - statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index f3e523f..bc7b0d6 100644 --- a/http/router.go +++ b/http/router.go @@ -91,6 +91,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() + sub2.Use(s.spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -100,7 +101,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } - sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index d354735..8a5fdc3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,9 +29,6 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -41,6 +38,9 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -596,6 +596,9 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { } func (c *WebconfigServer) Poke(ctx context.Context, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { + ctx, span := newSpan(ctx, c.webpaPokeSpanName) + defer span.End() + body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) if err != nil { @@ -888,8 +891,8 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { if pathTemplate != "" { spanName = pathTemplate } - ctx, span := newSpan(ctx, spanName, "POST") - defer endSpan(span, w) + ctx, span := newSpan(ctx, spanName) + defer span.End() // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -916,26 +919,15 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { +func newSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { var span trace.Span ctx, span = otelTracer.tracer.Start(ctx, spanName) + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) - envAttr := attribute.String("env", otelTracer.envName) - span.SetAttributes(envAttr) - - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) return ctx, span } -func endSpan(span trace.Span, w http.ResponseWriter) { - if xw, ok := w.(*XResponseWriter); ok { - statusAttr := attribute.Int("http.status_code", xw.Status()) - span.SetAttributes(statusAttr) - } - span.End() -} - func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes From 61348f563d4a0f5b0e87c876d06f9372a3456c30 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Fri, 21 Jun 2024 11:41:26 -0700 Subject: [PATCH 044/215] otel cherrypick --- http/http_client.go | 12 ++------ http/otel.go | 10 +++---- http/poke_handler.go | 19 +++++++++++- http/router.go | 2 +- http/webconfig_server.go | 62 ++++++++++++++++++++++++++++++---------- http/webpa_connector.go | 5 ++++ 6 files changed, 79 insertions(+), 31 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index c6cc70b..e7e66bc 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -32,13 +32,11 @@ import ( "strings" "time" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" ) const ( @@ -96,15 +94,11 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: tlsConfig, } - transport = otelhttp.NewTransport(transport, - otelhttp.WithPropagators(otelTracer.propagator), - otelhttp.WithTracerProvider(otelTracer.tracerProvider), - ) return &HttpClient{ Client: &http.Client{ Transport: transport, - Timeout: time.Duration(readTimeout) * time.Second, + Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, retryInMsecs: retryInMsecs, diff --git a/http/otel.go b/http/otel.go index 1faa742..e0a806e 100644 --- a/http/otel.go +++ b/http/otel.go @@ -27,10 +27,10 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" @@ -45,7 +45,7 @@ type otelTracing struct { envName string appName string tracerProvider trace.TracerProvider - propagator propagation.TextMapPropagator + propagator propagation.TextMapPropagator tracer trace.Tracer } @@ -54,10 +54,10 @@ type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, var ( ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") - providersConfig = map[string]providerConstructor{ - "http": httpTraceProvider, + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, "stdout": stdoutTraceProvider, - "noop": noopTraceProvider, + "noop": noopTraceProvider, } otelTracer otelTracing diff --git a/http/poke_handler.go b/http/poke_handler.go index ac50ba5..436fb2d 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,10 +25,12 @@ import ( "sort" "strings" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -156,10 +158,24 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } + statusCode := http.StatusOK + _, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + + // endSpan should reflect the real status of the webpa patch call + // not the transformed custom status + // e.g 404 from webpa patch is converted to 521, but we want to show 404 + defer func() { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + span.End() + }() + transactionId, err := s.Poke(mac, token, pokeStr, fields) + if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { + statusCode = rherr.StatusCode // webpa error handling status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { @@ -186,6 +202,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } + statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index c6ddaf1..f3e523f 100644 --- a/http/router.go +++ b/http/router.go @@ -91,7 +91,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() - sub2.Use(spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -101,6 +100,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } + sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 3a6082b..717413a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,6 +29,9 @@ import ( "strings" "time" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -38,9 +41,6 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -77,7 +77,6 @@ var ( "privatessid", "homessid", } - ws *WebconfigServer ) type WebconfigServer struct { @@ -107,6 +106,7 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing + webpaPokeSpanName string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -253,7 +253,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - ws = &WebconfigServer{ + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -285,6 +285,9 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } + // Init the child poke span name + ws.webpaPokeSpanName = ws.WebpaConnector.PokeSpanName() + return ws } @@ -856,14 +859,14 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func spanMiddleware(next http.Handler) http.Handler { +func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() spanContext := trace.SpanContextFromContext(ctx) remote := spanContext.IsRemote() sc := trace.SpanContext{}.WithRemote(remote) - traceIDStr, traceFlagsStr := parseTraceparent(r) + traceIDStr, traceFlagsStr := s.parseTraceparent(r) if traceIDStr != "" { traceID, _ := trace.TraceIDFromHex(traceIDStr) sc = sc.WithTraceID(traceID) @@ -872,17 +875,21 @@ func spanMiddleware(next http.Handler) http.Handler { traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) sc = sc.WithTraceFlags(traceFlags) } - tracestateStr := getTracestate(r) + tracestateStr := s.getTracestate(r) if tracestateStr != "" { tracestate, _ := trace.ParseTraceState(tracestateStr) sc = sc.WithTraceState(tracestate) } ctx = trace.ContextWithSpanContext(ctx, sc) - ctx, span := otelTracer.tracer.Start(ctx, "oswebconfig_poke_handler") - defer span.End() - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) + // Feedback: Better to use the "path"/API rather than a hard coded name + spanName := "oswebconfig_poke_handler" + pathTemplate, _ := mux.CurrentRoute(r).GetPathTemplate() + if pathTemplate != "" { + spanName = pathTemplate + } + ctx, span := newSpan(ctx, spanName, "POST") + defer endSpan(span, w) // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -890,7 +897,7 @@ func spanMiddleware(next http.Handler) http.Handler { } // extract traceparent from the header -func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { +func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) if len(inTraceparent) == 55 { traceID = inTraceparent[3:35] @@ -900,15 +907,40 @@ func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { } // extract tracestate from the header -func getTracestate(r *http.Request) string { +func (s *WebconfigServer) getTracestate(r *http.Request) string { inTracestate := r.Header.Get(common.HeaderTracestate) var outTracestate string if len(inTracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, ws.TracestateVendorID(), ws.TraceparentParentID()) + outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, s.TracestateVendorID(), s.TraceparentParentID()) } return outTracestate } +func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, spanName + " " + method) + + envAttr := attribute.String("env", otelTracer.envName) + span.SetAttributes(envAttr) + + /* + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) + */ + return ctx, span +} + +func endSpan(span trace.Span, w http.ResponseWriter) { + if xw, ok := w.(*XResponseWriter); ok { + fmt.Println("RV DEBUG endSpan status", xw.Status()) + statusAttr := attribute.Int("http.status_code", xw.Status()) + span.SetAttributes(statusAttr) + } else { + fmt.Println("RV DEBUG endSpan no status") + } + span.End() +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index d7681f4..ca57dc1 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,6 +168,11 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } +func (c *WebpaConnector) PokeSpanName() string { + // By convention, span name won't have the host, but only the base template + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") +} + func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") From 4af72cf516539faa0c1bca355687dd35277144c6 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Fri, 21 Jun 2024 11:58:02 -0700 Subject: [PATCH 045/215] otel - http.method is part of span name instead of a separate attribute --- http/poke_handler.go | 21 +++++++++++++++++-- http/router.go | 2 +- http/webconfig_server.go | 44 ++++++++++++++++++++++++++++------------ http/webpa_connector.go | 2 +- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 2192780..546f6ba 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,10 +25,12 @@ import ( "sort" "strings" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -156,11 +158,25 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(r.Context(), mac, token, pokeStr, fields) + statusCode := http.StatusOK + _, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + + // endSpan should reflect the real status of the webpa patch call + // not the transformed custom status + // e.g 404 from webpa patch is converted to 521, but we want to show 404 + defer func() { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + span.End() + }() + + transactionId, err := s.Poke(mac, token, pokeStr, fields) + if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling + statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 @@ -186,6 +202,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } + statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index bc7b0d6..f3e523f 100644 --- a/http/router.go +++ b/http/router.go @@ -91,7 +91,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() - sub2.Use(s.spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -101,6 +100,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } + sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 8a5fdc3..b3a5d72 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,6 +29,9 @@ import ( "strings" "time" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -38,9 +41,6 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -595,10 +595,7 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(ctx context.Context, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { - ctx, span := newSpan(ctx, c.webpaPokeSpanName) - defer span.End() - +func (c *WebconfigServer) Poke(cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) if err != nil { @@ -891,8 +888,8 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { if pathTemplate != "" { spanName = pathTemplate } - ctx, span := newSpan(ctx, spanName) - defer span.End() + ctx, span := newSpan(ctx, spanName, "POST") + defer endSpan(span, w) // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -919,15 +916,36 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { +func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { var span trace.Span - ctx, span = otelTracer.tracer.Start(ctx, spanName) - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) + spanNameWithMethod := spanName + " " + method + ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) + + envAttr := attribute.String("env", otelTracer.envName) + span.SetAttributes(envAttr) + + resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) + span.SetAttributes(resourceNameAttr) + + /* + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) + */ return ctx, span } +func endSpan(span trace.Span, w http.ResponseWriter) { + if xw, ok := w.(*XResponseWriter); ok { + fmt.Println("RV DEBUG endSpan status", xw.Status()) + statusAttr := attribute.Int("http.status_code", xw.Status()) + span.SetAttributes(statusAttr) + } else { + fmt.Println("RV DEBUG endSpan no status") + } + span.End() +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 0f5a1f8..ca57dc1 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -170,7 +170,7 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { func (c *WebpaConnector) PokeSpanName() string { // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") + " PATCH" + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") } func (c *WebpaConnector) NewQueue(capacity int) error { From d075d58ec152d525f848e44b8979610219e74c94 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Fri, 21 Jun 2024 12:35:09 -0700 Subject: [PATCH 046/215] otel: order of method + path template switched in span name --- http/webconfig_server.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index b3a5d72..10fb9f8 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -918,7 +918,9 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { var span trace.Span - spanNameWithMethod := spanName + " " + method + + // Convention: method followed by path template + spanNameWithMethod := method + " " + spanName ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) envAttr := attribute.String("env", otelTracer.envName) @@ -937,11 +939,8 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { - fmt.Println("RV DEBUG endSpan status", xw.Status()) statusAttr := attribute.Int("http.status_code", xw.Status()) span.SetAttributes(statusAttr) - } else { - fmt.Println("RV DEBUG endSpan no status") } span.End() } From d75f359cefa4e73b54bcf1f52a62d130aef29ae1 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Fri, 21 Jun 2024 12:41:14 -0700 Subject: [PATCH 047/215] Cherrypick of latest otel changes --- http/webconfig_server.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 717413a..159c89f 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -918,7 +918,10 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { var span trace.Span - ctx, span = otelTracer.tracer.Start(ctx, spanName + " " + method) + + // Convention: method followed by path template + spanNameWithMethod := method + " " + spanName + ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) @@ -932,11 +935,8 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { - fmt.Println("RV DEBUG endSpan status", xw.Status()) statusAttr := attribute.Int("http.status_code", xw.Status()) span.SetAttributes(statusAttr) - } else { - fmt.Println("RV DEBUG endSpan no status") } span.End() } From 8f603a69585a4e206dbb32eeaae7612321b5ebcf Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 15 Jun 2024 12:00:33 -0700 Subject: [PATCH 048/215] cherrypick a fix for corrupted blob handling and resolve conflicts --- db/cassandra/document.go | 7 +- http/document_handler.go | 1 + http/multipart_test.go | 164 +++++++++++++++++++++++++++++ http/supplementary_handler_test.go | 3 - 4 files changed, 170 insertions(+), 5 deletions(-) diff --git a/db/cassandra/document.go b/db/cassandra/document.go index 2eaf58c..df580b8 100644 --- a/db/cassandra/document.go +++ b/db/cassandra/document.go @@ -29,7 +29,6 @@ import ( log "github.com/sirupsen/logrus" ) -// NOTE this func (c *CassandraClient) GetSubDocument(cpeMac string, groupId string) (*common.SubDocument, error) { var err error var payload []byte @@ -283,7 +282,11 @@ func (c *CassandraClient) GetDocument(cpeMac string, xargs ...interface{}) (fndo if c.IsEncryptedGroup(groupId) { payload, err = c.DecryptBytes(payload) if err != nil { - return nil, common.NewError(err) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "subdoc" + tfields["subdoc_id"] = groupId + log.WithFields(tfields).Warn(err) + continue } } diff --git a/http/document_handler.go b/http/document_handler.go index f41f7ff..3f3e834 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -301,6 +301,7 @@ func (s *WebconfigServer) DeleteDocumentHandler(w http.ResponseWriter, r *http.R if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } diff --git a/http/multipart_test.go b/http/multipart_test.go index 30919f8..2aed59a 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -28,6 +28,7 @@ import ( "testing" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "github.com/vmihailenco/msgpack/v4" @@ -1104,3 +1105,166 @@ func TestStateCorrectionEnabled(t *testing.T) { assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) } + +func TestCorruptedEncryptedDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + tdbclient, ok := server.DatabaseClient.(*cassandra.CassandraClient) + if !ok { + t.Skip("Only test in cassandra env") + } + + cpeMac := util.GenerateRandomCpeMac() + encSubdocIds := []string{} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds := tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, !tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 1 setup lan subdoc ==== + // post + subdocId := "lan" + lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + lanBytes := util.RandomBytes(100, 150) + req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + _ = rbytes + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", lanUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 2 setup wan subdoc ==== + // post + subdocId = "wan" + wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + wanBytes := util.RandomBytes(100, 150) + req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", wanUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 3 setup privatessid subdoc ==== + // post + subdocId = "privatessid" + privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + privatessidBytes := util.RandomBytes(100, 150) + req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", privatessidUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 read the document ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 3) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + mpart, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + + // ==== step 5 set privatessid as an encrypted subdoc ==== + encSubdocIds = []string{"privatessid"} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds = tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 6 read the document expect no error but 1 less subdoc ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + _, ok = mpartMap["privatessid"] + assert.Assert(t, !ok) +} diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index b7007e5..942b0f8 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -732,9 +732,6 @@ func TestSupplementaryAppendingFlag(t *testing.T) { // ==== step 3 verify the query params ==== assert.Assert(t, strings.Contains(ss, "&stormReadyWifi=true")) - // okok := false - // assert.Assert(t, okok) - // ==== step 4 set append flag false ==== appendEnabled = false server.SetSupplementaryAppendingEnabled(appendEnabled) From dac69fee6ca173ed7c7422e2091b3087f8aae0f7 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 22 Jun 2024 00:48:34 -0700 Subject: [PATCH 049/215] handle non-ascii characters in headers from devices --- common/req_header.go | 51 +++++++++++++ common/req_header_test.go | 70 ++++++++++++++++++ common/root_document.go | 14 ++-- common/root_document_test.go | 10 +-- db/service.go | 62 ++++++++++++---- go.mod | 59 ++++++++-------- go.sum | 114 +++++------------------------- http/poke_handler.go | 5 +- http/rootdocument_handler_test.go | 74 +++++++++++++++++++ http/webconfig_server.go | 10 +-- 10 files changed, 307 insertions(+), 162 deletions(-) create mode 100644 common/req_header.go create mode 100644 common/req_header_test.go diff --git a/common/req_header.go b/common/req_header.go new file mode 100644 index 0000000..1a21f23 --- /dev/null +++ b/common/req_header.go @@ -0,0 +1,51 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "net/http" +) + +type ReqHeader struct { + http.Header +} + +func NewReqHeader(header http.Header) *ReqHeader { + return &ReqHeader{ + Header: header, + } +} + +func (h *ReqHeader) Get(k string) (string, error) { + v := h.Header.Get(k) + if !IsPrintable([]byte(v)) { + return "", fmt.Errorf("header %v invalid value %v discarded", k, v) + } + return v, nil +} + +func IsPrintable(bbytes []byte) bool { + for _, char := range bbytes { + // Check if the rune is outside the printable ASCII character range. + if char < 32 || char > 126 { + return false + } + } + return true +} diff --git a/common/req_header_test.go b/common/req_header_test.go new file mode 100644 index 0000000..829720c --- /dev/null +++ b/common/req_header_test.go @@ -0,0 +1,70 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "net/http" + "testing" + + "gotest.tools/assert" +) + +func TestIsPrintable(t *testing.T) { + b1 := []byte("hello world") + assert.Assert(t, IsPrintable(b1)) + b2 := []byte{0x00, 0x00, 0x00, 0x01} + assert.Assert(t, !IsPrintable(b2)) + b3 := append(b1, b2...) + assert.Assert(t, !IsPrintable(b3)) + + b4 := []byte("CGM4140COM_6.8p8s1_PROD_sey") + b5 := []byte{0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8} + b6 := append(b4, b5...) + assert.Assert(t, !IsPrintable(b6)) +} + +func TestReqHeader(t *testing.T) { + s1 := "hello world" + s2 := string([]byte{0x00, 0x00, 0x00, 0x01}) + s3 := s1 + s2 + + header := make(http.Header) + k1 := "maroon" + header.Set(k1, "helloworld") + k2 := "auburn" + header.Set(k2, s2) + k3 := "amber" + header.Set(k3, s3) + reqHeader := NewReqHeader(header) + + v1, err := reqHeader.Get(k1) + assert.NilError(t, err) + assert.Equal(t, v1, "helloworld") + + v2, err := reqHeader.Get(k2) + assert.Assert(t, err != nil) + assert.Equal(t, v2, "") + + v3, err := reqHeader.Get(k3) + assert.Assert(t, err != nil) + assert.Equal(t, v3, "") + + v4, err := reqHeader.Get("viridian") + assert.NilError(t, err) + assert.Equal(t, v4, "") +} diff --git a/common/root_document.go b/common/root_document.go index dc21075..a3542fd 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -113,23 +113,23 @@ func (d *RootDocument) Compare(r *RootDocument) int { return RootDocumentEquals } -func (d *RootDocument) IsDifferent(r *RootDocument) bool { +func (d *RootDocument) Equals(r *RootDocument) bool { if d.Bitmap != r.Bitmap { - return true + return false } if d.FirmwareVersion != r.FirmwareVersion { - return true + return false } if d.ModelName != r.ModelName { - return true + return false } if d.PartnerId != r.PartnerId { - return true + return false } if d.SchemaVersion != r.SchemaVersion { - return true + return false } - return false + return true } // update in place diff --git a/common/root_document_test.go b/common/root_document_test.go index a3c2d88..c2ed5e2 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -80,7 +80,7 @@ func TestRootDocumentUpdate(t *testing.T) { assert.Assert(t, !strings.Contains(line, "map[")) } -func TestRootDocumentIsDifferent(t *testing.T) { +func TestRootDocumentEquals(t *testing.T) { bitmap := 123 version := "foo" schemaVersion := "33554433-1.3,33554434-1.3" @@ -89,11 +89,11 @@ func TestRootDocumentIsDifferent(t *testing.T) { firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") rootdoc2 := rootdoc1.Clone() - isDiff := rootdoc1.IsDifferent(rootdoc2) - assert.Assert(t, !isDiff) + ok := rootdoc1.Equals(rootdoc2) + assert.Assert(t, ok) firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "") - isDiff = rootdoc1.IsDifferent(rootdoc3) - assert.Assert(t, isDiff) + ok = rootdoc1.Equals(rootdoc3) + assert.Assert(t, !ok) } diff --git a/db/service.go b/db/service.go index 882d0c4..7f438cd 100644 --- a/db/service.go +++ b/db/service.go @@ -45,14 +45,23 @@ var ( // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "request" + + // XPC-21583 Validate all headers + rHeader := common.NewReqHeader(inHeader) // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error - supportedDocs := rHeader.Get(common.HeaderSupportedDocs) + supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) + if err != nil { + log.WithFields(tfields).Warn(err) + } + if len(supportedDocs) > 0 { bitmap, err = util.GetCpeBitmap(supportedDocs) if err != nil { @@ -60,26 +69,43 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field } } - schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) - modelName := rHeader.Get(common.HeaderModelName) + schemaVersion, err := rHeader.Get(common.HeaderSchemaVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } + schemaVersion = strings.ToLower(schemaVersion) + + modelName, err := rHeader.Get(common.HeaderModelName) + if err != nil { + log.WithFields(tfields).Warn(err) + } - partnerId := rHeader.Get(common.HeaderPartnerID) + partnerId, err := rHeader.Get(common.HeaderPartnerID) + if err != nil { + log.WithFields(tfields).Warn(err) + } if len(partnerId) == 0 { partnerId = fieldsDict.GetString("partner") } - firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + firmwareVersion, err := rHeader.Get(common.HeaderFirmwareVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") // ==== parse mac ==== - mac := rHeader.Get(common.HeaderDeviceId) + mac, err := rHeader.Get(common.HeaderDeviceId) + if err != nil { + log.WithFields(tfields).Warn(err) + } var document *common.Document // get version map - deviceVersionMap, versions, err := parseVersionMap(rHeader) + deviceVersionMap, versions, err := parseVersionMap(rHeader, tfields) if err != nil { var gvmErr common.GroupVersionMismatchError if errors.As(err, &gvmErr) { @@ -126,13 +152,17 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== var rootCmpEnum int // mget fakes no meta change so that meta are not updated - if rHeader.Get("User-Agent") == "mget" { + userAgent, err := rHeader.Get("User-Agent") + if err != nil { + log.WithFields(tfields).Warn(err) + } + if userAgent == "mget" { rootCmpEnum = common.RootDocumentVersionOnlyChanged } else { rootCmpEnum = cloudRootDocument.Compare(deviceRootDocument) } - if isDiff := cloudRootDocument.IsDifferent(deviceRootDocument); isDiff { + if isEqual := cloudRootDocument.Equals(deviceRootDocument); !isEqual { // need to update rootDoc meta // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud clonedRootDoc := deviceRootDocument.Clone() @@ -250,16 +280,22 @@ func GetSetColumnsStr(columns []string) string { } // deviceVersionMap := parseVersionMap(rHeader, d) -func parseVersionMap(rHeader http.Header) (map[string]string, []string, error) { +func parseVersionMap(rHeader *common.ReqHeader, tfields log.Fields) (map[string]string, []string, error) { deviceVersionMap := make(map[string]string) - queryStr := rHeader.Get(common.HeaderDocName) + queryStr, err := rHeader.Get(common.HeaderDocName) + if err != nil { + log.WithFields(tfields).Warn(err) + } subdocIds := strings.Split(queryStr, ",") if len(queryStr) == 0 { return deviceVersionMap, nil, nil } - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + ifNoneMatch, err := rHeader.Get(common.HeaderIfNoneMatch) + if err != nil { + log.WithFields(tfields).Warn(err) + } versions := strings.Split(ifNoneMatch, ",") if len(subdocIds) != len(versions) { diff --git a/go.mod b/go.mod index 188e4df..95108f9 100644 --- a/go.mod +++ b/go.mod @@ -3,51 +3,48 @@ module github.com/rdkcentral/webconfig go 1.21 require ( - github.com/IBM/sarama v1.43.2 + github.com/IBM/sarama v1.42.1 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 - github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.1 - github.com/mattn/go-sqlite3 v1.14.22 - github.com/prometheus/client_golang v1.19.1 - github.com/prometheus/client_model v0.6.1 - github.com/sirupsen/logrus v1.9.3 - github.com/twmb/murmur3 v1.1.8 + github.com/gorilla/mux v1.8.0 + github.com/mattn/go-sqlite3 v1.14.15 + github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_model v0.2.0 + github.com/sirupsen/logrus v1.9.0 + github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/trace v1.27.0 - go.uber.org/automaxprocs v1.5.3 - go.uber.org/ratelimit v0.3.1 - golang.org/x/sync v0.7.0 + go.uber.org/automaxprocs v1.5.1 + go.uber.org/ratelimit v0.2.0 + golang.org/x/sync v0.6.0 gotest.tools v2.2.0+incompatible ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect - github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.6.0 // indirect + github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -55,25 +52,25 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.54.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect google.golang.org/grpc v1.64.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 9e2917d..74c2010 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= -github.com/IBM/sarama v1.43.2 h1:HABeEqRUh32z8yzY2hGB/j8mHSzC/HA9zlEjqFNCzSw= -github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMdXFQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -46,8 +44,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= -github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -63,8 +59,6 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -75,8 +69,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30= -github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= @@ -85,8 +77,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= @@ -105,8 +95,6 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -115,8 +103,6 @@ github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJr github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -179,20 +165,14 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -225,8 +205,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -239,12 +217,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -254,8 +228,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -271,23 +243,17 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= -github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -295,20 +261,16 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -325,8 +287,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= -github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= @@ -343,50 +303,28 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 h1:0W5o9SzoR15ocYHEQfvfipzcNog1lBxOLfnex91Hk6s= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0/go.mod h1:zVZ8nz+VSggWmnh6tTsJqXQ7rU4xLwRtna1M4x5jq58= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= -go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= -go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= -go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= -go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -395,10 +333,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -465,10 +401,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -488,8 +422,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -532,10 +464,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -548,10 +478,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -653,16 +581,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 h1:QW9+G6Fir4VcRXVH8x3LilNAb6cxBGLa6+GM4hRwexE= -google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3/go.mod h1:kdrSS/OiLkPrNUpzD4aHgCq2rVuC/YRxok32HXZ4vRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -675,8 +597,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -691,10 +611,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/poke_handler.go b/http/poke_handler.go index 546f6ba..4258031 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,12 +25,11 @@ import ( "sort" "strings" - "github.com/gorilla/mux" - "go.opentelemetry.io/otel/attribute" - "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 7bb66c6..d329258 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -233,3 +233,77 @@ func TestPostRootDocumentHandler(t *testing.T) { assert.Equal(t, getResp.Data, *srcDoc1) } + +func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 GET /rootdocument and expect 404 ==== + rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) + req, err := http.NewRequest("GET", rootdocUrl, nil) + req.Header.Set("Content-Type", "application/json") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 2 GET /config device ==== + // boots up but with out data in db + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + + expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + assert.NilError(t, err) + expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") + assert.DeepEqual(t, rootdoc, expectedRootdoc) + + // ==== step 2 ==== + supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(util.RandomBytes(10, 15)) + firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(util.RandomBytes(10, 15)) + modelName2 := "CGM4331COM" + string(util.RandomBytes(10, 15)) + partner2 := "comcast" + string(util.RandomBytes(10, 15)) + schemaVersion2 := "33554433-1.3,33554434-1.3" + string(util.RandomBytes(10, 15)) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs2) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion2) + req.Header.Set(common.HeaderModelName, modelName2) + req.Header.Set(common.HeaderPartnerID, partner2) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc2, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rootdoc.Equals(rootdoc2)) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 10fb9f8..da028a5 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,9 +29,6 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -41,6 +38,9 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -930,8 +930,8 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte span.SetAttributes(resourceNameAttr) /* - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) */ return ctx, span From 1e089b6d3022fef2b7fb99d85185688ced951b37 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 23 Jun 2024 17:43:29 -0700 Subject: [PATCH 050/215] fix a bug that updated_time was changed after calling /upstream even though the subdoc is not changed --- db/service.go | 10 +- http/upstream_test.go | 311 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 4 deletions(-) diff --git a/db/service.go b/db/service.go index 7f438cd..6f25653 100644 --- a/db/service.go +++ b/db/service.go @@ -457,18 +457,20 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old } labels["client"] = "default" - updatedTime := int(time.Now().UnixNano() / 1000000) - newSubdoc.SetUpdatedTime(&updatedTime) - newState := common.InDeployment if oldVersion, ok := versionMap[subdocId]; ok { if newSubdoc.Version() != nil { if oldVersion == *newSubdoc.Version() { - newState = common.Deployed + return nil } } } + updatedTime := int(time.Now().UnixNano() / 1000000) + newSubdoc.SetUpdatedTime(&updatedTime) + + newState := common.InDeployment newSubdoc.SetState(&newState) + err = c.SetSubDocument(cpeMac, subdocId, newSubdoc, oldState, labels, fields) if err != nil { return common.NewError(err) diff --git a/http/upstream_test.go b/http/upstream_test.go index f587467..b672d44 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -545,3 +545,314 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { assert.Equal(t, state, common.Deployed) } } + +func TestUpstreamUpdatedTime(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + privatessidV14Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + mparts, err := util.ParseMultipartAsList(r.Header, reqBytes) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // modify the payload + newMparts := []common.Multipart{} + for _, mpart := range mparts { + if mpart.Name == "privatessid" { + version := util.GetMurmur3Hash(privatessidV14Bytes) + newMpart := common.Multipart{ + Name: mpart.Name, + Version: version, + Bytes: privatessidV14Bytes, + } + newMparts = append(newMparts, newMpart) + } else { + newMparts = append(newMparts, mpart) + } + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + srcbytesMap["privatessid"] = privatessidV14Bytes + expectedStateMap := map[string]int{ + "privatessid": common.InDeployment, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + + switch subdocId { + case "privatessid": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) != subdocUpdatedTimeMap[subdocId]) + case "lan", "wan": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } + } +} From e29945fb9b2fe13190317b60e42276682daeefeb Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Tue, 25 Jun 2024 23:47:20 -0700 Subject: [PATCH 051/215] XPC-21774: Populate method and route attributes in otel --- http/webconfig_server.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index da028a5..41ba1f3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -929,10 +929,11 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) span.SetAttributes(resourceNameAttr) - /* - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) - */ + routeAttr := attribute.String("http.route", spanName) + span.SetAttributes(routeAttr) + + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) return ctx, span } From ffab4a12cb6d2da1fd6eea1262cc63ff612b9f8e Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 22 Jun 2024 00:48:34 -0700 Subject: [PATCH 052/215] cherrypick and resolve conflicts --- common/req_header.go | 51 +++++++++++++++++++++ common/req_header_test.go | 70 +++++++++++++++++++++++++++++ common/root_document.go | 14 +++--- common/root_document_test.go | 10 ++--- db/service.go | 62 ++++++++++++++++++++------ go.mod | 2 - go.sum | 4 -- http/poke_handler.go | 5 +-- http/rootdocument_handler_test.go | 74 +++++++++++++++++++++++++++++++ http/webconfig_server.go | 10 ++--- 10 files changed, 263 insertions(+), 39 deletions(-) create mode 100644 common/req_header.go create mode 100644 common/req_header_test.go diff --git a/common/req_header.go b/common/req_header.go new file mode 100644 index 0000000..1a21f23 --- /dev/null +++ b/common/req_header.go @@ -0,0 +1,51 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "net/http" +) + +type ReqHeader struct { + http.Header +} + +func NewReqHeader(header http.Header) *ReqHeader { + return &ReqHeader{ + Header: header, + } +} + +func (h *ReqHeader) Get(k string) (string, error) { + v := h.Header.Get(k) + if !IsPrintable([]byte(v)) { + return "", fmt.Errorf("header %v invalid value %v discarded", k, v) + } + return v, nil +} + +func IsPrintable(bbytes []byte) bool { + for _, char := range bbytes { + // Check if the rune is outside the printable ASCII character range. + if char < 32 || char > 126 { + return false + } + } + return true +} diff --git a/common/req_header_test.go b/common/req_header_test.go new file mode 100644 index 0000000..829720c --- /dev/null +++ b/common/req_header_test.go @@ -0,0 +1,70 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "net/http" + "testing" + + "gotest.tools/assert" +) + +func TestIsPrintable(t *testing.T) { + b1 := []byte("hello world") + assert.Assert(t, IsPrintable(b1)) + b2 := []byte{0x00, 0x00, 0x00, 0x01} + assert.Assert(t, !IsPrintable(b2)) + b3 := append(b1, b2...) + assert.Assert(t, !IsPrintable(b3)) + + b4 := []byte("CGM4140COM_6.8p8s1_PROD_sey") + b5 := []byte{0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8} + b6 := append(b4, b5...) + assert.Assert(t, !IsPrintable(b6)) +} + +func TestReqHeader(t *testing.T) { + s1 := "hello world" + s2 := string([]byte{0x00, 0x00, 0x00, 0x01}) + s3 := s1 + s2 + + header := make(http.Header) + k1 := "maroon" + header.Set(k1, "helloworld") + k2 := "auburn" + header.Set(k2, s2) + k3 := "amber" + header.Set(k3, s3) + reqHeader := NewReqHeader(header) + + v1, err := reqHeader.Get(k1) + assert.NilError(t, err) + assert.Equal(t, v1, "helloworld") + + v2, err := reqHeader.Get(k2) + assert.Assert(t, err != nil) + assert.Equal(t, v2, "") + + v3, err := reqHeader.Get(k3) + assert.Assert(t, err != nil) + assert.Equal(t, v3, "") + + v4, err := reqHeader.Get("viridian") + assert.NilError(t, err) + assert.Equal(t, v4, "") +} diff --git a/common/root_document.go b/common/root_document.go index dc21075..a3542fd 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -113,23 +113,23 @@ func (d *RootDocument) Compare(r *RootDocument) int { return RootDocumentEquals } -func (d *RootDocument) IsDifferent(r *RootDocument) bool { +func (d *RootDocument) Equals(r *RootDocument) bool { if d.Bitmap != r.Bitmap { - return true + return false } if d.FirmwareVersion != r.FirmwareVersion { - return true + return false } if d.ModelName != r.ModelName { - return true + return false } if d.PartnerId != r.PartnerId { - return true + return false } if d.SchemaVersion != r.SchemaVersion { - return true + return false } - return false + return true } // update in place diff --git a/common/root_document_test.go b/common/root_document_test.go index a3c2d88..c2ed5e2 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -80,7 +80,7 @@ func TestRootDocumentUpdate(t *testing.T) { assert.Assert(t, !strings.Contains(line, "map[")) } -func TestRootDocumentIsDifferent(t *testing.T) { +func TestRootDocumentEquals(t *testing.T) { bitmap := 123 version := "foo" schemaVersion := "33554433-1.3,33554434-1.3" @@ -89,11 +89,11 @@ func TestRootDocumentIsDifferent(t *testing.T) { firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") rootdoc2 := rootdoc1.Clone() - isDiff := rootdoc1.IsDifferent(rootdoc2) - assert.Assert(t, !isDiff) + ok := rootdoc1.Equals(rootdoc2) + assert.Assert(t, ok) firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "") - isDiff = rootdoc1.IsDifferent(rootdoc3) - assert.Assert(t, isDiff) + ok = rootdoc1.Equals(rootdoc3) + assert.Assert(t, !ok) } diff --git a/db/service.go b/db/service.go index 882d0c4..7f438cd 100644 --- a/db/service.go +++ b/db/service.go @@ -45,14 +45,23 @@ var ( // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "request" + + // XPC-21583 Validate all headers + rHeader := common.NewReqHeader(inHeader) // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error - supportedDocs := rHeader.Get(common.HeaderSupportedDocs) + supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) + if err != nil { + log.WithFields(tfields).Warn(err) + } + if len(supportedDocs) > 0 { bitmap, err = util.GetCpeBitmap(supportedDocs) if err != nil { @@ -60,26 +69,43 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field } } - schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) - modelName := rHeader.Get(common.HeaderModelName) + schemaVersion, err := rHeader.Get(common.HeaderSchemaVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } + schemaVersion = strings.ToLower(schemaVersion) + + modelName, err := rHeader.Get(common.HeaderModelName) + if err != nil { + log.WithFields(tfields).Warn(err) + } - partnerId := rHeader.Get(common.HeaderPartnerID) + partnerId, err := rHeader.Get(common.HeaderPartnerID) + if err != nil { + log.WithFields(tfields).Warn(err) + } if len(partnerId) == 0 { partnerId = fieldsDict.GetString("partner") } - firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + firmwareVersion, err := rHeader.Get(common.HeaderFirmwareVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") // ==== parse mac ==== - mac := rHeader.Get(common.HeaderDeviceId) + mac, err := rHeader.Get(common.HeaderDeviceId) + if err != nil { + log.WithFields(tfields).Warn(err) + } var document *common.Document // get version map - deviceVersionMap, versions, err := parseVersionMap(rHeader) + deviceVersionMap, versions, err := parseVersionMap(rHeader, tfields) if err != nil { var gvmErr common.GroupVersionMismatchError if errors.As(err, &gvmErr) { @@ -126,13 +152,17 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== var rootCmpEnum int // mget fakes no meta change so that meta are not updated - if rHeader.Get("User-Agent") == "mget" { + userAgent, err := rHeader.Get("User-Agent") + if err != nil { + log.WithFields(tfields).Warn(err) + } + if userAgent == "mget" { rootCmpEnum = common.RootDocumentVersionOnlyChanged } else { rootCmpEnum = cloudRootDocument.Compare(deviceRootDocument) } - if isDiff := cloudRootDocument.IsDifferent(deviceRootDocument); isDiff { + if isEqual := cloudRootDocument.Equals(deviceRootDocument); !isEqual { // need to update rootDoc meta // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud clonedRootDoc := deviceRootDocument.Clone() @@ -250,16 +280,22 @@ func GetSetColumnsStr(columns []string) string { } // deviceVersionMap := parseVersionMap(rHeader, d) -func parseVersionMap(rHeader http.Header) (map[string]string, []string, error) { +func parseVersionMap(rHeader *common.ReqHeader, tfields log.Fields) (map[string]string, []string, error) { deviceVersionMap := make(map[string]string) - queryStr := rHeader.Get(common.HeaderDocName) + queryStr, err := rHeader.Get(common.HeaderDocName) + if err != nil { + log.WithFields(tfields).Warn(err) + } subdocIds := strings.Split(queryStr, ",") if len(queryStr) == 0 { return deviceVersionMap, nil, nil } - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + ifNoneMatch, err := rHeader.Get(common.HeaderIfNoneMatch) + if err != nil { + log.WithFields(tfields).Warn(err) + } versions := strings.Split(ifNoneMatch, ",") if len(subdocIds) != len(versions) { diff --git a/go.mod b/go.mod index 2c15107..3b3e677 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 @@ -38,7 +37,6 @@ require ( github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/go.sum b/go.sum index b6af8c0..7cda15d 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= @@ -305,8 +303,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= diff --git a/http/poke_handler.go b/http/poke_handler.go index 436fb2d..bdccecd 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,12 +25,11 @@ import ( "sort" "strings" - "github.com/gorilla/mux" - "go.opentelemetry.io/otel/attribute" - "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 7bb66c6..d329258 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -233,3 +233,77 @@ func TestPostRootDocumentHandler(t *testing.T) { assert.Equal(t, getResp.Data, *srcDoc1) } + +func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 GET /rootdocument and expect 404 ==== + rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) + req, err := http.NewRequest("GET", rootdocUrl, nil) + req.Header.Set("Content-Type", "application/json") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 2 GET /config device ==== + // boots up but with out data in db + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + + expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + assert.NilError(t, err) + expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") + assert.DeepEqual(t, rootdoc, expectedRootdoc) + + // ==== step 2 ==== + supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(util.RandomBytes(10, 15)) + firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(util.RandomBytes(10, 15)) + modelName2 := "CGM4331COM" + string(util.RandomBytes(10, 15)) + partner2 := "comcast" + string(util.RandomBytes(10, 15)) + schemaVersion2 := "33554433-1.3,33554434-1.3" + string(util.RandomBytes(10, 15)) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs2) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion2) + req.Header.Set(common.HeaderModelName, modelName2) + req.Header.Set(common.HeaderPartnerID, partner2) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc2, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rootdoc.Equals(rootdoc2)) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 159c89f..abd66ee 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,9 +29,6 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -41,6 +38,9 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -927,8 +927,8 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte span.SetAttributes(envAttr) /* - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) */ return ctx, span } From 0a778c714a116d1bf7b6df9a145d658d312905ee Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 23 Jun 2024 17:43:29 -0700 Subject: [PATCH 053/215] fix a bug that updated_time was changed after calling /upstream even though the subdoc is not changed --- db/service.go | 10 +- http/upstream_test.go | 311 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 4 deletions(-) diff --git a/db/service.go b/db/service.go index 7f438cd..6f25653 100644 --- a/db/service.go +++ b/db/service.go @@ -457,18 +457,20 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old } labels["client"] = "default" - updatedTime := int(time.Now().UnixNano() / 1000000) - newSubdoc.SetUpdatedTime(&updatedTime) - newState := common.InDeployment if oldVersion, ok := versionMap[subdocId]; ok { if newSubdoc.Version() != nil { if oldVersion == *newSubdoc.Version() { - newState = common.Deployed + return nil } } } + updatedTime := int(time.Now().UnixNano() / 1000000) + newSubdoc.SetUpdatedTime(&updatedTime) + + newState := common.InDeployment newSubdoc.SetState(&newState) + err = c.SetSubDocument(cpeMac, subdocId, newSubdoc, oldState, labels, fields) if err != nil { return common.NewError(err) diff --git a/http/upstream_test.go b/http/upstream_test.go index f587467..b672d44 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -545,3 +545,314 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { assert.Equal(t, state, common.Deployed) } } + +func TestUpstreamUpdatedTime(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + privatessidV14Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + mparts, err := util.ParseMultipartAsList(r.Header, reqBytes) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // modify the payload + newMparts := []common.Multipart{} + for _, mpart := range mparts { + if mpart.Name == "privatessid" { + version := util.GetMurmur3Hash(privatessidV14Bytes) + newMpart := common.Multipart{ + Name: mpart.Name, + Version: version, + Bytes: privatessidV14Bytes, + } + newMparts = append(newMparts, newMpart) + } else { + newMparts = append(newMparts, mpart) + } + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + srcbytesMap["privatessid"] = privatessidV14Bytes + expectedStateMap := map[string]int{ + "privatessid": common.InDeployment, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + + switch subdocId { + case "privatessid": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) != subdocUpdatedTimeMap[subdocId]) + case "lan", "wan": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } + } +} From cf18789ec01e48bc10496a9c815a36a3e78aa69c Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 26 Jun 2024 12:23:57 -0700 Subject: [PATCH 054/215] otel cherrypick for method name population --- http/webconfig_server.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index abd66ee..cf8b0bf 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -925,11 +925,13 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) + + resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) + span.SetAttributes(resourceNameAttr) + + routeAttr := attribute.String("http.route", spanName) + span.SetAttributes(routeAttr) - /* - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) - */ return ctx, span } From c2ead08af58c88e4b7e54601900a87beec942160 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 26 Jun 2024 14:17:36 -0700 Subject: [PATCH 055/215] Git merge err, some lines were missed --- http/webconfig_server.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index cf8b0bf..41ba1f3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -925,13 +925,16 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) - + resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) span.SetAttributes(resourceNameAttr) routeAttr := attribute.String("http.route", spanName) span.SetAttributes(routeAttr) + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) + return ctx, span } From 205d67ab98cea9486dd2eb22cfcaabb150d9fe72 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 27 Jun 2024 18:39:28 -0700 Subject: [PATCH 056/215] Skip updating subdocs if instructed by upstream response --- common/const_var.go | 5 + http/multipart.go | 22 +- http/upstream_test.go | 579 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 598 insertions(+), 8 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 2fcf2af..6d57e4a 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -107,6 +107,11 @@ const ( HeaderTracestate = "Tracestate" HeaderContentLength = "Content-Length" HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" + HeaderUpstreamResponse = "X-Upstream-Response" +) + +const ( + SkipDbUpdate = "skip-db-update" ) // header X-System-Supported-Docs diff --git a/http/multipart.go b/http/multipart.go index b2860af..c1f29d4 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -285,10 +285,13 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } } finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) @@ -406,10 +409,13 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l finalDocument.DeleteSubDocument(subdocId) } - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } } if finalDocument.Length() == 0 { diff --git a/http/upstream_test.go b/http/upstream_test.go index b672d44..9a7807b 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -856,3 +856,582 @@ func TestUpstreamUpdatedTime(t *testing.T) { } } } + +func TestUpstreamResponseSkipDbUpdate(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := util.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} + +func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := util.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} From 72353ceae3dc33ace936da9194dbf8b5b5369145 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 27 Jun 2024 18:39:28 -0700 Subject: [PATCH 057/215] Skip updating subdocs if instructed by upstream response --- common/const_var.go | 5 + http/multipart.go | 22 +- http/upstream_test.go | 579 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 598 insertions(+), 8 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 2fcf2af..6d57e4a 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -107,6 +107,11 @@ const ( HeaderTracestate = "Tracestate" HeaderContentLength = "Content-Length" HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" + HeaderUpstreamResponse = "X-Upstream-Response" +) + +const ( + SkipDbUpdate = "skip-db-update" ) // header X-System-Supported-Docs diff --git a/http/multipart.go b/http/multipart.go index b2860af..c1f29d4 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -285,10 +285,13 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } } finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) @@ -406,10 +409,13 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l finalDocument.DeleteSubDocument(subdocId) } - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } } if finalDocument.Length() == 0 { diff --git a/http/upstream_test.go b/http/upstream_test.go index b672d44..9a7807b 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -856,3 +856,582 @@ func TestUpstreamUpdatedTime(t *testing.T) { } } } + +func TestUpstreamResponseSkipDbUpdate(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := util.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} + +func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := util.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} From de3eee0c2058a44c3420ff0278af4633cdbc48e8 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Sat, 29 Jun 2024 10:07:32 -0700 Subject: [PATCH 058/215] XPC-22084: Add operation-name, http.url, http.url_details attributes in otel --- config/sample_webconfig.conf | 1 + http/otel.go | 9 ++++--- http/poke_handler.go | 5 +++- http/webconfig_server.go | 51 ++++++++++++++++++++++-------------- http/webpa_connector.go | 9 +++++-- 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index fbfc38b..0ac059b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -14,6 +14,7 @@ webconfig { // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" env_name = "dev" + operation_name = "http.request" } // build info diff --git a/http/otel.go b/http/otel.go index e0a806e..0637e2f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -35,7 +35,6 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" ) @@ -44,6 +43,7 @@ type otelTracing struct { providerName string envName string appName string + opName string tracerProvider trace.TracerProvider propagator propagation.TextMapPropagator tracer trace.Tracer @@ -71,7 +71,7 @@ const defaultTracerProvider = "noop" // newOtel creates a structure with components that apps can use to initialize OpenTelemetry // tracing instrumentation code. func newOtel(conf *configuration.Config) (*otelTracing, error) { - if IsNoOpTracing(conf) { + if IsNoopTracing(conf) { log.Debug("open telemetry tracing disabled (noop)") } else { log.Debug("opentelemetry tracing enabled") @@ -80,6 +80,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { otelTracer.appName = conf.GetString("webconfig.app_name") otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") + otelTracer.opName = conf.GetString("webconfig.opentelemetryl.operation_name", "http.request") tracerProvider, err := newTracerProvider(conf) if err != nil { return &otelTracer, err @@ -96,8 +97,8 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { return &otelTracer, nil } -// IsNoOpTracing returns true if the provider is set to "noop" -func IsNoOpTracing(conf *configuration.Config) bool { +// IsNoopTracing returns true if the provider is set to "noop" +func IsNoopTracing(conf *configuration.Config) bool { providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) return strings.EqualFold(providerName, "noop") } diff --git a/http/poke_handler.go b/http/poke_handler.go index 4258031..4552769 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -158,7 +158,10 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } statusCode := http.StatusOK - _, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) + // We are not passing any qparams to webpa, so fullPath = path + fullPath := pokeSpanPath + _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") // endSpan should reflect the real status of the webpa patch call // not the transformed custom status diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 41ba1f3..a4b8ab3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -106,7 +106,7 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing - webpaPokeSpanName string + webpaPokeSpanTemplate string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -286,7 +286,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, } // Init the child poke span name - ws.webpaPokeSpanName = ws.WebpaConnector.PokeSpanName() + ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -882,13 +882,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { } ctx = trace.ContextWithSpanContext(ctx, sc) - // Feedback: Better to use the "path"/API rather than a hard coded name - spanName := "oswebconfig_poke_handler" - pathTemplate, _ := mux.CurrentRoute(r).GetPathTemplate() - if pathTemplate != "" { - spanName = pathTemplate - } - ctx, span := newSpan(ctx, spanName, "POST") + ctx, span := s.newParentPokeSpan(ctx, r) defer endSpan(span, w) // Pass the context with the span to the next handler @@ -916,26 +910,45 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { +func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { var span trace.Span - // Convention: method followed by path template - spanNameWithMethod := method + " " + spanName - ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) + + // Feedback: Better to use the "path"/API rather than a hard coded name + route := "oswebconfig_poke_handler" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + route, _ = mux.CurrentRoute(r).GetPathTemplate() + } + s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) + return ctx, span +} +func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) + + s.addAttributes(span, route, path, fullPath, method) + return ctx, span + } + +func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) - resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) - span.SetAttributes(resourceNameAttr) - - routeAttr := attribute.String("http.route", spanName) + routeAttr := attribute.String("http.route", route) span.SetAttributes(routeAttr) + pathAttr := attribute.String("http.url_details.path", path) + span.SetAttributes(pathAttr) + + fullPathAttr := attribute.String("http.url", fullPath) + span.SetAttributes(fullPathAttr) + methodAttr := attribute.String("http.method", method) span.SetAttributes(methodAttr) - - return ctx, span } func endSpan(span trace.Span, w http.ResponseWriter) { diff --git a/http/webpa_connector.go b/http/webpa_connector.go index ca57dc1..f6aadbe 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,9 +168,14 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } -func (c *WebpaConnector) PokeSpanName() string { +func (c *WebpaConnector) PokeSpanTemplate() string { // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") +} + +// Base URL with the cpemac populated +func (c *WebpaConnector) PokeSpanPath(mac string) string { + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) } func (c *WebpaConnector) NewQueue(capacity int) error { From df09776bc00ad6ca66259d6a55ff0eb1238908fa Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 29 Jun 2024 13:11:36 -0700 Subject: [PATCH 059/215] Add the capability to forward kafka messages --- common/log_fields.go | 16 ++++++++++ common/log_fields_test.go | 25 +++++++++++++++ http/otel.go | 3 +- http/webconfig_server.go | 65 ++++++++++++++++++++++++++++++++++++--- kafka/consumer.go | 4 +++ 5 files changed, 108 insertions(+), 5 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index 06289e6..8ec4f19 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -28,6 +28,12 @@ var ( "out_traceparent", "out_tracestate", } + coreFields = []string{ + "app_name", + "audit_id", + "body", + "cpe_mac", + } ) func FilterLogFields(src log.Fields, excludes ...string) log.Fields { @@ -53,3 +59,13 @@ func UpdateLogFields(fields, newfields log.Fields) { fields[k] = v } } + +func CopyCoreLogFields(src log.Fields) log.Fields { + fields := log.Fields{} + for _, k := range coreFields { + if itf, ok := src[k]; ok { + fields[k] = itf + } + } + return fields +} diff --git a/common/log_fields_test.go b/common/log_fields_test.go index e357855..4a95b9a 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -95,3 +95,28 @@ func TestUpdateLogFields(t *testing.T) { assert.DeepEqual(t, src, expected) } + +func TestCopyCoreLogFields(t *testing.T) { + body := map[string]interface{}{ + "device_id": "mac:29cf4fe3914e", + "http_status_code": 304, + "transaction_uuid": "f160f5f2-c899-4652-b066-c9b68328d74f", + "version": "1719689278", + } + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + expected := log.Fields{ + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + copied := CopyCoreLogFields(src) + assert.DeepEqual(t, copied, expected) +} diff --git a/http/otel.go b/http/otel.go index 0637e2f..69ca64f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -35,6 +35,7 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" ) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index a4b8ab3..1cfd799 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -32,6 +32,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" @@ -89,6 +90,7 @@ type WebconfigServer struct { *XconfConnector *MqttConnector *UpstreamConnector + sarama.AsyncProducer tlsConfig *tls.Config notLoggedHeaders []string metricsEnabled bool @@ -107,6 +109,8 @@ type WebconfigServer struct { supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing webpaPokeSpanTemplate string + kafkaProducerEnabled bool + kafkaProducerTopic string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -253,6 +257,25 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) + // kafka producer + var kafkaProducer sarama.AsyncProducer + kafkaProducerEnabled := conf.GetBoolean("webconfig.kafka_producer.enabled") + var kafkaProducerTopic string + if kafkaProducerEnabled { + brokersStr := conf.GetString("webconfig.kafka_producer.brokers") + if len(brokersStr) == 0 { + panic(fmt.Errorf("webconfig.kafka_producer.brokers is empty")) + } + brokers := strings.Split(brokersStr, ",") + kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") + + saramaConfig := sarama.NewConfig() + kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) + if err != nil { + panic(err) + } + } + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -267,6 +290,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer XconfConnector: NewXconfConnector(conf, tlsConfig), MqttConnector: NewMqttConnector(conf, tlsConfig), UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), + AsyncProducer: kafkaProducer, tlsConfig: tlsConfig, notLoggedHeaders: notLoggedHeaders, metricsEnabled: metricsEnabled, @@ -284,6 +308,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer tracestateVendorID: tracestateVendorID, otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, + kafkaProducerEnabled: kafkaProducerEnabled, + kafkaProducerTopic: kafkaProducerTopic, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -580,6 +606,22 @@ func (s *WebconfigServer) SetSupplementaryAppendingEnabled(enabled bool) { s.supplementaryAppendingEnabled = enabled } +func (s *WebconfigServer) KafkaProducerEnabled() bool { + return s.kafkaProducerEnabled +} + +func (s *WebconfigServer) SetKafkaProducerEnabled(enabled bool) { + s.kafkaProducerEnabled = enabled +} + +func (s *WebconfigServer) KafkaProducerTopic() string { + return s.kafkaProducerTopic +} + +func (s *WebconfigServer) SetKafkaProducerTopic(x string) { + s.kafkaProducerTopic = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { @@ -929,11 +971,11 @@ func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, pa var span trace.Span ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) - + s.addAttributes(span, route, path, fullPath, method) - return ctx, span - } - + return ctx, span +} + func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) @@ -963,3 +1005,18 @@ func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes } + +func (s *WebconfigServer) ForwardKafkaMessage(message *sarama.ConsumerMessage, fields log.Fields) { + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(message.Key), + Value: sarama.ByteEncoder(message.Value), + } + s.Input() <- outMessage + + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["kafka_topic"] = outMessage.Topic + tfields["kafka_key"] = string(message.Key) + log.WithFields(tfields).Info("send") +} diff --git a/kafka/consumer.go b/kafka/consumer.go index 1135d5d..83263ac 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -253,6 +253,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } metrics.CountKafkaEvents(eventName, status, message.Partition) } + + if c.KafkaProducerEnabled() { + c.ForwardKafkaMessage(message, fields) + } case <-session.Context().Done(): return nil } From 2a744ae380e46488df81efb266ff2fb7d99bf091 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Mon, 1 Jul 2024 09:09:01 -0700 Subject: [PATCH 060/215] otel Missed a line --- http/otel.go | 6 +++--- http/webconfig_server.go | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/http/otel.go b/http/otel.go index 69ca64f..1c56b63 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,8 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ + package http import ( @@ -35,7 +36,6 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" ) @@ -81,7 +81,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { otelTracer.appName = conf.GetString("webconfig.app_name") otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") - otelTracer.opName = conf.GetString("webconfig.opentelemetryl.operation_name", "http.request") + otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") tracerProvider, err := newTracerProvider(conf) if err != nil { return &otelTracer, err diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1cfd799..88d84b5 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -980,6 +980,9 @@ func (s *WebconfigServer) addAttributes(span trace.Span, route string, path stri envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) + operationNameAttr := attribute.String("operation.name", otelTracer.opName) + span.SetAttributes(operationNameAttr) + routeAttr := attribute.String("http.route", route) span.SetAttributes(routeAttr) From 7b1c5fe180aa98d2c9c8baa8817b21945b86ac0c Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 00:20:06 -0700 Subject: [PATCH 061/215] forward notifications on conditions --- config/sample_webconfig.conf | 7 ++++++ db/cassandra/state_update_test.go | 12 ++++++---- db/service.go | 30 ++++++++++++------------ http/otel.go | 1 - http/upstream_test.go | 15 ++++++++---- http/webconfig_server.go | 17 ++++++++++---- kafka/consumer.go | 38 +++++++++++++++++++++++-------- 7 files changed, 81 insertions(+), 39 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 0ac059b..60d9364 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -245,4 +245,11 @@ webconfig { // correct subdoc states if versions match but not "deployed" state_correction_enabled= false + + // forward kafka messages if needed + kafka_producer { + enabled = false + brokers = "localhost:9092" + topics = "webconfig_downstream" + } } diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 0857770..2d9bbb0 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -58,8 +58,9 @@ func TestStateUpdate1(t *testing.T) { var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -72,8 +73,9 @@ func TestStateUpdate1(t *testing.T) { m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -109,8 +111,9 @@ func TestStateUpdate2(t *testing.T) { var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -123,8 +126,9 @@ func TestStateUpdate2(t *testing.T) { m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, ok) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/service.go b/db/service.go index 6f25653..9493527 100644 --- a/db/service.go +++ b/db/service.go @@ -335,10 +335,11 @@ func HashRootVersion(itf interface{}) string { return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) error { +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) (bool, error) { + var updatedBy304 bool // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return nil + return updatedBy304, nil } updatedTime := int(time.Now().UnixNano() / 1000000) @@ -350,7 +351,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } labels["client"] = metricsAgent @@ -359,14 +360,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return nil + return updatedBy304, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } newState := common.Deployed @@ -375,20 +376,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { + updatedBy304 = true newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } } } - return nil + return updatedBy304, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return common.NewError(fmt.Errorf("ill-formatted event")) + return updatedBy304, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -401,13 +403,13 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return nil + return updatedBy304, nil } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } var oldState int @@ -417,7 +419,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return common.NewError(err) + return updatedBy304, common.NewError(err) } } @@ -427,7 +429,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return common.NewError(err) + return updatedBy304, common.NewError(err) } } @@ -440,9 +442,9 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } - return nil + return updatedBy304, nil } func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { diff --git a/http/otel.go b/http/otel.go index 1c56b63..468035f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -15,7 +15,6 @@ * * SPDX-License-Identifier: Apache-2.0 */ - package http import ( diff --git a/http/upstream_test.go b/http/upstream_test.go index 9a7807b..127dcc1 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -263,8 +263,9 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -525,8 +526,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -775,8 +777,9 @@ func TestUpstreamUpdatedTime(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -1074,8 +1077,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -1362,8 +1366,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 88d84b5..ae63adf 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1009,17 +1009,24 @@ func hexStringToBytes(hexString string) []byte { return bytes } -func (s *WebconfigServer) ForwardKafkaMessage(message *sarama.ConsumerMessage, fields log.Fields) { +func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } outMessage := &sarama.ProducerMessage{ Topic: s.KafkaProducerTopic(), - Key: sarama.ByteEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), + Key: sarama.ByteEncoder(kbytes), + Value: sarama.ByteEncoder(bbytes), } s.Input() <- outMessage - tfields := common.CopyCoreLogFields(fields) tfields["logger"] = "kafkaproducer" tfields["kafka_topic"] = outMessage.Topic - tfields["kafka_key"] = string(message.Key) + tfields["kafka_key"] = string(kbytes) log.WithFields(tfields).Info("send") } diff --git a/kafka/consumer.go b/kafka/consumer.go index 83263ac..52b22fb 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/http" "strings" "time" @@ -75,31 +76,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, bool, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, common.NewError(err) + return nil, false, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, common.NewError(err) + return nil, false, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, nil + return &m, false, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - err = db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedBy304, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, common.NewError(err) + return &m, updatedBy304, common.NewError(err) } - return &m, nil + return &m, updatedBy304, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -203,6 +204,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage + var updatedBy304 bool eventName, rptHeaderValue := getEventName(message) switch eventName { @@ -212,10 +214,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, err = c.handleNotification(bbytes, fields) + m, updatedBy304, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, err = c.handleNotification(message.Value, fields) + m, updatedBy304, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -255,7 +257,23 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() { - c.ForwardKafkaMessage(message, fields) + if len(m.Reports) == 0 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { + // build a root/success message + namespace := "root" + applicationStatus := "success" + em := &common.EventMessage{ + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) + } else if m.ApplicationStatus != nil && *m.ApplicationStatus == "success" && m.Namespace != nil && *m.Namespace != "telemetry" { + c.ForwardKafkaMessage(message.Key, m, fields) + } + } } case <-session.Context().Done(): return nil From 96ae446e7a7241c35067814bc9332b098e8c7070 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 00:26:55 -0700 Subject: [PATCH 062/215] fix a typo in sample config --- config/sample_webconfig.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 60d9364..4b67a55 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -250,6 +250,6 @@ webconfig { kafka_producer { enabled = false brokers = "localhost:9092" - topics = "webconfig_downstream" + topic = "webconfig_downstream" } } From 3fc8f95a0a7ebea63474f8fa6f51aa47542f2f63 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Tue, 2 Jul 2024 17:24:25 -0700 Subject: [PATCH 063/215] populate error graph in DD --- http/webconfig_server.go | 44 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index ae63adf..27da9c7 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -30,7 +30,9 @@ import ( "time" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" @@ -977,29 +979,35 @@ func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, pa } func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { - envAttr := attribute.String("env", otelTracer.envName) - span.SetAttributes(envAttr) - - operationNameAttr := attribute.String("operation.name", otelTracer.opName) - span.SetAttributes(operationNameAttr) - - routeAttr := attribute.String("http.route", route) - span.SetAttributes(routeAttr) - - pathAttr := attribute.String("http.url_details.path", path) - span.SetAttributes(pathAttr) - - fullPathAttr := attribute.String("http.url", fullPath) - span.SetAttributes(fullPathAttr) - - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) + span.SetAttributes( + attribute.String("env", otelTracer.envName), + attribute.String("operation.name", otelTracer.opName), + attribute.String("http.url_details.path", path), + semconv.HTTPMethodKey.String(method), + semconv.HTTPRouteKey.String(route), + semconv.HTTPURLKey.String(fullPath), + ) + + log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) + log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) + log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) + log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) + log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) + log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) } func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { - statusAttr := attribute.Int("http.status_code", xw.Status()) + statusCode := xw.Status() + statusAttr := attribute.Int("http.status_code", statusCode) span.SetAttributes(statusAttr) + log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) + } } span.End() } From 13fe5c5fc61193daa8af7ceb6cc341aa89a86fab Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 14:09:43 -0700 Subject: [PATCH 064/215] handle ill-formatted kafka messages in forwarding --- kafka/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 52b22fb..5d94e00 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -256,7 +256,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram metrics.CountKafkaEvents(eventName, status, message.Partition) } - if c.KafkaProducerEnabled() { + if c.KafkaProducerEnabled() && m != nil { if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { // build a root/success message From 84c399e27fe2d178811d6a87a3b1e873ca16630b Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 3 Jul 2024 16:18:46 -0700 Subject: [PATCH 065/215] otel cherrypick for populating err graphs in DD --- config/sample_webconfig.conf | 1 + http/otel.go | 9 ++--- http/poke_handler.go | 7 ++-- http/webconfig_server.go | 70 ++++++++++++++++++++++++------------ http/webpa_connector.go | 9 +++-- 5 files changed, 65 insertions(+), 31 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index fbfc38b..0ac059b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -14,6 +14,7 @@ webconfig { // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" env_name = "dev" + operation_name = "http.request" } // build info diff --git a/http/otel.go b/http/otel.go index e0a806e..468035f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -35,7 +35,6 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" ) @@ -44,6 +43,7 @@ type otelTracing struct { providerName string envName string appName string + opName string tracerProvider trace.TracerProvider propagator propagation.TextMapPropagator tracer trace.Tracer @@ -71,7 +71,7 @@ const defaultTracerProvider = "noop" // newOtel creates a structure with components that apps can use to initialize OpenTelemetry // tracing instrumentation code. func newOtel(conf *configuration.Config) (*otelTracing, error) { - if IsNoOpTracing(conf) { + if IsNoopTracing(conf) { log.Debug("open telemetry tracing disabled (noop)") } else { log.Debug("opentelemetry tracing enabled") @@ -80,6 +80,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { otelTracer.appName = conf.GetString("webconfig.app_name") otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") + otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") tracerProvider, err := newTracerProvider(conf) if err != nil { return &otelTracer, err @@ -96,8 +97,8 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { return &otelTracer, nil } -// IsNoOpTracing returns true if the provider is set to "noop" -func IsNoOpTracing(conf *configuration.Config) bool { +// IsNoopTracing returns true if the provider is set to "noop" +func IsNoopTracing(conf *configuration.Config) bool { providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) return strings.EqualFold(providerName, "noop") } diff --git a/http/poke_handler.go b/http/poke_handler.go index bdccecd..4552769 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -158,7 +158,10 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } statusCode := http.StatusOK - _, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) + // We are not passing any qparams to webpa, so fullPath = path + fullPath := pokeSpanPath + _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") // endSpan should reflect the real status of the webpa patch call // not the transformed custom status @@ -174,8 +177,8 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - statusCode = rherr.StatusCode // webpa error handling + statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 41ba1f3..570246b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -30,7 +30,9 @@ import ( "time" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" @@ -106,7 +108,7 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing - webpaPokeSpanName string + webpaPokeSpanTemplate string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -286,7 +288,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, } // Init the child poke span name - ws.webpaPokeSpanName = ws.WebpaConnector.PokeSpanName() + ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -882,13 +884,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { } ctx = trace.ContextWithSpanContext(ctx, sc) - // Feedback: Better to use the "path"/API rather than a hard coded name - spanName := "oswebconfig_poke_handler" - pathTemplate, _ := mux.CurrentRoute(r).GetPathTemplate() - if pathTemplate != "" { - spanName = pathTemplate - } - ctx, span := newSpan(ctx, spanName, "POST") + ctx, span := s.newParentPokeSpan(ctx, r) defer endSpan(span, w) // Pass the context with the span to the next handler @@ -916,32 +912,60 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { +func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { var span trace.Span - // Convention: method followed by path template - spanNameWithMethod := method + " " + spanName - ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) - envAttr := attribute.String("env", otelTracer.envName) - span.SetAttributes(envAttr) + // Feedback: Better to use the "path"/API rather than a hard coded name + route := "oswebconfig_poke_handler" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + route, _ = mux.CurrentRoute(r).GetPathTemplate() + } + s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) + return ctx, span +} - resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) - span.SetAttributes(resourceNameAttr) +func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) - routeAttr := attribute.String("http.route", spanName) - span.SetAttributes(routeAttr) + s.addAttributes(span, route, path, fullPath, method) + return ctx, span +} - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) +func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { + span.SetAttributes( + attribute.String("env", otelTracer.envName), + attribute.String("operation.name", otelTracer.opName), + attribute.String("http.url_details.path", path), + semconv.HTTPMethodKey.String(method), + semconv.HTTPRouteKey.String(route), + semconv.HTTPURLKey.String(fullPath), + ) - return ctx, span + log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) + log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) + log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) + log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) + log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) + log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) } func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { - statusAttr := attribute.Int("http.status_code", xw.Status()) + statusCode := xw.Status() + statusAttr := attribute.Int("http.status_code", statusCode) span.SetAttributes(statusAttr) + log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) + } } span.End() } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index ca57dc1..f6aadbe 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,9 +168,14 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } -func (c *WebpaConnector) PokeSpanName() string { +func (c *WebpaConnector) PokeSpanTemplate() string { // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") +} + +// Base URL with the cpemac populated +func (c *WebpaConnector) PokeSpanPath(mac string) string { + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) } func (c *WebpaConnector) NewQueue(capacity int) error { From c7a9e82a74bcdc6793af4db4af8248d6eb5ab07a Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 29 Jun 2024 13:11:36 -0700 Subject: [PATCH 066/215] Add the capability to forward kafka messages --- common/log_fields.go | 16 +++++++++++ common/log_fields_test.go | 25 +++++++++++++++++ http/otel.go | 3 ++- http/webconfig_server.go | 57 +++++++++++++++++++++++++++++++++++++++ kafka/consumer.go | 4 +++ 5 files changed, 104 insertions(+), 1 deletion(-) diff --git a/common/log_fields.go b/common/log_fields.go index 06289e6..8ec4f19 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -28,6 +28,12 @@ var ( "out_traceparent", "out_tracestate", } + coreFields = []string{ + "app_name", + "audit_id", + "body", + "cpe_mac", + } ) func FilterLogFields(src log.Fields, excludes ...string) log.Fields { @@ -53,3 +59,13 @@ func UpdateLogFields(fields, newfields log.Fields) { fields[k] = v } } + +func CopyCoreLogFields(src log.Fields) log.Fields { + fields := log.Fields{} + for _, k := range coreFields { + if itf, ok := src[k]; ok { + fields[k] = itf + } + } + return fields +} diff --git a/common/log_fields_test.go b/common/log_fields_test.go index e357855..4a95b9a 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -95,3 +95,28 @@ func TestUpdateLogFields(t *testing.T) { assert.DeepEqual(t, src, expected) } + +func TestCopyCoreLogFields(t *testing.T) { + body := map[string]interface{}{ + "device_id": "mac:29cf4fe3914e", + "http_status_code": 304, + "transaction_uuid": "f160f5f2-c899-4652-b066-c9b68328d74f", + "version": "1719689278", + } + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + expected := log.Fields{ + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + copied := CopyCoreLogFields(src) + assert.DeepEqual(t, copied, expected) +} diff --git a/http/otel.go b/http/otel.go index 468035f..f36c0a1 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -35,6 +35,7 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" ) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 570246b..d3311a0 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -34,6 +34,7 @@ import ( "go.opentelemetry.io/otel/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" @@ -91,6 +92,7 @@ type WebconfigServer struct { *XconfConnector *MqttConnector *UpstreamConnector + sarama.AsyncProducer tlsConfig *tls.Config notLoggedHeaders []string metricsEnabled bool @@ -109,6 +111,8 @@ type WebconfigServer struct { supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing webpaPokeSpanTemplate string + kafkaProducerEnabled bool + kafkaProducerTopic string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -255,6 +259,25 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) + // kafka producer + var kafkaProducer sarama.AsyncProducer + kafkaProducerEnabled := conf.GetBoolean("webconfig.kafka_producer.enabled") + var kafkaProducerTopic string + if kafkaProducerEnabled { + brokersStr := conf.GetString("webconfig.kafka_producer.brokers") + if len(brokersStr) == 0 { + panic(fmt.Errorf("webconfig.kafka_producer.brokers is empty")) + } + brokers := strings.Split(brokersStr, ",") + kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") + + saramaConfig := sarama.NewConfig() + kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) + if err != nil { + panic(err) + } + } + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -269,6 +292,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer XconfConnector: NewXconfConnector(conf, tlsConfig), MqttConnector: NewMqttConnector(conf, tlsConfig), UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), + AsyncProducer: kafkaProducer, tlsConfig: tlsConfig, notLoggedHeaders: notLoggedHeaders, metricsEnabled: metricsEnabled, @@ -286,6 +310,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer tracestateVendorID: tracestateVendorID, otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, + kafkaProducerEnabled: kafkaProducerEnabled, + kafkaProducerTopic: kafkaProducerTopic, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -582,6 +608,22 @@ func (s *WebconfigServer) SetSupplementaryAppendingEnabled(enabled bool) { s.supplementaryAppendingEnabled = enabled } +func (s *WebconfigServer) KafkaProducerEnabled() bool { + return s.kafkaProducerEnabled +} + +func (s *WebconfigServer) SetKafkaProducerEnabled(enabled bool) { + s.kafkaProducerEnabled = enabled +} + +func (s *WebconfigServer) KafkaProducerTopic() string { + return s.kafkaProducerTopic +} + +func (s *WebconfigServer) SetKafkaProducerTopic(x string) { + s.kafkaProducerTopic = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { @@ -974,3 +1016,18 @@ func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes } + +func (s *WebconfigServer) ForwardKafkaMessage(message *sarama.ConsumerMessage, fields log.Fields) { + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(message.Key), + Value: sarama.ByteEncoder(message.Value), + } + s.Input() <- outMessage + + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["kafka_topic"] = outMessage.Topic + tfields["kafka_key"] = string(message.Key) + log.WithFields(tfields).Info("send") +} diff --git a/kafka/consumer.go b/kafka/consumer.go index 1135d5d..83263ac 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -253,6 +253,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } metrics.CountKafkaEvents(eventName, status, message.Partition) } + + if c.KafkaProducerEnabled() { + c.ForwardKafkaMessage(message, fields) + } case <-session.Context().Done(): return nil } From 74652a9f668462fc6aa858332f857b7a85300bb2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 00:20:06 -0700 Subject: [PATCH 067/215] cherrypick and resolve conflicts --- config/sample_webconfig.conf | 7 ++++++ db/cassandra/state_update_test.go | 12 ++++++---- db/service.go | 30 ++++++++++++------------ http/otel.go | 2 +- http/upstream_test.go | 15 ++++++++---- http/webconfig_server.go | 17 ++++++++++---- kafka/consumer.go | 38 +++++++++++++++++++++++-------- 7 files changed, 82 insertions(+), 39 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 0ac059b..60d9364 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -245,4 +245,11 @@ webconfig { // correct subdoc states if versions match but not "deployed" state_correction_enabled= false + + // forward kafka messages if needed + kafka_producer { + enabled = false + brokers = "localhost:9092" + topics = "webconfig_downstream" + } } diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 0857770..2d9bbb0 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -58,8 +58,9 @@ func TestStateUpdate1(t *testing.T) { var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -72,8 +73,9 @@ func TestStateUpdate1(t *testing.T) { m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -109,8 +111,9 @@ func TestStateUpdate2(t *testing.T) { var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -123,8 +126,9 @@ func TestStateUpdate2(t *testing.T) { m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, ok) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/service.go b/db/service.go index 6f25653..9493527 100644 --- a/db/service.go +++ b/db/service.go @@ -335,10 +335,11 @@ func HashRootVersion(itf interface{}) string { return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) error { +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) (bool, error) { + var updatedBy304 bool // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return nil + return updatedBy304, nil } updatedTime := int(time.Now().UnixNano() / 1000000) @@ -350,7 +351,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } labels["client"] = metricsAgent @@ -359,14 +360,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return nil + return updatedBy304, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } newState := common.Deployed @@ -375,20 +376,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { + updatedBy304 = true newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } } } - return nil + return updatedBy304, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return common.NewError(fmt.Errorf("ill-formatted event")) + return updatedBy304, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -401,13 +403,13 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return nil + return updatedBy304, nil } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } var oldState int @@ -417,7 +419,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return common.NewError(err) + return updatedBy304, common.NewError(err) } } @@ -427,7 +429,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return common.NewError(err) + return updatedBy304, common.NewError(err) } } @@ -440,9 +442,9 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } - return nil + return updatedBy304, nil } func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { diff --git a/http/otel.go b/http/otel.go index f36c0a1..8c82c39 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( diff --git a/http/upstream_test.go b/http/upstream_test.go index 9a7807b..127dcc1 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -263,8 +263,9 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -525,8 +526,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -775,8 +777,9 @@ func TestUpstreamUpdatedTime(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -1074,8 +1077,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -1362,8 +1366,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== diff --git a/http/webconfig_server.go b/http/webconfig_server.go index d3311a0..27da9c7 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1017,17 +1017,24 @@ func hexStringToBytes(hexString string) []byte { return bytes } -func (s *WebconfigServer) ForwardKafkaMessage(message *sarama.ConsumerMessage, fields log.Fields) { +func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } outMessage := &sarama.ProducerMessage{ Topic: s.KafkaProducerTopic(), - Key: sarama.ByteEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), + Key: sarama.ByteEncoder(kbytes), + Value: sarama.ByteEncoder(bbytes), } s.Input() <- outMessage - tfields := common.CopyCoreLogFields(fields) tfields["logger"] = "kafkaproducer" tfields["kafka_topic"] = outMessage.Topic - tfields["kafka_key"] = string(message.Key) + tfields["kafka_key"] = string(kbytes) log.WithFields(tfields).Info("send") } diff --git a/kafka/consumer.go b/kafka/consumer.go index 83263ac..52b22fb 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/http" "strings" "time" @@ -75,31 +76,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, bool, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, common.NewError(err) + return nil, false, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, common.NewError(err) + return nil, false, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, nil + return &m, false, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - err = db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedBy304, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, common.NewError(err) + return &m, updatedBy304, common.NewError(err) } - return &m, nil + return &m, updatedBy304, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -203,6 +204,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage + var updatedBy304 bool eventName, rptHeaderValue := getEventName(message) switch eventName { @@ -212,10 +214,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, err = c.handleNotification(bbytes, fields) + m, updatedBy304, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, err = c.handleNotification(message.Value, fields) + m, updatedBy304, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -255,7 +257,23 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() { - c.ForwardKafkaMessage(message, fields) + if len(m.Reports) == 0 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { + // build a root/success message + namespace := "root" + applicationStatus := "success" + em := &common.EventMessage{ + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) + } else if m.ApplicationStatus != nil && *m.ApplicationStatus == "success" && m.Namespace != nil && *m.Namespace != "telemetry" { + c.ForwardKafkaMessage(message.Key, m, fields) + } + } } case <-session.Context().Done(): return nil From e57980525b7e8fb085f1660d61e5f81a52145994 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 00:26:55 -0700 Subject: [PATCH 068/215] fix a typo in sample config --- config/sample_webconfig.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 60d9364..4b67a55 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -250,6 +250,6 @@ webconfig { kafka_producer { enabled = false brokers = "localhost:9092" - topics = "webconfig_downstream" + topic = "webconfig_downstream" } } From 2ee6174f26391faf08c7f435a54a4f53312c7538 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 14:09:43 -0700 Subject: [PATCH 069/215] handle ill-formatted kafka messages in forwarding --- kafka/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 52b22fb..5d94e00 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -256,7 +256,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram metrics.CountKafkaEvents(eventName, status, message.Partition) } - if c.KafkaProducerEnabled() { + if c.KafkaProducerEnabled() && m != nil { if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { // build a root/success message From c741d32cdca2180baa7b481ce4569990858c5bc0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 16 Jul 2024 14:12:21 -0700 Subject: [PATCH 070/215] fix a bug that forwarded kafka messages were not logged properly --- http/webconfig_server.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 27da9c7..8deb28e 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -31,8 +31,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/trace" "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" @@ -1034,7 +1034,8 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess s.Input() <- outMessage tfields["logger"] = "kafkaproducer" - tfields["kafka_topic"] = outMessage.Topic - tfields["kafka_key"] = string(kbytes) + tfields["output_topic"] = outMessage.Topic + tfields["output_key"] = string(kbytes) + tfields["output_body"] = m log.WithFields(tfields).Info("send") } From 1f6c9e3c900b725017d96af76b367cba381e6cf9 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 16 Jul 2024 14:12:21 -0700 Subject: [PATCH 071/215] fix a bug that forwarded kafka messages were not logged properly --- http/webconfig_server.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 27da9c7..8deb28e 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -31,8 +31,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/trace" "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" @@ -1034,7 +1034,8 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess s.Input() <- outMessage tfields["logger"] = "kafkaproducer" - tfields["kafka_topic"] = outMessage.Topic - tfields["kafka_key"] = string(kbytes) + tfields["output_topic"] = outMessage.Topic + tfields["output_key"] = string(kbytes) + tfields["output_body"] = m log.WithFields(tfields).Info("send") } From 37e8626cb3d84309fbc5a95453cc40010b142065 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 19 Jul 2024 15:34:31 -0700 Subject: [PATCH 072/215] Forward webconfig notifications without filtering --- kafka/consumer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 5d94e00..85b6fd9 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -257,6 +257,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() && m != nil { + c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { // build a root/success message @@ -270,8 +271,6 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram Version: m.Version, } c.ForwardKafkaMessage(message.Key, em, fields) - } else if m.ApplicationStatus != nil && *m.ApplicationStatus == "success" && m.Namespace != nil && *m.Namespace != "telemetry" { - c.ForwardKafkaMessage(message.Key, m, fields) } } } From 510eecada3b84ca94ef9e6805a1182f073680412 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 19 Jul 2024 15:34:31 -0700 Subject: [PATCH 073/215] Forward webconfig notifications without filtering --- kafka/consumer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 5d94e00..85b6fd9 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -257,6 +257,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() && m != nil { + c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { // build a root/success message @@ -270,8 +271,6 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram Version: m.Version, } c.ForwardKafkaMessage(message.Key, em, fields) - } else if m.ApplicationStatus != nil && *m.ApplicationStatus == "success" && m.Namespace != nil && *m.Namespace != "telemetry" { - c.ForwardKafkaMessage(message.Key, m, fields) } } } From 40ae4a14f420be789144b3be952e65d820b24c9e Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 30 Jul 2024 19:16:43 -0700 Subject: [PATCH 074/215] generate notifications if states are corrected by versions from devices --- db/cassandra/document_test.go | 2 +- db/service.go | 42 ++++++++++++++++++++++------------- http/multipart.go | 5 ++++- http/otel.go | 2 +- http/webconfig_server.go | 34 ++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 19 deletions(-) diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index b314690..8e63844 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -206,7 +206,7 @@ func TestBlockedSubdocIds(t *testing.T) { rHeader.Set(common.HeaderSupportedDocs, rdkSupportedDocsHeaderStr) - document, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) + document, _, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) assert.NilError(t, err) assert.Assert(t, document.Length() == 3) versionMap := document.VersionMap() diff --git a/db/service.go b/db/service.go index 9493527..bce94fa 100644 --- a/db/service.go +++ b/db/service.go @@ -45,7 +45,7 @@ var ( // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, []common.EventMessage, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) tfields := common.FilterLogFields(fields) @@ -57,6 +57,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error + messages := []common.EventMessage{} supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) if err != nil { log.WithFields(tfields).Warn(err) @@ -115,11 +116,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel document, err = c.GetDocument(mac, fields) if err != nil { // TODO what about 404 should be included here - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } deviceVersionMap = RebuildDeviceVersionMap(versions, document.VersionMap()) } else { - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -130,7 +131,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel cloudRootDocument, err := c.GetRootDocument(mac) if err != nil { if !c.IsDbNotFound(err) { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // no root doc in db, create a new one // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud @@ -143,10 +144,10 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // the returned err is dbNotFound - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== @@ -174,7 +175,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -182,7 +183,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } for subdocId, subdocument := range document.Items() { @@ -201,11 +202,20 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel newState := common.Deployed subdocument.SetState(&newState) if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } + applicationStatus := "success" + namespace := subdocId + version := cloudVersion + m := common.EventMessage{ + DeviceId: "mac:" + mac, + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + Version: &version, + } + messages = append(messages, m) } } - } switch rootCmpEnum { @@ -213,7 +223,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // create an empty "document" document := common.NewDocument(cloudRootDocument) // no need to update root doc - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentVersionOnlyChanged, common.RootDocumentMissing: // meta unchanged but subdoc versions change ==> new configs // getDoc, then filter @@ -221,7 +231,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) @@ -229,23 +239,23 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel for _, subdocId := range c.BlockedSubdocIds() { filteredDocument.DeleteSubDocument(subdocId) } - return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentMetaChanged: // getDoc, send it upstream if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, messages, nil } // default, should not come here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil } func GetValuesStr(length int) string { diff --git a/http/multipart.go b/http/multipart.go index c1f29d4..a8676a9 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -128,7 +128,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return status, respHeader, rbytes, nil } - document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, err := db.BuildGetDocument(c, rHeader, route, fields) + document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, messages, err := db.BuildGetDocument(c, rHeader, route, fields) + if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { + s.ForwardSuccessKafkaMessages(messages, fields) + } if uconn == nil { if err != nil { if !s.IsDbNotFound(err) { diff --git a/http/otel.go b/http/otel.go index 4c649cc..468035f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 8deb28e..59da474 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -42,6 +42,7 @@ import ( "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" + "github.com/google/uuid" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -1039,3 +1040,36 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess tfields["output_body"] = m log.WithFields(tfields).Info("send") } + +func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["output_topic"] = s.KafkaProducerTopic() + + for _, m := range messages { + if len(m.DeviceId) != 16 { + log.WithFields(tfields).Warn("invalid device_id " + m.DeviceId) + continue + } + mac := m.DeviceId[4:] + transactionUuid := s.AppName() + "_____" + uuid.New().String() + m.TransactionUuid = &transactionUuid + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(mac), + Value: sarama.ByteEncoder(bbytes), + } + s.Input() <- outMessage + + tfields["output_key"] = mac + tfields["output_body"] = m + log.WithFields(tfields).Info("send") + } +} From 658c0ade33698bf517557a3c67628a30431e72cf Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 3 Aug 2024 10:52:13 -0700 Subject: [PATCH 075/215] revise bitmap test codes --- common/bitmap.go | 169 ++++++ common/bitmap_test.go | 47 ++ common/const_var.go | 127 ----- http/supported_groups_handler_test.go | 184 +++---- util/firmware_bitmap_test.go | 734 +++++++++----------------- 5 files changed, 533 insertions(+), 728 deletions(-) create mode 100644 common/bitmap.go create mode 100644 common/bitmap_test.go diff --git a/common/bitmap.go b/common/bitmap.go new file mode 100644 index 0000000..dd9c777 --- /dev/null +++ b/common/bitmap.go @@ -0,0 +1,169 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +// header X-System-Supported-Docs +type BitMaskTuple struct { + GroupBit int + CpeBit int +} + +// The group based bitmaps will be merged into 1 cpe bitmap +// 1: []BitMaskTuple{ // meta_group_id: defined by RDK +// +// BitMaskTuple{1, 1}, // {"index_of_bit_from_lsb" for a group bitmap, "index_of_bit_from_lsb" for the cpe bitmap +var ( + SupportedDocsBitMaskMap = map[int][]BitMaskTuple{ + 1: { + {1, 1}, + {2, 2}, + {3, 3}, + {4, 4}, + {5, 5}, + {6, 6}, + {7, 29}, // connectedbuilding + {8, 35}, // xmspeedboost + }, + 2: { + {1, 7}, + {2, 8}, + {3, 9}, + }, + 3: { + {1, 10}, + }, + 4: { + {1, 11}, + }, + 5: { + {1, 12}, + }, + 6: { + {1, 13}, // mesh + {2, 31}, // clienttosteeringprofile + {3, 36}, // meshsteeringprofiles + {4, 37}, // wifistatsconfig + {5, 38}, // mwoconfigs + {6, 39}, // interference + {7, 34}, // wifimotionsettings + }, + 7: { + {1, 14}, + }, + 8: { + {1, 15}, + {2, 32}, + {3, 33}, + }, + 9: { + {1, 16}, + {2, 17}, + }, + 10: { + {1, 18}, + {2, 19}, + }, + 11: { + {1, 20}, + {2, 25}, + }, + 12: { + {1, 21}, + {2, 23}, + }, + 13: { + {1, 22}, + }, + 14: { + {1, 24}, + }, + 15: { + {1, 26}, + {2, 27}, + }, + 16: { + {1, 28}, + }, + 17: { + {1, 30}, + }, + } +) + +var ( + SubdocBitIndexMap = map[string]int{ + "portforwarding": 1, + "lan": 2, + "wan": 3, + "macbinding": 4, + "hotspot": 5, + "bridge": 6, + "privatessid": 7, + "homessid": 8, + "radio": 9, + "moca": 10, + "xdns": 11, + "advsecurity": 12, + "mesh": 13, + "aker": 14, + "telemetry": 15, + "statusreport": 16, + "trafficreport": 17, + "interfacereport": 18, + "radioreport": 19, + "telcovoip": 20, + "wanmanager": 21, + "voiceservice": 22, + "wanfailover": 23, + "cellularconfig": 24, + "telcovoice": 25, + "gwfailover": 26, + "gwrestore": 27, + "prioritizedmacs": 28, + "connectedbuilding": 29, + "lldqoscontrol": 30, + "clienttosteeringprofile": 31, + "defaultrfc": 32, + "rfc": 33, + "wifimotionsettings": 34, + "xmspeedboost": 35, + "meshsteeringprofiles": 36, + "wifistatsconfig": 37, + "mwoconfigs": 38, + "interference": 39, + } +) + +func GetDefaultSupportedSubdocMap() map[string]bool { + m := make(map[string]bool) + for k := range SubdocBitIndexMap { + m[k] = false + } + return m +} + +func BuildSupportedSubdocMapWithDefaults(supportedSubdocIds []string) map[string]bool { + m := make(map[string]bool) + for k := range SubdocBitIndexMap { + m[k] = false + } + for _, s := range supportedSubdocIds { + m[s] = true + } + return m +} diff --git a/common/bitmap_test.go b/common/bitmap_test.go new file mode 100644 index 0000000..9c75139 --- /dev/null +++ b/common/bitmap_test.go @@ -0,0 +1,47 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "slices" + "testing" + + "gotest.tools/assert" +) + +func TestGetDefaultSupportedSubdocMap(t *testing.T) { + m := GetDefaultSupportedSubdocMap() + assert.Equal(t, len(m), len(SubdocBitIndexMap)) +} + +func TestBuildSupportedSubdocMapWithDefaults(t *testing.T) { + supportedSubdocIds := []string{ + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + } + m := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.Equal(t, len(m), len(SubdocBitIndexMap)) + for k := range m { + if m[k] { + assert.Assert(t, slices.Contains(supportedSubdocIds, k)) + } + } +} diff --git a/common/const_var.go b/common/const_var.go index 6d57e4a..513807e 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -114,133 +114,6 @@ const ( SkipDbUpdate = "skip-db-update" ) -// header X-System-Supported-Docs -type BitMaskTuple struct { - GroupBit int - CpeBit int -} - -// The group based bitmaps will be merged into 1 cpe bitmap -// 1: []BitMaskTuple{ // meta_group_id: defined by RDK -// -// BitMaskTuple{1, 1}, // {"index_of_bit_from_lsb" for a group bitmap, "index_of_bit_from_lsb" for the cpe bitmap -var ( - SupportedDocsBitMaskMap = map[int][]BitMaskTuple{ - 1: { - {1, 1}, - {2, 2}, - {3, 3}, - {4, 4}, - {5, 5}, - {6, 6}, - {7, 29}, // connectedbuilding - {8, 35}, // xmspeedboost - }, - 2: { - {1, 7}, - {2, 8}, - {3, 9}, - }, - 3: { - {1, 10}, - }, - 4: { - {1, 11}, - }, - 5: { - {1, 12}, - }, - 6: { - {1, 13}, // mesh - {2, 31}, // clienttosteeringprofile - {3, 36}, // meshsteeringprofiles - {4, 37}, // wifistatsconfig - {5, 38}, // mwoconfigs - {6, 39}, // interference - {7, 34}, // wifimotionsettings - }, - 7: { - {1, 14}, - }, - 8: { - {1, 15}, - {2, 32}, - {3, 33}, - }, - 9: { - {1, 16}, - {2, 17}, - }, - 10: { - {1, 18}, - {2, 19}, - }, - 11: { - {1, 20}, - {2, 25}, - }, - 12: { - {1, 21}, - {2, 23}, - }, - 13: { - {1, 22}, - }, - 14: { - {1, 24}, - }, - 15: { - {1, 26}, - {2, 27}, - }, - 16: { - {1, 28}, - }, - 17: { - {1, 30}, - }, - } -) - -var ( - SubdocBitIndexMap = map[string]int{ - "portforwarding": 1, - "lan": 2, - "wan": 3, - "macbinding": 4, - "hotspot": 5, - "bridge": 6, - "privatessid": 7, - "homessid": 8, - "radio": 9, - "moca": 10, - "xdns": 11, - "advsecurity": 12, - "mesh": 13, - "aker": 14, - "telemetry": 15, - "statusreport": 16, - "trafficreport": 17, - "interfacereport": 18, - "radioreport": 19, - "telcovoip": 20, - "wanmanager": 21, - "voiceservice": 22, - "wanfailover": 23, - "cellularconfig": 24, - "telcovoice": 25, - "gwfailover": 26, - "gwrestore": 27, - "prioritizedmacs": 28, - "connectedbuilding": 29, - "lldqoscontrol": 30, - "clienttosteeringprofile": 31, - "defaultrfc": 32, - "rfc": 33, - "wifimotionsettings": 34, - "xmspeedboost": 35, - } -) var ( SupportedPokeDocs = []string{"primary", "telemetry"} diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 13ffbe9..833d656 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -46,6 +46,7 @@ func TestSupportedGroupsHandler(t *testing.T) { res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -66,43 +67,21 @@ func TestSupportedGroupsHandler(t *testing.T) { assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - "wanfailover": false, - "cellularconfig": false, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -115,7 +94,7 @@ func TestSupportedGroupsHandler(t *testing.T) { var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) // ==== step 2 add lan data ==== subdocId := "lan" @@ -132,6 +111,7 @@ func TestSupportedGroupsHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusOK) // get @@ -170,7 +150,7 @@ func TestSupportedGroupsHandler(t *testing.T) { err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) // ==== step 5 setup supported docs for fw version 2 ===== sids := strings.Split(rdkSupportedDocsHeaderStr, ",") @@ -179,17 +159,17 @@ func TestSupportedGroupsHandler(t *testing.T) { group1Bitmap, err := util.BitarrayToBitmap(newGroup1Bitarray) assert.NilError(t, err) sids[0] = fmt.Sprintf("%v", group1Bitmap) - expectedEnabled["wan"] = false - expectedEnabled["macbinding"] = false - expectedEnabled["hotspot"] = true - expectedEnabled["bridge"] = true + supportedSubdocMap["wan"] = false + supportedSubdocMap["macbinding"] = false + supportedSubdocMap["hotspot"] = true + supportedSubdocMap["bridge"] = true newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" group2Bitmap, err := util.BitarrayToBitmap(newGroup2Bitarray) assert.NilError(t, err) sids[1] = fmt.Sprintf("%v", group2Bitmap) - expectedEnabled["privatessid"] = false - expectedEnabled["radio"] = true + supportedSubdocMap["privatessid"] = false + supportedSubdocMap["radio"] = true rdkSupportedDocsHeaderStr = strings.Join(sids, ",") @@ -218,7 +198,7 @@ func TestSupportedGroupsHandler(t *testing.T) { err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { @@ -235,6 +215,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -255,43 +236,22 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": false, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": true, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telcovoice", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -304,7 +264,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testing.T) { @@ -321,6 +281,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -341,43 +302,28 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwfailover": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "gwfailover", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -390,5 +336,5 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 28c3159..8572f68 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -22,6 +22,7 @@ import ( "strings" "testing" + "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) @@ -155,20 +156,15 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { } parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocIds := []string{} + for k, v := range expectedEnabled { + if v { + supportedSubdocIds = append(supportedSubdocIds, k) + } + } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseTelcovoipAndWanmanager(t *testing.T) { @@ -176,55 +172,30 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { sids := strings.Split(rdkSupportedDocsHeaderStr, ",") // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": false, - "radio": false, - "moca": false, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": true, - "telcovoice": false, - "wanmanager": true, - "voiceservice": false, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "privatessid", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "telcovoip", + "wanmanager", } rdkSupportedDocsHeaderStr = strings.Join(sids, ",") cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) + for _, subdocId := range supportedSubdocIds { + assert.Assert(t, IsSubdocSupported(cpeBitmap, subdocId)) } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestBitmapParsing(t *testing.T) { @@ -233,110 +204,52 @@ func TestBitmapParsing(t *testing.T) { rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,33554435,50331649,67108865,83886081,100663297,117440513,134217729", newBitmap) // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": false, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "macbinding", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseVoiceService(t *testing.T) { rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809" - // sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": true, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": true, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", } - // rdkSupportedDocsHeaderStr = strings.Join(sids, ",") cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestManualBitmap(t *testing.T) { @@ -355,425 +268,282 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": true, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": true, - "cellularconfig": true, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", + "cellularconfig", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { rdkSupportedDocsHeaderStr := "16777231,33554435,67108865,100663297,117440513,134217729,201326595,234881025" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": false, - "aker": true, - "bridge": false, - "cellularconfig": true, - "homessid": true, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, + supportedSubdocIds := []string{ + "aker", + "cellularconfig", + "homessid", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549378,201326595" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": false, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": true, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telcovoice", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, - "clienttosteeringprofile": false, - "rfc": false, - "defaultrfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,251658241,268435457" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": false, - "lldqoscontrol": false, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *testing.T) { rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241,268435457,285212673" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - "clienttosteeringprofile": true, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderRfc(t *testing.T) { rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - "clienttosteeringprofile": true, - "defaultrfc": true, - "rfc": true, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "defaultrfc", + "rfc", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderHcm(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", } + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } From 3c32f125f99f381c070ada060952c45cf25dfb45 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 30 Jul 2024 19:16:43 -0700 Subject: [PATCH 076/215] generate notifications if states are corrected by versions from devices --- db/cassandra/document_test.go | 2 +- db/service.go | 42 ++++++++++++++++++++++------------- http/multipart.go | 5 ++++- http/webconfig_server.go | 34 ++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index b314690..8e63844 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -206,7 +206,7 @@ func TestBlockedSubdocIds(t *testing.T) { rHeader.Set(common.HeaderSupportedDocs, rdkSupportedDocsHeaderStr) - document, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) + document, _, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) assert.NilError(t, err) assert.Assert(t, document.Length() == 3) versionMap := document.VersionMap() diff --git a/db/service.go b/db/service.go index 9493527..bce94fa 100644 --- a/db/service.go +++ b/db/service.go @@ -45,7 +45,7 @@ var ( // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, []common.EventMessage, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) tfields := common.FilterLogFields(fields) @@ -57,6 +57,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error + messages := []common.EventMessage{} supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) if err != nil { log.WithFields(tfields).Warn(err) @@ -115,11 +116,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel document, err = c.GetDocument(mac, fields) if err != nil { // TODO what about 404 should be included here - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } deviceVersionMap = RebuildDeviceVersionMap(versions, document.VersionMap()) } else { - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -130,7 +131,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel cloudRootDocument, err := c.GetRootDocument(mac) if err != nil { if !c.IsDbNotFound(err) { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // no root doc in db, create a new one // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud @@ -143,10 +144,10 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // the returned err is dbNotFound - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== @@ -174,7 +175,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -182,7 +183,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } for subdocId, subdocument := range document.Items() { @@ -201,11 +202,20 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel newState := common.Deployed subdocument.SetState(&newState) if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } + applicationStatus := "success" + namespace := subdocId + version := cloudVersion + m := common.EventMessage{ + DeviceId: "mac:" + mac, + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + Version: &version, + } + messages = append(messages, m) } } - } switch rootCmpEnum { @@ -213,7 +223,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // create an empty "document" document := common.NewDocument(cloudRootDocument) // no need to update root doc - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentVersionOnlyChanged, common.RootDocumentMissing: // meta unchanged but subdoc versions change ==> new configs // getDoc, then filter @@ -221,7 +231,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) @@ -229,23 +239,23 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel for _, subdocId := range c.BlockedSubdocIds() { filteredDocument.DeleteSubDocument(subdocId) } - return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentMetaChanged: // getDoc, send it upstream if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, messages, nil } // default, should not come here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil } func GetValuesStr(length int) string { diff --git a/http/multipart.go b/http/multipart.go index c1f29d4..a8676a9 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -128,7 +128,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return status, respHeader, rbytes, nil } - document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, err := db.BuildGetDocument(c, rHeader, route, fields) + document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, messages, err := db.BuildGetDocument(c, rHeader, route, fields) + if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { + s.ForwardSuccessKafkaMessages(messages, fields) + } if uconn == nil { if err != nil { if !s.IsDbNotFound(err) { diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 8deb28e..59da474 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -42,6 +42,7 @@ import ( "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" + "github.com/google/uuid" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -1039,3 +1040,36 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess tfields["output_body"] = m log.WithFields(tfields).Info("send") } + +func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["output_topic"] = s.KafkaProducerTopic() + + for _, m := range messages { + if len(m.DeviceId) != 16 { + log.WithFields(tfields).Warn("invalid device_id " + m.DeviceId) + continue + } + mac := m.DeviceId[4:] + transactionUuid := s.AppName() + "_____" + uuid.New().String() + m.TransactionUuid = &transactionUuid + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(mac), + Value: sarama.ByteEncoder(bbytes), + } + s.Input() <- outMessage + + tfields["output_key"] = mac + tfields["output_body"] = m + log.WithFields(tfields).Info("send") + } +} From 423ac08a47233b9fa44cc250a1c5397b2cf9d1a0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 7 Aug 2024 14:58:20 -0700 Subject: [PATCH 077/215] Return 504 if xconf GET returns timeout/context_cancellation --- http/http_client.go | 15 ++++- http/supplementary_handler_test.go | 95 +++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index e7e66bc..63d989c 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -263,6 +263,15 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] fields[fmt.Sprintf("%v_status", loggerName)] = res.StatusCode rbytes, err := io.ReadAll(res.Body) if err != nil { + // XPC-23206 catch the timeout/context_cancellation error + // ex: context deadline exceeded (Client.Timeout or context cancellation while reading body) + lowerErrText := strings.ToLower(err.Error()) + if strings.Contains(lowerErrText, "timeout") { + err = common.RemoteHttpError{ + Message: err.Error(), + StatusCode: http.StatusGatewayTimeout, + } + } fields[errorKey] = err.Error() if userAgent != "mget" { log.WithFields(fields).Info(endMessage) diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 942b0f8..ff6f445 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -797,3 +797,96 @@ func TestSupplementaryApiBadRequest(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusBadRequest) } + +func TestSupplementaryXconfTimeout(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(time.Duration(1) * time.Second) + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + server.XconfConnector.SetXconfHost(mockServer.URL) + server.XconfConnector.HttpClient.Client.Timeout = time.Duration(500) * time.Millisecond + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} + +type errorRoundTripper struct{} + +func (ert *errorRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Return a response with an error-inducing reader + return &http.Response{ + Body: io.NopCloser(&errorReader{}), + }, nil +} + +// errorReader simulates an error when reading +type errorReader struct{} + +func (er *errorReader) Read(p []byte) (int, error) { + return 0, fmt.Errorf("context deadline exceeded (Client.Timeout or context cancellation while reading body)") +} + +func TestSupplementaryXconfReadAllErr(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock client ==== + mockedClient := &http.Client{ + Transport: &errorRoundTripper{}, + } + + server.XconfConnector.HttpClient.Client = mockedClient + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} From 1ad7969441a11e97bb5f3fa8981630af554e41af Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Sep 2024 15:16:27 -0700 Subject: [PATCH 078/215] Add debug loggings for 403 error analysis --- http/http_client.go | 6 +++--- http/supplementary_handler_test.go | 2 +- http/webconfig_server.go | 31 ++++++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index 63d989c..aad3085 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/google/uuid" log "github.com/sirupsen/logrus" ) diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index ff6f445..54e2642 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 59da474..b3884af 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -81,6 +82,7 @@ var ( "privatessid", "homessid", } + codec *security.AesCodec ) type WebconfigServer struct { @@ -368,6 +370,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + var tokenErr error if len(token) > 0 { params := mux.Vars(r) mac, ok := params["mac"] @@ -382,18 +385,20 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { fields := xw.Audit() fields["src_partner"] = partnerId partnerId = "unknown" + tokenErr = common.NewError(err) } xw.SetPartnerId(partnerId) } else { - xw.LogDebug(r, "token", fmt.Sprintf("CpeMiddleware() VerifyCpeToken()=false, err=%v", err)) + tokenErr = common.NewError(err) } } else { - xw.LogDebug(r, "token", "CpeMiddleware() error no token") + tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) } if isValid { next.ServeHTTP(xw, r) } else { + s.LogToken(xw, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -1073,3 +1078,25 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes log.WithFields(tfields).Info("send") } } + +func (s *WebconfigServer) LogToken(xw *XResponseWriter, token string, tokenErr error) { + fields := xw.Audit() + fields["logger"] = "token" + tfields := common.FilterLogFields(fields) + + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken + } + + log.WithFields(tfields).Debug(tokenErr) +} From 367450ddf0c83b24dfb4a23c92fbfbe01849ec43 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Sep 2024 16:37:06 -0700 Subject: [PATCH 079/215] Add debug loggings for 403 error analysis --- http/webconfig_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index b3884af..da6fe3a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -398,6 +398,8 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { + fields := xw.Audit() + fields["full_header"] = util.HeaderToMap(getFilteredHeader(r, nil)) s.LogToken(xw, token, tokenErr) Error(xw, http.StatusForbidden, nil) } From 7e2b44c04238a061da125870e7dcbc3fd7d6f5ea Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 5 Sep 2024 11:54:46 -0700 Subject: [PATCH 080/215] Add debug loggings for 403 error analysis --- common/log_fields.go | 11 ++++++- common/log_fields_test.go | 69 +++++++++++++++++++++++++++++++++++++++ http/webconfig_server.go | 43 ++++++++++++++++-------- 3 files changed, 108 insertions(+), 15 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index 8ec4f19..5ce267b 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -18,6 +18,8 @@ package common import ( + "maps" + log "github.com/sirupsen/logrus" ) @@ -39,7 +41,14 @@ var ( func FilterLogFields(src log.Fields, excludes ...string) log.Fields { fields := log.Fields{} for k, v := range src { - fields[k] = v + switch ty := v.(type) { + case map[string]string: + fields[k] = maps.Clone(ty) + case map[string]interface{}: + fields[k] = maps.Clone(ty) + default: + fields[k] = ty + } } for _, x := range unloggedFields { diff --git a/common/log_fields_test.go b/common/log_fields_test.go index 4a95b9a..960af77 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -119,4 +119,73 @@ func TestCopyCoreLogFields(t *testing.T) { } copied := CopyCoreLogFields(src) assert.DeepEqual(t, copied, expected) + + body["violet"] = "purple" + +} + +func TestFilterLogFieldsWithItfMap(t *testing.T) { + weekday := map[string]interface{}{ + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]interface{}) + fw["fri"] = 5 + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]interface{}) + assert.Assert(t, len(sw) == 4) +} + +func TestFilterLogFieldsWithStrMap(t *testing.T) { + weekday := map[string]string{ + "mon": "1", + "tue": "2", + "wed": "3", + "thu": "4", + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]string) + fw["fri"] = "5" + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]string) + assert.Assert(t, len(sw) == 4) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index da6fe3a..05d4ea2 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -66,6 +66,7 @@ const ( defaultTraceparentParentID = "0000000000000001" defaultTracestateVendorID = "webconfig" defaultSupplementaryAppendingEnabled = true + authPrefixLength = 60 ) var ( @@ -370,6 +371,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { params := mux.Vars(r) @@ -398,9 +400,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { - fields := xw.Audit() - fields["full_header"] = util.HeaderToMap(getFilteredHeader(r, nil)) - s.LogToken(xw, token, tokenErr) + s.LogToken(xw, authorization, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -1081,23 +1081,38 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } } -func (s *WebconfigServer) LogToken(xw *XResponseWriter, token string, tokenErr error) { +func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token string, tokenErr error) { fields := xw.Audit() fields["logger"] = "token" tfields := common.FilterLogFields(fields) - - if codec == nil { - codec, _ = security.NewAesCodec(s.Config) + var headerMap map[string]string + var isObfuscated bool + if itf, ok := tfields["header"]; ok { + headerMap = itf.(map[string]string) + if len(headerMap) > 0 { + ss := authorization + if len(ss) > authPrefixLength { + ss = authorization[:authPrefixLength] + "****" + isObfuscated = true + } + headerMap["Authorization"] = ss + } } - if codec == nil { - tfields["plaintoken"] = token - } else { - var encToken string - if encryptedB64, err := codec.Encrypt(token); err == nil { - encToken = encryptedB64 + if isObfuscated { + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken } - tfields["enctoken"] = encToken } log.WithFields(tfields).Debug(tokenErr) From 3a24698ce19636b7a20965af8affbd112104e294 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Sep 2024 15:16:27 -0700 Subject: [PATCH 081/215] Add debug loggings for 403 error analysis --- http/http_client.go | 6 +++--- http/supplementary_handler_test.go | 2 +- http/webconfig_server.go | 31 ++++++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index 63d989c..aad3085 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/google/uuid" log "github.com/sirupsen/logrus" ) diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index ff6f445..54e2642 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 59da474..b3884af 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -81,6 +82,7 @@ var ( "privatessid", "homessid", } + codec *security.AesCodec ) type WebconfigServer struct { @@ -368,6 +370,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + var tokenErr error if len(token) > 0 { params := mux.Vars(r) mac, ok := params["mac"] @@ -382,18 +385,20 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { fields := xw.Audit() fields["src_partner"] = partnerId partnerId = "unknown" + tokenErr = common.NewError(err) } xw.SetPartnerId(partnerId) } else { - xw.LogDebug(r, "token", fmt.Sprintf("CpeMiddleware() VerifyCpeToken()=false, err=%v", err)) + tokenErr = common.NewError(err) } } else { - xw.LogDebug(r, "token", "CpeMiddleware() error no token") + tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) } if isValid { next.ServeHTTP(xw, r) } else { + s.LogToken(xw, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -1073,3 +1078,25 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes log.WithFields(tfields).Info("send") } } + +func (s *WebconfigServer) LogToken(xw *XResponseWriter, token string, tokenErr error) { + fields := xw.Audit() + fields["logger"] = "token" + tfields := common.FilterLogFields(fields) + + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken + } + + log.WithFields(tfields).Debug(tokenErr) +} From 820b6780c038e0f205b9b40eb38ec375027e277f Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Sep 2024 16:37:06 -0700 Subject: [PATCH 082/215] Add debug loggings for 403 error analysis --- http/webconfig_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index b3884af..da6fe3a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -398,6 +398,8 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { + fields := xw.Audit() + fields["full_header"] = util.HeaderToMap(getFilteredHeader(r, nil)) s.LogToken(xw, token, tokenErr) Error(xw, http.StatusForbidden, nil) } From 98e9e7dfc3edf77c9fe1fcfbc9c43f5e36af72ff Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 5 Sep 2024 11:54:46 -0700 Subject: [PATCH 083/215] Add debug loggings for 403 error analysis --- common/log_fields.go | 11 ++++++- common/log_fields_test.go | 69 +++++++++++++++++++++++++++++++++++++++ http/webconfig_server.go | 43 ++++++++++++++++-------- 3 files changed, 108 insertions(+), 15 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index 8ec4f19..5ce267b 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -18,6 +18,8 @@ package common import ( + "maps" + log "github.com/sirupsen/logrus" ) @@ -39,7 +41,14 @@ var ( func FilterLogFields(src log.Fields, excludes ...string) log.Fields { fields := log.Fields{} for k, v := range src { - fields[k] = v + switch ty := v.(type) { + case map[string]string: + fields[k] = maps.Clone(ty) + case map[string]interface{}: + fields[k] = maps.Clone(ty) + default: + fields[k] = ty + } } for _, x := range unloggedFields { diff --git a/common/log_fields_test.go b/common/log_fields_test.go index 4a95b9a..960af77 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -119,4 +119,73 @@ func TestCopyCoreLogFields(t *testing.T) { } copied := CopyCoreLogFields(src) assert.DeepEqual(t, copied, expected) + + body["violet"] = "purple" + +} + +func TestFilterLogFieldsWithItfMap(t *testing.T) { + weekday := map[string]interface{}{ + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]interface{}) + fw["fri"] = 5 + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]interface{}) + assert.Assert(t, len(sw) == 4) +} + +func TestFilterLogFieldsWithStrMap(t *testing.T) { + weekday := map[string]string{ + "mon": "1", + "tue": "2", + "wed": "3", + "thu": "4", + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]string) + fw["fri"] = "5" + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]string) + assert.Assert(t, len(sw) == 4) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index da6fe3a..05d4ea2 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -66,6 +66,7 @@ const ( defaultTraceparentParentID = "0000000000000001" defaultTracestateVendorID = "webconfig" defaultSupplementaryAppendingEnabled = true + authPrefixLength = 60 ) var ( @@ -370,6 +371,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { params := mux.Vars(r) @@ -398,9 +400,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { - fields := xw.Audit() - fields["full_header"] = util.HeaderToMap(getFilteredHeader(r, nil)) - s.LogToken(xw, token, tokenErr) + s.LogToken(xw, authorization, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -1081,23 +1081,38 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } } -func (s *WebconfigServer) LogToken(xw *XResponseWriter, token string, tokenErr error) { +func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token string, tokenErr error) { fields := xw.Audit() fields["logger"] = "token" tfields := common.FilterLogFields(fields) - - if codec == nil { - codec, _ = security.NewAesCodec(s.Config) + var headerMap map[string]string + var isObfuscated bool + if itf, ok := tfields["header"]; ok { + headerMap = itf.(map[string]string) + if len(headerMap) > 0 { + ss := authorization + if len(ss) > authPrefixLength { + ss = authorization[:authPrefixLength] + "****" + isObfuscated = true + } + headerMap["Authorization"] = ss + } } - if codec == nil { - tfields["plaintoken"] = token - } else { - var encToken string - if encryptedB64, err := codec.Encrypt(token); err == nil { - encToken = encryptedB64 + if isObfuscated { + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken } - tfields["enctoken"] = encToken } log.WithFields(tfields).Debug(tokenErr) From bdd12bb835fca64b8a000f66963224f578db3ad2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Sep 2024 16:11:41 -0700 Subject: [PATCH 084/215] return 404 for NONE-REBOOT without subdocs --- http/factory_reset_upstream_test.go | 2 +- http/multipart.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index f6bbe0c..3240ea6 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -382,7 +382,7 @@ func TestFactoryResetUpstreamAddData(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusOK) + assert.Equal(t, res.StatusCode, http.StatusNotFound) rootDocument, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) diff --git a/http/multipart.go b/http/multipart.go index a8676a9..8b9414e 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -339,7 +339,9 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { - if !s.IsDbNotFound(err) { + if s.IsDbNotFound(err) { + return http.StatusNotFound, respHeader, nil, nil + } else { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } } From ec1f5b7bbf4b5a44d1e3c0362f2a1c13cc0cec52 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Sep 2024 16:11:41 -0700 Subject: [PATCH 085/215] return 404 for NONE-REBOOT without subdocs --- http/factory_reset_upstream_test.go | 2 +- http/multipart.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index f6bbe0c..3240ea6 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -382,7 +382,7 @@ func TestFactoryResetUpstreamAddData(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusOK) + assert.Equal(t, res.StatusCode, http.StatusNotFound) rootDocument, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) diff --git a/http/multipart.go b/http/multipart.go index a8676a9..8b9414e 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -339,7 +339,9 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { - if !s.IsDbNotFound(err) { + if s.IsDbNotFound(err) { + return http.StatusNotFound, respHeader, nil, nil + } else { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } } From d165d6df1f2912a413cc1f1265b2cb88bb96c9df Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 17 Sep 2024 13:36:18 -0700 Subject: [PATCH 086/215] Reduce unnecessary upstream calls when no meta headers change --- db/service.go | 8 +++++--- http/multipart.go | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/db/service.go b/db/service.go index bce94fa..617ffc8 100644 --- a/db/service.go +++ b/db/service.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( @@ -27,9 +27,9 @@ import ( "strings" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -183,7 +183,9 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + if !c.IsDbNotFound(err) { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + } } } for subdocId, subdocument := range document.Items() { diff --git a/http/multipart.go b/http/multipart.go index 8b9414e..d46f665 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,10 +24,10 @@ import ( "strconv" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -204,6 +204,8 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } respStatus = http.StatusOK + } else if len(document.RootVersion()) == 0 { + respStatus = http.StatusNotFound } // mget ==> no upstream From ad941aabdfab98f5bddf431b8549d0c826926c0b Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 18 Sep 2024 13:51:15 -0700 Subject: [PATCH 087/215] add a flag to control the enabling of sarama logger --- config/sample_webconfig.conf | 1 + main.go | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 4b67a55..d460eef 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -33,6 +33,7 @@ webconfig { log { level = "info" file = "/tmp/webconfig.log" + sarama_logger_enabled = false } metrics { diff --git a/main.go b/main.go index d5684ef..fe0cf0a 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package main import ( @@ -26,10 +26,10 @@ import ( "syscall" "github.com/IBM/sarama" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rdkcentral/webconfig/common" wchttp "github.com/rdkcentral/webconfig/http" "github.com/rdkcentral/webconfig/kafka" - "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" _ "go.uber.org/automaxprocs" "golang.org/x/sync/errgroup" @@ -93,7 +93,9 @@ func main() { log.SetLevel(logLevel) // setup sarama logger - sarama.Logger = log.StandardLogger() + if server.GetBoolean("webconfig.log.sarama_logger_enabled") { + sarama.Logger = log.StandardLogger() + } // setup router router := server.GetRouter(false) From 0c4814c089c3c605578051a1a908c9b22c87f7a1 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 20 Sep 2024 11:06:53 -0700 Subject: [PATCH 088/215] change frequently used headers from literals to consts --- common/const_var.go | 7 +- common/document.go | 4 +- common/writer.go | 8 +- http/document_handler.go | 6 +- http/document_handler_test.go | 56 ++++++------- http/factory_reset_upstream_test.go | 18 ++--- http/mqtt_connector_test.go | 4 +- http/multipart.go | 10 +-- http/multipart_test.go | 82 +++++++++---------- http/poke_handler_test.go | 20 ++--- http/refsubdocument_handler.go | 4 +- http/refsubdocument_handler_test.go | 13 +-- http/response.go | 14 ++-- http/rootdocument_handler_test.go | 14 ++-- http/supplementary_handler.go | 6 +- http/supported_groups_handler_test.go | 6 +- http/upstream_test.go | 110 +++++++++++++------------- http/validator.go | 12 +-- http/validator_test.go | 33 ++++---- util/multipart.go | 4 +- util/parser_test.go | 7 +- 21 files changed, 222 insertions(+), 216 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 513807e..8e56f84 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common const ( @@ -64,6 +64,9 @@ var ( ) const ( + HeaderContentType = "Content-Type" + HeaderApplicationJson = "application/json" + HeaderApplicationMsgpack = "application/msgpack" HeaderEtag = "Etag" HeaderIfNoneMatch = "If-None-Match" HeaderFirmwareVersion = "X-System-Firmware-Version" @@ -73,6 +76,7 @@ const ( HeaderProfileVersion = "X-System-Telemetry-Profile-Version" HeaderPartnerID = "X-System-PartnerID" HeaderAccountID = "X-System-AccountID" + HeaderProductClass = "X-System-Product-Class" HeaderUserAgent = "User-Agent" HeaderSchemaVersion = "X-System-Schema-Version" HeaderMetricsAgent = "X-Metrics-Agent" @@ -114,7 +118,6 @@ const ( SkipDbUpdate = "skip-db-update" ) - var ( SupportedPokeDocs = []string{"primary", "telemetry"} SupportedPokeRoutes = []string{"mqtt"} diff --git a/common/document.go b/common/document.go index d99f245..a3122c4 100644 --- a/common/document.go +++ b/common/document.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -194,7 +194,7 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { } header := make(http.Header) - header.Set("Content-type", MultipartContentType) + header.Set(HeaderContentType, MultipartContentType) header.Set("Etag", rootVersion) var traceId string diff --git a/common/writer.go b/common/writer.go index e8a16ee..8ea8504 100644 --- a/common/writer.go +++ b/common/writer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -38,9 +38,9 @@ func WriteMultipartBytes(mparts []Multipart) ([]byte, error) { writer.SetBoundary(Boundary) for _, m := range mparts { header := textproto.MIMEHeader{ - "Content-type": {"application/msgpack"}, - "Namespace": {m.Name}, - "Etag": {m.Version}, + HeaderContentType: {HeaderApplicationMsgpack}, + "Namespace": {m.Name}, + "Etag": {m.Version}, } p, err := writer.CreatePart(header) if err != nil { diff --git a/http/document_handler.go b/http/document_handler.go index 3f3e834..a662f7b 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,10 +24,10 @@ import ( "strings" "time" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) // TODO @@ -84,7 +84,7 @@ func (s *WebconfigServer) GetSubDocumentHandler(w http.ResponseWriter, r *http.R return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) writeStateHeaders(w, subdoc) w.WriteHeader(http.StatusOK) w.Write(subdoc.Payload()) diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 7d11287..5d274c1 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -48,7 +48,7 @@ func TestSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -58,7 +58,7 @@ func TestSubDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -73,7 +73,7 @@ func TestSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -82,7 +82,7 @@ func TestSubDocumentHandler(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -107,7 +107,7 @@ func TestDeleteDocumentHandler(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -117,7 +117,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -137,7 +137,7 @@ func TestDeleteDocumentHandler(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -146,7 +146,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -173,7 +173,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -182,7 +182,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -214,7 +214,7 @@ func TestPostWithDeviceId(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -226,7 +226,7 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -246,7 +246,7 @@ func TestPostWithDeviceId(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -257,7 +257,7 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -283,7 +283,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) gwrestoreBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header now := time.Now() @@ -298,7 +298,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) @@ -320,7 +320,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -339,7 +339,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -364,7 +364,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -373,7 +373,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -443,7 +443,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) gwrestoreBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare a version header now := time.Now() @@ -458,7 +458,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) @@ -480,7 +480,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -499,7 +499,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -524,7 +524,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -533,7 +533,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -597,7 +597,7 @@ func TestBadHeaderExpiryHandler(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // manage version and expiry headers now := time.Now() diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 3240ea6..5fc3c3a 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -100,7 +100,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -110,7 +110,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -126,7 +126,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -136,7 +136,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -196,7 +196,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -206,7 +206,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -222,7 +222,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -232,7 +232,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/mqtt_connector_test.go b/http/mqtt_connector_test.go index 48a2066..b6cdaae 100644 --- a/http/mqtt_connector_test.go +++ b/http/mqtt_connector_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -30,7 +30,7 @@ import ( func TestPayloadBuilder(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcData := util.Dict{ "device_id": "mac:044e5a22c9bf", diff --git a/http/multipart.go b/http/multipart.go index d46f665..fb4e8d0 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -166,7 +166,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -210,7 +210,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // mget ==> no upstream if userAgent == "mget" { - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -221,7 +221,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return respStatus, respHeader, respBytes, nil } @@ -230,7 +230,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // upstream handling // ============================= upstreamHeader := rHeader.Clone() - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) @@ -370,7 +370,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // upstream handling // ============================= upstreamHeader := rHeader.Clone() - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) diff --git a/http/multipart_test.go b/http/multipart_test.go index 2aed59a..4111af3 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -50,7 +50,7 @@ func TestMultipartConfigHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -60,7 +60,7 @@ func TestMultipartConfigHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -79,7 +79,7 @@ func TestMultipartConfigHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -89,7 +89,7 @@ func TestMultipartConfigHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -194,7 +194,7 @@ func TestCpeMiddleware(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -204,7 +204,7 @@ func TestCpeMiddleware(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -251,7 +251,7 @@ func TestVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -261,7 +261,7 @@ func TestVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -277,7 +277,7 @@ func TestVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -287,7 +287,7 @@ func TestVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -361,7 +361,7 @@ func TestVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -395,7 +395,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -405,7 +405,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -421,7 +421,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -431,7 +431,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -570,7 +570,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -580,7 +580,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -596,7 +596,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -606,7 +606,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -623,7 +623,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err := util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -687,7 +687,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -722,7 +722,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -747,7 +747,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -757,7 +757,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -774,7 +774,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -784,7 +784,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -868,7 +868,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -878,7 +878,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -895,7 +895,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -905,7 +905,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -922,7 +922,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -932,7 +932,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -949,7 +949,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -959,7 +959,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1127,7 +1127,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -1137,7 +1137,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1151,7 +1151,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1160,7 +1160,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1174,7 +1174,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) privatessidBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1183,7 +1183,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", privatessidUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index ba63157..2e2a862 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -119,7 +119,7 @@ func TestBuildMqttSendDocument(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -128,7 +128,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -151,7 +151,7 @@ func TestBuildMqttSendDocument(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -160,7 +160,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -196,7 +196,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -208,7 +208,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -227,7 +227,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 7 change the subdoc again ==== lanBytes2 := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes2)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -236,7 +236,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -262,7 +262,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go index 6413ee4..cd8743d 100644 --- a/http/refsubdocument_handler.go +++ b/http/refsubdocument_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -53,7 +53,7 @@ func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *htt return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) if refsubdoc.Version() != nil { w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) } diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index 49171b0..c0bd996 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,8 +24,9 @@ import ( "net/http" "testing" - "github.com/rdkcentral/webconfig/util" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -39,7 +40,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/reference/%v/document", refId) req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -48,7 +49,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -58,7 +59,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -67,7 +68,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/response.go b/http/response.go index 560a467..7cfeacb 100644 --- a/http/response.go +++ b/http/response.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -53,7 +53,7 @@ func WriteByMarshal(w http.ResponseWriter, status int, o interface{}) { LogError(w, common.NewError(err)) return } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) w.WriteHeader(status) w.Write(rbytes) } @@ -106,14 +106,14 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, SetAuditValue(w, "response", resp) rbytes = []byte(fmt.Sprintf(OkResponseTemplate, s, stateText, updatedTime)) } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) w.WriteHeader(http.StatusOK) w.Write(rbytes) } // this is used to return default tr-181 payload while the cpe is not in the db func WriteContentTypeAndResponse(w http.ResponseWriter, rbytes []byte, version string, contentType string) { - w.Header().Set("Content-type", contentType) + w.Header().Set(common.HeaderContentType, contentType) w.Header().Set(common.HeaderEtag, version) w.WriteHeader(http.StatusOK) w.Write(rbytes) @@ -135,7 +135,7 @@ func WriteErrorResponse(w http.ResponseWriter, status int, err error) { } func Error(w http.ResponseWriter, status int, err error) { - // calling WriteHeader() multiple times will cause errors in "content-type" + // calling WriteHeader() multiple times will cause errors in common.HeaderContentType // ==> errors like 'superfluous response.WriteHeader call' in stderr switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: @@ -147,14 +147,14 @@ func Error(w http.ResponseWriter, status int, err error) { func WriteResponseBytes(w http.ResponseWriter, rbytes []byte, statusCode int, vargs ...string) { if len(vargs) > 0 { - w.Header().Set("Content-type", vargs[0]) + w.Header().Set(common.HeaderContentType, vargs[0]) } w.WriteHeader(statusCode) w.Write(rbytes) } func WriteFactoryResetResponse(w http.ResponseWriter) { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.WriteHeader(http.StatusOK) w.Write([]byte{}) } diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index d329258..8c4e0f6 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -48,7 +48,7 @@ func TestRootDocumentHandler(t *testing.T) { // ==== step 0 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -94,7 +94,7 @@ func TestRootDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -103,7 +103,7 @@ func TestRootDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -121,7 +121,7 @@ func TestRootDocumentHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -130,7 +130,7 @@ func TestRootDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -242,7 +242,7 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { // ==== step 1 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 52e5660..82f0743 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,9 +23,9 @@ import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -101,7 +101,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } rootVersion := util.GetRandomRootVersion() - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, rootVersion) // help with unit tests diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 833d656..b0c7f52 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -106,7 +106,7 @@ func TestSupportedGroupsHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -116,7 +116,7 @@ func TestSupportedGroupsHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/upstream_test.go b/http/upstream_test.go index 127dcc1..ff1025e 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -64,7 +64,7 @@ func TestUpstream(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -103,7 +103,7 @@ func TestUpstream(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -142,7 +142,7 @@ func TestUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -151,7 +151,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -165,7 +165,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -174,7 +174,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -188,7 +188,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -197,7 +197,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -244,7 +244,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -272,7 +272,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -318,7 +318,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -360,7 +360,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -371,7 +371,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { respBytes := reqBytes // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, etag) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -411,7 +411,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -420,7 +420,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -434,7 +434,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -443,7 +443,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -457,7 +457,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -466,7 +466,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -507,7 +507,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -535,7 +535,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -578,7 +578,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -617,7 +617,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -656,7 +656,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -665,7 +665,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -679,7 +679,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -688,7 +688,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -702,7 +702,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -711,7 +711,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -758,7 +758,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -787,7 +787,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -840,7 +840,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -889,7 +889,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -916,7 +916,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) @@ -956,7 +956,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -965,7 +965,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -979,7 +979,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -988,7 +988,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1002,7 +1002,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1011,7 +1011,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1058,7 +1058,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1087,7 +1087,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1139,7 +1139,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1182,7 +1182,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -1209,7 +1209,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) @@ -1249,7 +1249,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1258,7 +1258,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -1272,7 +1272,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1281,7 +1281,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1295,7 +1295,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1304,7 +1304,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1347,7 +1347,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1376,7 +1376,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1427,7 +1427,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/validator.go b/http/validator.go index d85921e..bde64c6 100644 --- a/http/validator.go +++ b/http/validator.go @@ -14,16 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -56,8 +56,8 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") @@ -93,8 +93,8 @@ func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") diff --git a/http/validator_test.go b/http/validator_test.go index 8d36dcf..aafc71c 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -27,6 +27,7 @@ import ( "strings" "testing" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -46,7 +47,7 @@ func TestValidatorDisabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -55,7 +56,7 @@ func TestValidatorDisabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -86,7 +87,7 @@ func TestValidatorDisabled(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -95,7 +96,7 @@ func TestValidatorDisabled(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -139,7 +140,7 @@ func TestValidatorEnabled(t *testing.T) { server.SetValidateMacEnabled(true) url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -149,7 +150,7 @@ func TestValidatorEnabled(t *testing.T) { // post without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -159,7 +160,7 @@ func TestValidatorEnabled(t *testing.T) { // get with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -169,7 +170,7 @@ func TestValidatorEnabled(t *testing.T) { // get without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -211,7 +212,7 @@ func TestValidatorEnabled(t *testing.T) { // delete with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -221,7 +222,7 @@ func TestValidatorEnabled(t *testing.T) { // delete without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -230,7 +231,7 @@ func TestValidatorEnabled(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -272,7 +273,7 @@ func TestValidatorWithLowerCase(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -281,7 +282,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -301,7 +302,7 @@ func TestValidatorWithLowerCase(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -310,7 +311,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/util/multipart.go b/util/multipart.go index 70c4683..ee91abf 100644 --- a/util/multipart.go +++ b/util/multipart.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -49,7 +49,7 @@ func ParseMultipartAsList(header http.Header, bbytes []byte) ([]common.Multipart return mparts, nil } - mediaType, params, err := mime.ParseMediaType(header.Get("Content-Type")) + mediaType, params, err := mime.ParseMediaType(header.Get(common.HeaderContentType)) if err != nil { return nil, common.NewError(err) } diff --git a/util/parser_test.go b/util/parser_test.go index 1835dae..23fbc65 100644 --- a/util/parser_test.go +++ b/util/parser_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -22,13 +22,14 @@ import ( "net/http" "testing" + "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) func TestParseWebconfigResponseMessage(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("Content-length", "120") srcData := Dict{ @@ -55,7 +56,7 @@ func TestParseWebconfigResponseMessageMultipleHeaders(t *testing.T) { srcHeader.Add("X-Color", "color:red") srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") srcHeader.Add("X-Color", "color:orange") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("X-Color", "color:yellow:green") srcHeader.Add("Content-length", "120") srcHeader.Add("X-Color", "color:blue:indigo:violet") From 42329905ceca761d829e5d6e2b3992505e762991 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 30 Sep 2024 22:53:48 -0700 Subject: [PATCH 089/215] ix a bug that status=0 was logged when the last subdoc was deleted --- http/document_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/http/document_handler.go b/http/document_handler.go index a662f7b..e07890f 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -245,6 +245,7 @@ func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *htt if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } From b766d8cfa9411ea3a01bbb3e6f633f741b022a64 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 3 Oct 2024 21:43:34 -0700 Subject: [PATCH 090/215] log kafka async producer errors --- common/metrics.go | 19 ++++++++++++- http/webconfig_server.go | 60 ++++++++++++++++++++++++++++++++++++---- main.go | 9 ++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/common/metrics.go b/common/metrics.go index ad34337..18b11f2 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -74,6 +74,7 @@ type AppMetrics struct { indeploymentDecCount *prometheus.CounterVec failureIncCount *prometheus.CounterVec failureDecCount *prometheus.CounterVec + kafkaProducerErrCount *prometheus.CounterVec watchedCpes []string logrusLevel log.Level } @@ -298,6 +299,13 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet }, stateMetricsLabels, ), + kafkaProducerErrCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: appName + "_kafka_producer_err_count", + Help: "A counter for the number of errors by kafka producer.", + }, + []string{"topic", "partition"}, + ), watchedCpes: watchedCpes, logrusLevel: logrusLevel, } @@ -326,6 +334,7 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet appMetrics.indeploymentDecCount, appMetrics.failureIncCount, appMetrics.failureDecCount, + appMetrics.kafkaProducerErrCount, ) return appMetrics } @@ -534,6 +543,14 @@ func (m *AppMetrics) CountKafkaEvents(eventName string, status string, partition m.eventCounter.With(labels).Inc() } +func (m *AppMetrics) ObserveKafkaProducerErr(topic string, partition int32) { + labels := prometheus.Labels{ + "topic": topic, + "partition": strconv.Itoa(int(partition)), + } + m.kafkaProducerErrCount.With(labels).Inc() +} + func (m *AppMetrics) GetStateCounter(labels prometheus.Labels) (*StateCounter, error) { // REMINDER if a label is defined with 2 dimensions, then it must be referred // with 2 dimensions. Aggregation happens at prometheus level diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 05d4ea2..c51b811 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "context" "crypto/tls" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -36,15 +37,15 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/IBM/sarama" + "github.com/go-akka/configuration" + "github.com/google/uuid" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/google/uuid" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -276,6 +277,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") saramaConfig := sarama.NewConfig() + saramaConfig.Producer.Return.Errors = true kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) if err != nil { panic(err) @@ -1070,7 +1072,7 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } outMessage := &sarama.ProducerMessage{ Topic: s.KafkaProducerTopic(), - Key: sarama.ByteEncoder(mac), + Key: sarama.ByteEncoder(strings.ToLower(mac)), Value: sarama.ByteEncoder(bbytes), } s.Input() <- outMessage @@ -1117,3 +1119,51 @@ func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token str log.WithFields(tfields).Debug(tokenErr) } + +func (s *WebconfigServer) HandleKafkaProducerResults() { + if s.AsyncProducer == nil { + return + } + + for { + select { + case success := <-s.Successes(): + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = success.Topic + fields["output_partition"] = success.Partition + fields["output_offset"] = success.Offset + log.WithFields(fields).Debug("sent") + case pErr := <-s.Errors(): + if m := s.Metrics(); m != nil { + m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) + } + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = pErr.Msg.Topic + fields["output_partition"] = pErr.Msg.Partition + kbytes, err := pErr.Msg.Key.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + fields["output_key"] = string(kbytes) + } + + vbytes, err := pErr.Msg.Value.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + var itf interface{} + err1 := json.Unmarshal(vbytes, &itf) + if err1 != nil { + log.WithFields(fields).Error(common.NewError(err1)) + fields["output_body_text"] = base64.StdEncoding.EncodeToString(vbytes) + } else { + fields["output_body"] = itf + } + } + + log.WithFields(fields).Error(pErr.Err) + } + } +} diff --git a/main.go b/main.go index fe0cf0a..8a47cb9 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/IBM/sarama" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -105,6 +106,9 @@ func main() { router.Handle("/metrics", promhttp.Handler()) metrics = common.NewMetrics(sc.Config) server.SetMetrics(metrics) + if server.KafkaProducerEnabled() { + go server.HandleKafkaProducerResults() + } handler := metrics.WebMetrics(router) server.Handler = handler } else { @@ -125,6 +129,11 @@ func main() { func() error { <-gCtx.Done() fmt.Printf("HTTP server shutdown NOW !!\n") + if server.KafkaProducerEnabled() { + if err := server.AsyncProducer.Close(); err != nil { + fmt.Fprintf(os.Stderr, "%v AsyncProducer.Close() err=%v\n", time.Now().Format(common.LoggingTimeFormat), err) + } + } return server.Shutdown(context.Background()) }, ) From da5879cb2cdb2cef46c898d0726c08168ce2e784 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 30 Sep 2024 22:53:48 -0700 Subject: [PATCH 091/215] ix a bug that status=0 was logged when the last subdoc was deleted --- http/document_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/http/document_handler.go b/http/document_handler.go index 3f3e834..957bbd0 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -245,6 +245,7 @@ func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *htt if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } From 217d7cfaaa1893845fc739f9e5df351a271acd76 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 3 Oct 2024 21:43:34 -0700 Subject: [PATCH 092/215] log kafka async producer errors --- common/metrics.go | 19 ++++++++++++- http/webconfig_server.go | 60 ++++++++++++++++++++++++++++++++++++---- main.go | 9 ++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/common/metrics.go b/common/metrics.go index ad34337..18b11f2 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -74,6 +74,7 @@ type AppMetrics struct { indeploymentDecCount *prometheus.CounterVec failureIncCount *prometheus.CounterVec failureDecCount *prometheus.CounterVec + kafkaProducerErrCount *prometheus.CounterVec watchedCpes []string logrusLevel log.Level } @@ -298,6 +299,13 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet }, stateMetricsLabels, ), + kafkaProducerErrCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: appName + "_kafka_producer_err_count", + Help: "A counter for the number of errors by kafka producer.", + }, + []string{"topic", "partition"}, + ), watchedCpes: watchedCpes, logrusLevel: logrusLevel, } @@ -326,6 +334,7 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet appMetrics.indeploymentDecCount, appMetrics.failureIncCount, appMetrics.failureDecCount, + appMetrics.kafkaProducerErrCount, ) return appMetrics } @@ -534,6 +543,14 @@ func (m *AppMetrics) CountKafkaEvents(eventName string, status string, partition m.eventCounter.With(labels).Inc() } +func (m *AppMetrics) ObserveKafkaProducerErr(topic string, partition int32) { + labels := prometheus.Labels{ + "topic": topic, + "partition": strconv.Itoa(int(partition)), + } + m.kafkaProducerErrCount.With(labels).Inc() +} + func (m *AppMetrics) GetStateCounter(labels prometheus.Labels) (*StateCounter, error) { // REMINDER if a label is defined with 2 dimensions, then it must be referred // with 2 dimensions. Aggregation happens at prometheus level diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 05d4ea2..c51b811 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "context" "crypto/tls" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -36,15 +37,15 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/IBM/sarama" + "github.com/go-akka/configuration" + "github.com/google/uuid" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/google/uuid" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -276,6 +277,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") saramaConfig := sarama.NewConfig() + saramaConfig.Producer.Return.Errors = true kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) if err != nil { panic(err) @@ -1070,7 +1072,7 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } outMessage := &sarama.ProducerMessage{ Topic: s.KafkaProducerTopic(), - Key: sarama.ByteEncoder(mac), + Key: sarama.ByteEncoder(strings.ToLower(mac)), Value: sarama.ByteEncoder(bbytes), } s.Input() <- outMessage @@ -1117,3 +1119,51 @@ func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token str log.WithFields(tfields).Debug(tokenErr) } + +func (s *WebconfigServer) HandleKafkaProducerResults() { + if s.AsyncProducer == nil { + return + } + + for { + select { + case success := <-s.Successes(): + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = success.Topic + fields["output_partition"] = success.Partition + fields["output_offset"] = success.Offset + log.WithFields(fields).Debug("sent") + case pErr := <-s.Errors(): + if m := s.Metrics(); m != nil { + m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) + } + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = pErr.Msg.Topic + fields["output_partition"] = pErr.Msg.Partition + kbytes, err := pErr.Msg.Key.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + fields["output_key"] = string(kbytes) + } + + vbytes, err := pErr.Msg.Value.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + var itf interface{} + err1 := json.Unmarshal(vbytes, &itf) + if err1 != nil { + log.WithFields(fields).Error(common.NewError(err1)) + fields["output_body_text"] = base64.StdEncoding.EncodeToString(vbytes) + } else { + fields["output_body"] = itf + } + } + + log.WithFields(fields).Error(pErr.Err) + } + } +} diff --git a/main.go b/main.go index fe0cf0a..8a47cb9 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/IBM/sarama" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -105,6 +106,9 @@ func main() { router.Handle("/metrics", promhttp.Handler()) metrics = common.NewMetrics(sc.Config) server.SetMetrics(metrics) + if server.KafkaProducerEnabled() { + go server.HandleKafkaProducerResults() + } handler := metrics.WebMetrics(router) server.Handler = handler } else { @@ -125,6 +129,11 @@ func main() { func() error { <-gCtx.Done() fmt.Printf("HTTP server shutdown NOW !!\n") + if server.KafkaProducerEnabled() { + if err := server.AsyncProducer.Close(); err != nil { + fmt.Fprintf(os.Stderr, "%v AsyncProducer.Close() err=%v\n", time.Now().Format(common.LoggingTimeFormat), err) + } + } return server.Shutdown(context.Background()) }, ) From 7ff2a1f26b7a7e8f4ae13443871bae461a2e3b37 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 8 Oct 2024 23:12:08 -0700 Subject: [PATCH 093/215] handle 202 returned by webpa poke --- http/http_client.go | 52 +++++++++++++++++++++++++-------------- http/poke_handler_test.go | 30 ++++++++++++++++++++++ http/response.go | 3 +++ http/webpa_connector.go | 22 ++++++++--------- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index aad3085..f67598c 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -52,7 +52,7 @@ type ErrorResponse struct { Message string `json:"message"` } -type StatusHandlerFunc func([]byte) ([]byte, http.Header, error, bool) +type StatusHandlerFunc func([]byte) ([]byte, http.Header, bool, error) type HttpClient struct { *http.Client @@ -107,7 +107,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } } -func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, error, bool) { +func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields) // verify a response is received @@ -121,11 +121,11 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] case "DELETE": req, err = http.NewRequest(method, url, nil) default: - return nil, nil, common.NewError(fmt.Errorf("method=%v", method)), false + return nil, nil, false, common.NewError(fmt.Errorf("method=%v", method)) } if err != nil { - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if header == nil { @@ -209,8 +209,8 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] res, err := c.Client.Do(req) // err should be *url.Error - tdiff := time.Now().Sub(startTime) - duration := tdiff.Nanoseconds() / 1000000 + tdiff := time.Since(startTime) + duration := tdiff.Milliseconds() fields[fmt.Sprintf("%v_duration", loggerName)] = duration delete(fields, bodyKey) @@ -236,25 +236,25 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] Message: ue.Error(), StatusCode: http.StatusGatewayTimeout, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if errors.Is(innerErr, io.EOF) { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusBadGateway, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if _, ok := innerErr.(*net.OpError); ok { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusServiceUnavailable, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } // Unknown err still appear as 500 } - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if res.Body != nil { defer res.Body.Close() @@ -276,7 +276,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] if userAgent != "mget" { log.WithFields(fields).Info(endMessage) } - return nil, nil, common.NewError(err), false + return nil, nil, true, common.NewError(err) } rbody := string(rbytes) @@ -331,11 +331,27 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] switch res.StatusCode { case http.StatusForbidden, http.StatusBadRequest, http.StatusNotFound: - return rbytes, nil, common.NewError(err), false + return rbytes, nil, false, common.NewError(err) + } + return rbytes, nil, true, common.NewError(err) + } else if res.StatusCode > 200 { + var pokeResponse PokeResponse + var message string + if err := json.Unmarshal(rbytes, &pokeResponse); err == nil { + if len(pokeResponse.Parameters) > 0 { + message = pokeResponse.Parameters[0].Message + } + } + if len(message) == 0 { + message = http.StatusText(res.StatusCode) + } + rherr := common.RemoteHttpError{ + Message: message, + StatusCode: res.StatusCode, } - return rbytes, nil, common.NewError(err), true + return rbytes, nil, false, common.NewError(rherr) } - return rbytes, res.Header, nil, false + return rbytes, res.Header, false, nil } func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { @@ -352,7 +368,7 @@ func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Heade if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, err, cont = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 2e2a862..5a599ea 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -37,6 +37,7 @@ import ( var ( mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) + mockWebpaPoke202Response = []byte(`{"parameters":[{"message":"Previous request is in progress","name":"Device.X_RDK_WebConfig.ForceSync"}],"statusCode":202}`) ) func TestPokeHandler(t *testing.T) { @@ -335,3 +336,32 @@ func TestPokeHandlerWebpa403(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa202(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write(mockWebpaPoke202Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusAccepted) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/response.go b/http/response.go index 7cfeacb..7107bfa 100644 --- a/http/response.go +++ b/http/response.go @@ -140,6 +140,9 @@ func Error(w http.ResponseWriter, status int, err error) { switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: w.WriteHeader(status) + case http.StatusAccepted: + SetAuditValue(w, "response", err) + WriteByMarshal(w, status, err) default: WriteErrorResponse(w, status, err) } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index f6aadbe..52378d9 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,9 +25,9 @@ import ( "net/http" "time" - "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) @@ -72,7 +72,7 @@ type WebpaConnector struct { apiVersion string } -func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func syncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -82,16 +82,16 @@ func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } -func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func asyncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -101,13 +101,13 @@ func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), true + return rbytes, nil, true, common.NewError(rerr) } func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *WebpaConnector { @@ -231,7 +231,7 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field header.Set(common.HeaderTracestate, outTracestate) method := "PATCH" - _, _, err, cont := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -269,7 +269,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header ht if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { msg := fmt.Sprintf("finished success after 1 retry") if i > 1 { @@ -297,7 +297,7 @@ func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header htt if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, err, cont = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError From 261b2e5135c2dafb1ba7bf2183578f37a174eb3b Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 10 Oct 2024 00:15:59 -0700 Subject: [PATCH 094/215] update states only when the version in the notification matches the version in db --- db/cassandra/state_update_test.go | 34 +++++++++++------------ db/service.go | 45 +++++++++++++++++++------------ http/upstream_test.go | 20 +++++++------- kafka/consumer.go | 41 ++++++++++++++-------------- 4 files changed, 76 insertions(+), 64 deletions(-) diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 2d9bbb0..e9bdae6 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -53,14 +53,14 @@ func TestStateUpdate1(t *testing.T) { assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -68,14 +68,14 @@ func TestStateUpdate1(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success - template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:11.959437", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -106,14 +106,14 @@ func TestStateUpdate2(t *testing.T) { assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -121,14 +121,14 @@ func TestStateUpdate2(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success by http 304 - template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "2023-05-05 07:42:50.395876"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "%v"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, ok) + assert.Assert(t, len(updatedSubdocIds) > 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/service.go b/db/service.go index 617ffc8..2f4cee7 100644 --- a/db/service.go +++ b/db/service.go @@ -347,14 +347,14 @@ func HashRootVersion(itf interface{}) string { return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) (bool, error) { - var updatedBy304 bool +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) ([]string, error) { + updatedSubdocIds := []string{} // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return updatedBy304, nil + return updatedSubdocIds, nil } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) // set metrics labels metricsAgent := "default" @@ -363,7 +363,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } labels["client"] = metricsAgent @@ -372,14 +372,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return updatedBy304, nil + return updatedSubdocIds, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } newState := common.Deployed @@ -388,21 +388,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { - updatedBy304 = true + updatedSubdocIds = append(updatedSubdocIds, groupId) newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } } - return updatedBy304, nil + return updatedSubdocIds, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return updatedBy304, common.NewError(fmt.Errorf("ill-formatted event")) + return updatedSubdocIds, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -415,23 +415,34 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return updatedBy304, nil + return updatedSubdocIds, nil } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } var oldState int if subdoc.State() != nil { oldState = *subdoc.State() + if oldState == common.Deployed { + log.WithFields(fields).Warn("state=1 already") + return updatedSubdocIds, nil + } if oldState < common.Deployed || oldState > common.Failure { err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) + } + } + + if subdoc.Version() != nil && m.Version != nil { + if *subdoc.Version() != *m.Version { + log.WithFields(fields).Warnf("skip update dbversion=%v, m.version=%v", *subdoc.Version(), *m.Version) + return updatedSubdocIds, nil } } @@ -441,7 +452,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } @@ -454,9 +465,9 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } - return updatedBy304, nil + return updatedSubdocIds, nil } func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { diff --git a/http/upstream_test.go b/http/upstream_test.go index ff1025e..7dfb386 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -263,9 +263,9 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -526,9 +526,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -777,9 +777,9 @@ func TestUpstreamUpdatedTime(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -1077,9 +1077,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -1366,9 +1366,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== diff --git a/kafka/consumer.go b/kafka/consumer.go index 85b6fd9..40e8ec1 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package kafka import ( @@ -76,31 +76,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, bool, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, []string, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, false, common.NewError(err) + return nil, nil, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, false, common.NewError(err) + return nil, nil, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, false, nil + return &m, nil, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - updatedBy304, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, updatedBy304, common.NewError(err) + return &m, updatedSubdocIds, common.NewError(err) } - return &m, updatedBy304, nil + return &m, updatedSubdocIds, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -204,7 +204,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage - var updatedBy304 bool + var updatedSubdocIds []string eventName, rptHeaderValue := getEventName(message) switch eventName { @@ -214,10 +214,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, updatedBy304, err = c.handleNotification(bbytes, fields) + m, updatedSubdocIds, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, updatedBy304, err = c.handleNotification(message.Value, fields) + m, updatedSubdocIds, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -259,18 +259,19 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram if c.KafkaProducerEnabled() && m != nil { c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { - if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && len(updatedSubdocIds) > 0 { // build a root/success message - namespace := "root" applicationStatus := "success" - em := &common.EventMessage{ - Namespace: &namespace, - ApplicationStatus: &applicationStatus, - DeviceId: m.DeviceId, - TransactionUuid: m.TransactionUuid, - Version: m.Version, + for _, subdocId := range updatedSubdocIds { + em := &common.EventMessage{ + Namespace: &subdocId, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) } - c.ForwardKafkaMessage(message.Key, em, fields) } } } From 9108c39455f7867c13709d772a418a8f49defc58 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 11 Oct 2024 21:50:17 -0700 Subject: [PATCH 095/215] change the poke 204 condition to all deployed --- common/document.go | 2 +- http/poke_handler_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/document.go b/common/document.go index a3122c4..2fdb88d 100644 --- a/common/document.go +++ b/common/document.go @@ -120,7 +120,7 @@ func (d *Document) FilterForMqttSend() *Document { for subdocId, subDocument := range d.docmap { if subDocument.State() != nil { state := *subDocument.State() - if state == PendingDownload || state == Failure { + if state > Deployed { newdoc.SetSubDocument(subdocId, &subDocument) } } diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 5a599ea..9a9d3a2 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -223,7 +223,7 @@ func TestBuildMqttSendDocument(t *testing.T) { fields = make(log.Fields) document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) // ==== step 7 change the subdoc again ==== lanBytes2 := util.RandomBytes(100, 150) @@ -251,7 +251,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 8 check the document length === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 1) + assert.Equal(t, document.Length(), 2) // ==== step 9 send a document through mqtt ==== req, err = http.NewRequest("POST", mqttUrl, nil) @@ -276,7 +276,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 10 check the length again === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) } func TestPokeHandlerInvalidMac(t *testing.T) { From a0a7c0106a97a892a446f2b2428583beec3b722d Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 15 Oct 2024 23:31:00 -0700 Subject: [PATCH 096/215] fix a bug that version matched subdocs are not backfilled from upstream --- db/service.go | 4 +- http/upstream_test.go | 221 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 2 deletions(-) diff --git a/db/service.go b/db/service.go index 2f4cee7..20dfbad 100644 --- a/db/service.go +++ b/db/service.go @@ -484,13 +484,13 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old if oldVersion, ok := versionMap[subdocId]; ok { if newSubdoc.Version() != nil { - if oldVersion == *newSubdoc.Version() { + if oldVersion == *newSubdoc.Version() && oldSubdoc != nil { return nil } } } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) newSubdoc.SetUpdatedTime(&updatedTime) newState := common.InDeployment diff --git a/http/upstream_test.go b/http/upstream_test.go index 7dfb386..f8c1bf3 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strconv" "testing" @@ -1440,3 +1441,223 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) } } + +func TestUpstreamBackfill(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidBytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidBytes, + "lan": lanBytes, + "wan": wanBytes, + } + + var mockedRespBytes []byte + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mockedRespBytes == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + for k := range r.Header { + if k == "Content-Length" { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + w.Write(mockedRespBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderIfNoneMatch, "NONE-POST") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + mockedRespBytes = slices.Clone(rbytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 handle a condition when no subdocs in db ==== + err = server.DeleteDocument(cpeMac) + assert.NilError(t, err) + err = server.DeleteRootDocument(cpeMac) + assert.NilError(t, err) + + // ==== step 9 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotModified) + res.Body.Close() + + // ==== step 10 verify all states, versions and payloads of all subdocs are backfilled ==== + document, err := server.GetDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, document.Length(), 3) + for _, sd := range document.Items() { + assert.Assert(t, sd.GetState() > 0) + assert.Assert(t, len(sd.GetVersion()) > 0) + assert.Assert(t, len(sd.Payload()) > 0) + } +} From 5f819c06bdc48eeb2953186b9a3cc9947e37afe7 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 10 Oct 2024 00:15:59 -0700 Subject: [PATCH 097/215] update states only when the version in the notification matches the version in db --- db/cassandra/state_update_test.go | 34 +++++++++++------------ db/service.go | 45 +++++++++++++++++++------------ http/upstream_test.go | 20 +++++++------- kafka/consumer.go | 41 ++++++++++++++-------------- 4 files changed, 76 insertions(+), 64 deletions(-) diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 2d9bbb0..e9bdae6 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -53,14 +53,14 @@ func TestStateUpdate1(t *testing.T) { assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -68,14 +68,14 @@ func TestStateUpdate1(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success - template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:11.959437", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -106,14 +106,14 @@ func TestStateUpdate2(t *testing.T) { assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -121,14 +121,14 @@ func TestStateUpdate2(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success by http 304 - template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "2023-05-05 07:42:50.395876"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "%v"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, ok) + assert.Assert(t, len(updatedSubdocIds) > 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/service.go b/db/service.go index 617ffc8..2f4cee7 100644 --- a/db/service.go +++ b/db/service.go @@ -347,14 +347,14 @@ func HashRootVersion(itf interface{}) string { return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) (bool, error) { - var updatedBy304 bool +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) ([]string, error) { + updatedSubdocIds := []string{} // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return updatedBy304, nil + return updatedSubdocIds, nil } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) // set metrics labels metricsAgent := "default" @@ -363,7 +363,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } labels["client"] = metricsAgent @@ -372,14 +372,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return updatedBy304, nil + return updatedSubdocIds, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } newState := common.Deployed @@ -388,21 +388,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { - updatedBy304 = true + updatedSubdocIds = append(updatedSubdocIds, groupId) newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } } - return updatedBy304, nil + return updatedSubdocIds, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return updatedBy304, common.NewError(fmt.Errorf("ill-formatted event")) + return updatedSubdocIds, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -415,23 +415,34 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return updatedBy304, nil + return updatedSubdocIds, nil } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } var oldState int if subdoc.State() != nil { oldState = *subdoc.State() + if oldState == common.Deployed { + log.WithFields(fields).Warn("state=1 already") + return updatedSubdocIds, nil + } if oldState < common.Deployed || oldState > common.Failure { err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) + } + } + + if subdoc.Version() != nil && m.Version != nil { + if *subdoc.Version() != *m.Version { + log.WithFields(fields).Warnf("skip update dbversion=%v, m.version=%v", *subdoc.Version(), *m.Version) + return updatedSubdocIds, nil } } @@ -441,7 +452,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } @@ -454,9 +465,9 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } - return updatedBy304, nil + return updatedSubdocIds, nil } func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { diff --git a/http/upstream_test.go b/http/upstream_test.go index 127dcc1..4041dfe 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -263,9 +263,9 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -526,9 +526,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -777,9 +777,9 @@ func TestUpstreamUpdatedTime(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -1077,9 +1077,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -1366,9 +1366,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== diff --git a/kafka/consumer.go b/kafka/consumer.go index 85b6fd9..40e8ec1 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package kafka import ( @@ -76,31 +76,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, bool, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, []string, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, false, common.NewError(err) + return nil, nil, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, false, common.NewError(err) + return nil, nil, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, false, nil + return &m, nil, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - updatedBy304, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, updatedBy304, common.NewError(err) + return &m, updatedSubdocIds, common.NewError(err) } - return &m, updatedBy304, nil + return &m, updatedSubdocIds, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -204,7 +204,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage - var updatedBy304 bool + var updatedSubdocIds []string eventName, rptHeaderValue := getEventName(message) switch eventName { @@ -214,10 +214,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, updatedBy304, err = c.handleNotification(bbytes, fields) + m, updatedSubdocIds, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, updatedBy304, err = c.handleNotification(message.Value, fields) + m, updatedSubdocIds, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -259,18 +259,19 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram if c.KafkaProducerEnabled() && m != nil { c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { - if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && len(updatedSubdocIds) > 0 { // build a root/success message - namespace := "root" applicationStatus := "success" - em := &common.EventMessage{ - Namespace: &namespace, - ApplicationStatus: &applicationStatus, - DeviceId: m.DeviceId, - TransactionUuid: m.TransactionUuid, - Version: m.Version, + for _, subdocId := range updatedSubdocIds { + em := &common.EventMessage{ + Namespace: &subdocId, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) } - c.ForwardKafkaMessage(message.Key, em, fields) } } } From b83d57addf32b958c0d0c4cf6c70ef67e2566fa0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 11 Oct 2024 21:50:17 -0700 Subject: [PATCH 098/215] change the poke 204 condition to all deployed --- common/document.go | 2 +- http/poke_handler_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/document.go b/common/document.go index d99f245..edd9d57 100644 --- a/common/document.go +++ b/common/document.go @@ -120,7 +120,7 @@ func (d *Document) FilterForMqttSend() *Document { for subdocId, subDocument := range d.docmap { if subDocument.State() != nil { state := *subDocument.State() - if state == PendingDownload || state == Failure { + if state > Deployed { newdoc.SetSubDocument(subdocId, &subDocument) } } diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index ba63157..f354dcd 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -222,7 +222,7 @@ func TestBuildMqttSendDocument(t *testing.T) { fields = make(log.Fields) document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) // ==== step 7 change the subdoc again ==== lanBytes2 := util.RandomBytes(100, 150) @@ -250,7 +250,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 8 check the document length === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 1) + assert.Equal(t, document.Length(), 2) // ==== step 9 send a document through mqtt ==== req, err = http.NewRequest("POST", mqttUrl, nil) @@ -275,7 +275,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 10 check the length again === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) } func TestPokeHandlerInvalidMac(t *testing.T) { From 872d747b3836f7614fe0d10efab85eaf506c522b Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 15 Oct 2024 23:31:00 -0700 Subject: [PATCH 099/215] fix a bug that version matched subdocs are not backfilled from upstream --- db/service.go | 4 +- http/upstream_test.go | 221 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 2 deletions(-) diff --git a/db/service.go b/db/service.go index 2f4cee7..20dfbad 100644 --- a/db/service.go +++ b/db/service.go @@ -484,13 +484,13 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old if oldVersion, ok := versionMap[subdocId]; ok { if newSubdoc.Version() != nil { - if oldVersion == *newSubdoc.Version() { + if oldVersion == *newSubdoc.Version() && oldSubdoc != nil { return nil } } } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) newSubdoc.SetUpdatedTime(&updatedTime) newState := common.InDeployment diff --git a/http/upstream_test.go b/http/upstream_test.go index 4041dfe..90fcc94 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strconv" "testing" @@ -1440,3 +1441,223 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) } } + +func TestUpstreamBackfill(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidBytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidBytes, + "lan": lanBytes, + "wan": wanBytes, + } + + var mockedRespBytes []byte + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mockedRespBytes == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + for k := range r.Header { + if k == "Content-Length" { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + w.Write(mockedRespBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderIfNoneMatch, "NONE-POST") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + mockedRespBytes = slices.Clone(rbytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 handle a condition when no subdocs in db ==== + err = server.DeleteDocument(cpeMac) + assert.NilError(t, err) + err = server.DeleteRootDocument(cpeMac) + assert.NilError(t, err) + + // ==== step 9 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotModified) + res.Body.Close() + + // ==== step 10 verify all states, versions and payloads of all subdocs are backfilled ==== + document, err := server.GetDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, document.Length(), 3) + for _, sd := range document.Items() { + assert.Assert(t, sd.GetState() > 0) + assert.Assert(t, len(sd.GetVersion()) > 0) + assert.Assert(t, len(sd.Payload()) > 0) + } +} From 132e15a2c63fb120c7ee4f3643acb603c012c38f Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 20 Sep 2024 11:06:53 -0700 Subject: [PATCH 100/215] change frequently used headers from literals to consts --- common/const_var.go | 7 +- common/document.go | 4 +- common/writer.go | 8 +- http/document_handler.go | 6 +- http/document_handler_test.go | 56 ++++++------- http/factory_reset_upstream_test.go | 18 ++--- http/mqtt_connector_test.go | 4 +- http/multipart.go | 10 +-- http/multipart_test.go | 82 +++++++++---------- http/poke_handler_test.go | 20 ++--- http/refsubdocument_handler.go | 4 +- http/refsubdocument_handler_test.go | 13 +-- http/response.go | 14 ++-- http/rootdocument_handler_test.go | 14 ++-- http/supplementary_handler.go | 6 +- http/supported_groups_handler_test.go | 6 +- http/upstream_test.go | 110 +++++++++++++------------- http/validator.go | 12 +-- http/validator_test.go | 33 ++++---- util/multipart.go | 4 +- util/parser_test.go | 7 +- 21 files changed, 222 insertions(+), 216 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 513807e..8e56f84 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common const ( @@ -64,6 +64,9 @@ var ( ) const ( + HeaderContentType = "Content-Type" + HeaderApplicationJson = "application/json" + HeaderApplicationMsgpack = "application/msgpack" HeaderEtag = "Etag" HeaderIfNoneMatch = "If-None-Match" HeaderFirmwareVersion = "X-System-Firmware-Version" @@ -73,6 +76,7 @@ const ( HeaderProfileVersion = "X-System-Telemetry-Profile-Version" HeaderPartnerID = "X-System-PartnerID" HeaderAccountID = "X-System-AccountID" + HeaderProductClass = "X-System-Product-Class" HeaderUserAgent = "User-Agent" HeaderSchemaVersion = "X-System-Schema-Version" HeaderMetricsAgent = "X-Metrics-Agent" @@ -114,7 +118,6 @@ const ( SkipDbUpdate = "skip-db-update" ) - var ( SupportedPokeDocs = []string{"primary", "telemetry"} SupportedPokeRoutes = []string{"mqtt"} diff --git a/common/document.go b/common/document.go index edd9d57..2fdb88d 100644 --- a/common/document.go +++ b/common/document.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -194,7 +194,7 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { } header := make(http.Header) - header.Set("Content-type", MultipartContentType) + header.Set(HeaderContentType, MultipartContentType) header.Set("Etag", rootVersion) var traceId string diff --git a/common/writer.go b/common/writer.go index e8a16ee..8ea8504 100644 --- a/common/writer.go +++ b/common/writer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -38,9 +38,9 @@ func WriteMultipartBytes(mparts []Multipart) ([]byte, error) { writer.SetBoundary(Boundary) for _, m := range mparts { header := textproto.MIMEHeader{ - "Content-type": {"application/msgpack"}, - "Namespace": {m.Name}, - "Etag": {m.Version}, + HeaderContentType: {HeaderApplicationMsgpack}, + "Namespace": {m.Name}, + "Etag": {m.Version}, } p, err := writer.CreatePart(header) if err != nil { diff --git a/http/document_handler.go b/http/document_handler.go index 957bbd0..e07890f 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,10 +24,10 @@ import ( "strings" "time" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) // TODO @@ -84,7 +84,7 @@ func (s *WebconfigServer) GetSubDocumentHandler(w http.ResponseWriter, r *http.R return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) writeStateHeaders(w, subdoc) w.WriteHeader(http.StatusOK) w.Write(subdoc.Payload()) diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 7d11287..5d274c1 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -48,7 +48,7 @@ func TestSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -58,7 +58,7 @@ func TestSubDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -73,7 +73,7 @@ func TestSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -82,7 +82,7 @@ func TestSubDocumentHandler(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -107,7 +107,7 @@ func TestDeleteDocumentHandler(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -117,7 +117,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -137,7 +137,7 @@ func TestDeleteDocumentHandler(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -146,7 +146,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -173,7 +173,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -182,7 +182,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -214,7 +214,7 @@ func TestPostWithDeviceId(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -226,7 +226,7 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -246,7 +246,7 @@ func TestPostWithDeviceId(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -257,7 +257,7 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -283,7 +283,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) gwrestoreBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header now := time.Now() @@ -298,7 +298,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) @@ -320,7 +320,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -339,7 +339,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -364,7 +364,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -373,7 +373,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -443,7 +443,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) gwrestoreBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare a version header now := time.Now() @@ -458,7 +458,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) @@ -480,7 +480,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -499,7 +499,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -524,7 +524,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -533,7 +533,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -597,7 +597,7 @@ func TestBadHeaderExpiryHandler(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // manage version and expiry headers now := time.Now() diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 3240ea6..5fc3c3a 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -100,7 +100,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -110,7 +110,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -126,7 +126,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -136,7 +136,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -196,7 +196,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -206,7 +206,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -222,7 +222,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -232,7 +232,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/mqtt_connector_test.go b/http/mqtt_connector_test.go index 48a2066..b6cdaae 100644 --- a/http/mqtt_connector_test.go +++ b/http/mqtt_connector_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -30,7 +30,7 @@ import ( func TestPayloadBuilder(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcData := util.Dict{ "device_id": "mac:044e5a22c9bf", diff --git a/http/multipart.go b/http/multipart.go index d46f665..fb4e8d0 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -166,7 +166,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -210,7 +210,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // mget ==> no upstream if userAgent == "mget" { - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -221,7 +221,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return respStatus, respHeader, respBytes, nil } @@ -230,7 +230,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // upstream handling // ============================= upstreamHeader := rHeader.Clone() - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) @@ -370,7 +370,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // upstream handling // ============================= upstreamHeader := rHeader.Clone() - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) diff --git a/http/multipart_test.go b/http/multipart_test.go index 2aed59a..4111af3 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -50,7 +50,7 @@ func TestMultipartConfigHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -60,7 +60,7 @@ func TestMultipartConfigHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -79,7 +79,7 @@ func TestMultipartConfigHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -89,7 +89,7 @@ func TestMultipartConfigHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -194,7 +194,7 @@ func TestCpeMiddleware(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -204,7 +204,7 @@ func TestCpeMiddleware(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -251,7 +251,7 @@ func TestVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -261,7 +261,7 @@ func TestVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -277,7 +277,7 @@ func TestVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -287,7 +287,7 @@ func TestVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -361,7 +361,7 @@ func TestVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -395,7 +395,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -405,7 +405,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -421,7 +421,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -431,7 +431,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -570,7 +570,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -580,7 +580,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -596,7 +596,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -606,7 +606,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -623,7 +623,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err := util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -687,7 +687,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -722,7 +722,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -747,7 +747,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -757,7 +757,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -774,7 +774,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -784,7 +784,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -868,7 +868,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -878,7 +878,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -895,7 +895,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -905,7 +905,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -922,7 +922,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -932,7 +932,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -949,7 +949,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -959,7 +959,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1127,7 +1127,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -1137,7 +1137,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1151,7 +1151,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1160,7 +1160,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1174,7 +1174,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) privatessidBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1183,7 +1183,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", privatessidUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index f354dcd..b2e0f91 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -119,7 +119,7 @@ func TestBuildMqttSendDocument(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -128,7 +128,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -151,7 +151,7 @@ func TestBuildMqttSendDocument(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -160,7 +160,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -196,7 +196,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -208,7 +208,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -227,7 +227,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 7 change the subdoc again ==== lanBytes2 := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes2)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -236,7 +236,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -262,7 +262,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go index 6413ee4..cd8743d 100644 --- a/http/refsubdocument_handler.go +++ b/http/refsubdocument_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -53,7 +53,7 @@ func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *htt return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) if refsubdoc.Version() != nil { w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) } diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index 49171b0..c0bd996 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,8 +24,9 @@ import ( "net/http" "testing" - "github.com/rdkcentral/webconfig/util" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -39,7 +40,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/reference/%v/document", refId) req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -48,7 +49,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -58,7 +59,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -67,7 +68,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/response.go b/http/response.go index 560a467..7cfeacb 100644 --- a/http/response.go +++ b/http/response.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -53,7 +53,7 @@ func WriteByMarshal(w http.ResponseWriter, status int, o interface{}) { LogError(w, common.NewError(err)) return } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) w.WriteHeader(status) w.Write(rbytes) } @@ -106,14 +106,14 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, SetAuditValue(w, "response", resp) rbytes = []byte(fmt.Sprintf(OkResponseTemplate, s, stateText, updatedTime)) } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) w.WriteHeader(http.StatusOK) w.Write(rbytes) } // this is used to return default tr-181 payload while the cpe is not in the db func WriteContentTypeAndResponse(w http.ResponseWriter, rbytes []byte, version string, contentType string) { - w.Header().Set("Content-type", contentType) + w.Header().Set(common.HeaderContentType, contentType) w.Header().Set(common.HeaderEtag, version) w.WriteHeader(http.StatusOK) w.Write(rbytes) @@ -135,7 +135,7 @@ func WriteErrorResponse(w http.ResponseWriter, status int, err error) { } func Error(w http.ResponseWriter, status int, err error) { - // calling WriteHeader() multiple times will cause errors in "content-type" + // calling WriteHeader() multiple times will cause errors in common.HeaderContentType // ==> errors like 'superfluous response.WriteHeader call' in stderr switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: @@ -147,14 +147,14 @@ func Error(w http.ResponseWriter, status int, err error) { func WriteResponseBytes(w http.ResponseWriter, rbytes []byte, statusCode int, vargs ...string) { if len(vargs) > 0 { - w.Header().Set("Content-type", vargs[0]) + w.Header().Set(common.HeaderContentType, vargs[0]) } w.WriteHeader(statusCode) w.Write(rbytes) } func WriteFactoryResetResponse(w http.ResponseWriter) { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.WriteHeader(http.StatusOK) w.Write([]byte{}) } diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index d329258..8c4e0f6 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -48,7 +48,7 @@ func TestRootDocumentHandler(t *testing.T) { // ==== step 0 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -94,7 +94,7 @@ func TestRootDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -103,7 +103,7 @@ func TestRootDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -121,7 +121,7 @@ func TestRootDocumentHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -130,7 +130,7 @@ func TestRootDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -242,7 +242,7 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { // ==== step 1 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 52e5660..82f0743 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,9 +23,9 @@ import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -101,7 +101,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } rootVersion := util.GetRandomRootVersion() - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, rootVersion) // help with unit tests diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 833d656..b0c7f52 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -106,7 +106,7 @@ func TestSupportedGroupsHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -116,7 +116,7 @@ func TestSupportedGroupsHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/upstream_test.go b/http/upstream_test.go index 90fcc94..f8c1bf3 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -65,7 +65,7 @@ func TestUpstream(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -104,7 +104,7 @@ func TestUpstream(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -143,7 +143,7 @@ func TestUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -152,7 +152,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -166,7 +166,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -175,7 +175,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -189,7 +189,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -198,7 +198,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -245,7 +245,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -273,7 +273,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -319,7 +319,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -361,7 +361,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -372,7 +372,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { respBytes := reqBytes // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, etag) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -412,7 +412,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -421,7 +421,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -435,7 +435,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -444,7 +444,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -458,7 +458,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -467,7 +467,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -508,7 +508,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -536,7 +536,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -579,7 +579,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -618,7 +618,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -657,7 +657,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -666,7 +666,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -680,7 +680,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -689,7 +689,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -703,7 +703,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -712,7 +712,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -759,7 +759,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -788,7 +788,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -841,7 +841,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -890,7 +890,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -917,7 +917,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) @@ -957,7 +957,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -966,7 +966,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -980,7 +980,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -989,7 +989,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1003,7 +1003,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1012,7 +1012,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1059,7 +1059,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1088,7 +1088,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1140,7 +1140,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1183,7 +1183,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -1210,7 +1210,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) @@ -1250,7 +1250,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1259,7 +1259,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -1273,7 +1273,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1282,7 +1282,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1296,7 +1296,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1305,7 +1305,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1348,7 +1348,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1377,7 +1377,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1428,7 +1428,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/validator.go b/http/validator.go index d85921e..bde64c6 100644 --- a/http/validator.go +++ b/http/validator.go @@ -14,16 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -56,8 +56,8 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") @@ -93,8 +93,8 @@ func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") diff --git a/http/validator_test.go b/http/validator_test.go index 8d36dcf..aafc71c 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -27,6 +27,7 @@ import ( "strings" "testing" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -46,7 +47,7 @@ func TestValidatorDisabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -55,7 +56,7 @@ func TestValidatorDisabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -86,7 +87,7 @@ func TestValidatorDisabled(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -95,7 +96,7 @@ func TestValidatorDisabled(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -139,7 +140,7 @@ func TestValidatorEnabled(t *testing.T) { server.SetValidateMacEnabled(true) url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -149,7 +150,7 @@ func TestValidatorEnabled(t *testing.T) { // post without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -159,7 +160,7 @@ func TestValidatorEnabled(t *testing.T) { // get with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -169,7 +170,7 @@ func TestValidatorEnabled(t *testing.T) { // get without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -211,7 +212,7 @@ func TestValidatorEnabled(t *testing.T) { // delete with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -221,7 +222,7 @@ func TestValidatorEnabled(t *testing.T) { // delete without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -230,7 +231,7 @@ func TestValidatorEnabled(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -272,7 +273,7 @@ func TestValidatorWithLowerCase(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -281,7 +282,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -301,7 +302,7 @@ func TestValidatorWithLowerCase(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -310,7 +311,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/util/multipart.go b/util/multipart.go index 70c4683..ee91abf 100644 --- a/util/multipart.go +++ b/util/multipart.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -49,7 +49,7 @@ func ParseMultipartAsList(header http.Header, bbytes []byte) ([]common.Multipart return mparts, nil } - mediaType, params, err := mime.ParseMediaType(header.Get("Content-Type")) + mediaType, params, err := mime.ParseMediaType(header.Get(common.HeaderContentType)) if err != nil { return nil, common.NewError(err) } diff --git a/util/parser_test.go b/util/parser_test.go index 1835dae..23fbc65 100644 --- a/util/parser_test.go +++ b/util/parser_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -22,13 +22,14 @@ import ( "net/http" "testing" + "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) func TestParseWebconfigResponseMessage(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("Content-length", "120") srcData := Dict{ @@ -55,7 +56,7 @@ func TestParseWebconfigResponseMessageMultipleHeaders(t *testing.T) { srcHeader.Add("X-Color", "color:red") srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") srcHeader.Add("X-Color", "color:orange") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("X-Color", "color:yellow:green") srcHeader.Add("Content-length", "120") srcHeader.Add("X-Color", "color:blue:indigo:violet") From 04a2dc42384a3a78eba25bbf7917119878eba871 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 18 Oct 2024 23:03:22 -0700 Subject: [PATCH 101/215] check nil conditions for kafka async producer errors --- http/webconfig_server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index c51b811..1842ceb 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1135,6 +1135,9 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { fields["output_offset"] = success.Offset log.WithFields(fields).Debug("sent") case pErr := <-s.Errors(): + if pErr == nil || pErr.Msg == nil { + continue + } if m := s.Metrics(); m != nil { m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) } From 63e902ba926f65baa0e7bcf4a4d70e52a91b6a8a Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 21 Oct 2024 16:22:38 -0700 Subject: [PATCH 102/215] check the nil condition for kafka async producer successes --- http/webconfig_server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1842ceb..a7e32c8 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1128,6 +1128,9 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { for { select { case success := <-s.Successes(): + if success == nil { + continue + } fields := make(log.Fields) fields["logger"] = "kafkaproducer" fields["output_topic"] = success.Topic From bef8fc72d3cdc102bd70c59f981edd90d4b1afe4 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 18 Oct 2024 23:03:22 -0700 Subject: [PATCH 103/215] check nil conditions for kafka async producer errors --- http/webconfig_server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index c51b811..1842ceb 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1135,6 +1135,9 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { fields["output_offset"] = success.Offset log.WithFields(fields).Debug("sent") case pErr := <-s.Errors(): + if pErr == nil || pErr.Msg == nil { + continue + } if m := s.Metrics(); m != nil { m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) } From 9bd2c8fa49541e3e3777ecb4d215ca8ae3ce5e17 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 21 Oct 2024 16:22:38 -0700 Subject: [PATCH 104/215] check the nil condition for kafka async producer successes --- http/webconfig_server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1842ceb..a7e32c8 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1128,6 +1128,9 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { for { select { case success := <-s.Successes(): + if success == nil { + continue + } fields := make(log.Fields) fields["logger"] = "kafkaproducer" fields["output_topic"] = success.Topic From b2548730585d7a64909f9110692080e062d488d2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 22 Oct 2024 15:37:40 -0700 Subject: [PATCH 105/215] add bitmap supprot for subdoc webui --- common/bitmap.go | 4 +++- util/firmware_bitmap_test.go | 42 +++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/common/bitmap.go b/common/bitmap.go index dd9c777..89492cc 100644 --- a/common/bitmap.go +++ b/common/bitmap.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common // header X-System-Supported-Docs @@ -38,6 +38,7 @@ var ( {6, 6}, {7, 29}, // connectedbuilding {8, 35}, // xmspeedboost + {9, 40}, // webui }, 2: { {1, 7}, @@ -146,6 +147,7 @@ var ( "wifistatsconfig": 37, "mwoconfigs": 38, "interference": 39, + "webui": 40, } ) diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 8572f68..cf3d5a5 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -547,3 +547,43 @@ func TestParseSupportedDocsHeaderHcm(t *testing.T) { supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } + +func TestParseSupportedDocsHeaderWebui(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} From 1ae6e582212a594513a1219e1ee0573c84e7089d Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 25 Oct 2024 21:42:48 -0700 Subject: [PATCH 106/215] fix a bug that 500 was returned when the reference doc was not found --- db/service.go | 3 + http/refsubdocument_handler_test.go | 144 ++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/db/service.go b/db/service.go index 20dfbad..7bf6edb 100644 --- a/db/service.go +++ b/db/service.go @@ -649,6 +649,9 @@ func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log if refId, ok := GetRefId(payload); ok { refsubdocument, err := c.GetRefSubDocument(refId) if err != nil { + if c.IsDbNotFound(err) { + continue + } return nil, common.NewError(err) } subDocument.SetPayload(refsubdocument.Payload()) diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index c0bd996..c9a1e2f 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -75,3 +75,147 @@ func TestRefSubDocumentHandler(t *testing.T) { assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) } + +func TestSubDocumentWithInvalidRefDoc(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + referenceIndicatorBytes := make([]byte, 4) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 setup refdoc1 and subdoc1 ==== + refId1 := uuid.New().String() + bbytes1 := util.RandomBytes(100, 150) + subdocId1 := "defaultrfc" + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId1) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes1)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes1) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId1) + xbytes := []byte(refId1) + tmpbytes := append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 2 setup refdoc2 and subdoc2 ==== + refId2 := uuid.New().String() + bbytes2 := util.RandomBytes(100, 150) + subdocId2 := "defaulttelemetry" + + // post + url = fmt.Sprintf("/api/v1/reference/%v/document", refId2) + req, err = http.NewRequest("POST", url, bytes.NewReader(bbytes2)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes2) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId2) + xbytes = []byte(refId2) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 3 GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok := mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) + + // ==== step 4 setup 3rd subdoc but link it to an non-existent refdoc3 ==== + refId3 := uuid.New().String() + subdocId3 := "defaultdcm" + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId3) + xbytes = []byte(refId3) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 5 GET /config returns 2 subdocs and no errors ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) +} From cbca2d60aef9dacbd9a3a3420ca281b00155bee4 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 8 Oct 2024 23:12:08 -0700 Subject: [PATCH 107/215] handle 202 returned by webpa poke --- http/http_client.go | 52 +++++++++++++++++++++++++-------------- http/poke_handler_test.go | 30 ++++++++++++++++++++++ http/response.go | 3 +++ http/webpa_connector.go | 22 ++++++++--------- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index aad3085..f67598c 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -52,7 +52,7 @@ type ErrorResponse struct { Message string `json:"message"` } -type StatusHandlerFunc func([]byte) ([]byte, http.Header, error, bool) +type StatusHandlerFunc func([]byte) ([]byte, http.Header, bool, error) type HttpClient struct { *http.Client @@ -107,7 +107,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } } -func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, error, bool) { +func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields) // verify a response is received @@ -121,11 +121,11 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] case "DELETE": req, err = http.NewRequest(method, url, nil) default: - return nil, nil, common.NewError(fmt.Errorf("method=%v", method)), false + return nil, nil, false, common.NewError(fmt.Errorf("method=%v", method)) } if err != nil { - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if header == nil { @@ -209,8 +209,8 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] res, err := c.Client.Do(req) // err should be *url.Error - tdiff := time.Now().Sub(startTime) - duration := tdiff.Nanoseconds() / 1000000 + tdiff := time.Since(startTime) + duration := tdiff.Milliseconds() fields[fmt.Sprintf("%v_duration", loggerName)] = duration delete(fields, bodyKey) @@ -236,25 +236,25 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] Message: ue.Error(), StatusCode: http.StatusGatewayTimeout, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if errors.Is(innerErr, io.EOF) { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusBadGateway, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if _, ok := innerErr.(*net.OpError); ok { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusServiceUnavailable, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } // Unknown err still appear as 500 } - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if res.Body != nil { defer res.Body.Close() @@ -276,7 +276,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] if userAgent != "mget" { log.WithFields(fields).Info(endMessage) } - return nil, nil, common.NewError(err), false + return nil, nil, true, common.NewError(err) } rbody := string(rbytes) @@ -331,11 +331,27 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] switch res.StatusCode { case http.StatusForbidden, http.StatusBadRequest, http.StatusNotFound: - return rbytes, nil, common.NewError(err), false + return rbytes, nil, false, common.NewError(err) + } + return rbytes, nil, true, common.NewError(err) + } else if res.StatusCode > 200 { + var pokeResponse PokeResponse + var message string + if err := json.Unmarshal(rbytes, &pokeResponse); err == nil { + if len(pokeResponse.Parameters) > 0 { + message = pokeResponse.Parameters[0].Message + } + } + if len(message) == 0 { + message = http.StatusText(res.StatusCode) + } + rherr := common.RemoteHttpError{ + Message: message, + StatusCode: res.StatusCode, } - return rbytes, nil, common.NewError(err), true + return rbytes, nil, false, common.NewError(rherr) } - return rbytes, res.Header, nil, false + return rbytes, res.Header, false, nil } func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { @@ -352,7 +368,7 @@ func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Heade if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, err, cont = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index b2e0f91..9a9d3a2 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -37,6 +37,7 @@ import ( var ( mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) + mockWebpaPoke202Response = []byte(`{"parameters":[{"message":"Previous request is in progress","name":"Device.X_RDK_WebConfig.ForceSync"}],"statusCode":202}`) ) func TestPokeHandler(t *testing.T) { @@ -335,3 +336,32 @@ func TestPokeHandlerWebpa403(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa202(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write(mockWebpaPoke202Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusAccepted) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/response.go b/http/response.go index 7cfeacb..7107bfa 100644 --- a/http/response.go +++ b/http/response.go @@ -140,6 +140,9 @@ func Error(w http.ResponseWriter, status int, err error) { switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: w.WriteHeader(status) + case http.StatusAccepted: + SetAuditValue(w, "response", err) + WriteByMarshal(w, status, err) default: WriteErrorResponse(w, status, err) } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index f6aadbe..52378d9 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,9 +25,9 @@ import ( "net/http" "time" - "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) @@ -72,7 +72,7 @@ type WebpaConnector struct { apiVersion string } -func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func syncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -82,16 +82,16 @@ func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } -func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func asyncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -101,13 +101,13 @@ func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), true + return rbytes, nil, true, common.NewError(rerr) } func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *WebpaConnector { @@ -231,7 +231,7 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field header.Set(common.HeaderTracestate, outTracestate) method := "PATCH" - _, _, err, cont := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -269,7 +269,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header ht if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { msg := fmt.Sprintf("finished success after 1 retry") if i > 1 { @@ -297,7 +297,7 @@ func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header htt if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, err, cont = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError From bf83753199d6de488e25fb3bf8d6d5136b5cdf2b Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 30 Oct 2024 16:39:25 -0700 Subject: [PATCH 108/215] change the updated_time as usual even if state=1 --- db/service.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/db/service.go b/db/service.go index 7bf6edb..00d4532 100644 --- a/db/service.go +++ b/db/service.go @@ -427,10 +427,6 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage var oldState int if subdoc.State() != nil { oldState = *subdoc.State() - if oldState == common.Deployed { - log.WithFields(fields).Warn("state=1 already") - return updatedSubdocIds, nil - } if oldState < common.Deployed || oldState > common.Failure { err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), From 3845b7d8c9567387fbc10bd20fbf398c870df351 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 31 Oct 2024 23:37:31 -0700 Subject: [PATCH 109/215] return 409 if the root_document is locked --- common/error.go | 5 +++-- common/root_document.go | 11 ++++++++++- common/root_document_test.go | 19 ++++++++++++++++++- db/cassandra/root_document.go | 15 +++++++++++---- db/cassandra/root_document_test.go | 30 +++++++++++++++++++++++++++++- db/cassandra/schema.go | 4 +++- db/service.go | 5 +++++ db/sqlite/schema.go | 4 +++- http/multipart.go | 15 ++++++++++++--- http/multipart_test.go | 23 +++++++++++++++++++++++ 10 files changed, 117 insertions(+), 14 deletions(-) diff --git a/common/error.go b/common/error.go index e733968..18b207f 100644 --- a/common/error.go +++ b/common/error.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -31,7 +31,8 @@ const ( ) var ( - ErrNotOK = fmt.Errorf("!ok") + ErrNotOK = fmt.Errorf("!ok") + ErrRootDocumentLocked = fmt.Errorf("root document is locked") ) type Http400Error struct { diff --git a/common/root_document.go b/common/root_document.go index a3542fd..c662f09 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "encoding/json" "fmt" + "time" ) const ( @@ -36,6 +37,7 @@ type RootDocument struct { SchemaVersion string `json:"schema_version"` Version string `json:"version"` QueryParams string `json:"query_params"` + LockedTill int `json:"locked_till"` } // (bitmap, firmware_version, model_name, partner_id, schema_version, version), nil @@ -69,6 +71,9 @@ func (d *RootDocument) NonEmptyColumnMap() map[string]interface{} { if d.Bitmap > 0 { dict["bitmap"] = d.Bitmap } + if d.LockedTill > 0 { + dict["locked_till"] = int64(d.LockedTill) + } tempDict := map[string]string{ "firmware_version": d.FirmwareVersion, @@ -170,3 +175,7 @@ func (d *RootDocument) Clone() *RootDocument { obj := *d return &obj } + +func (d *RootDocument) Locked() bool { + return d.LockedTill > 0 && int(time.Now().UnixMilli()) < d.LockedTill +} diff --git a/common/root_document_test.go b/common/root_document_test.go index c2ed5e2..50c149f 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "strings" "testing" + "time" "gotest.tools/assert" ) @@ -97,3 +98,19 @@ func TestRootDocumentEquals(t *testing.T) { ok = rootdoc1.Equals(rootdoc3) assert.Assert(t, !ok) } + +func TestRootDocumentLocked(t *testing.T) { + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + assert.Assert(t, !rootdoc.Locked()) + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + assert.Assert(t, rootdoc.Locked()) + time.Sleep(time.Duration(1) * time.Second) + assert.Assert(t, !rootdoc.Locked()) +} diff --git a/db/cassandra/root_document.go b/db/cassandra/root_document.go index 6d3d554..ff9321a 100644 --- a/db/cassandra/root_document.go +++ b/db/cassandra/root_document.go @@ -14,15 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "fmt" + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" - "github.com/prometheus/client_golang/prometheus" ) // shared.go: err := c.Query(stmt, cpeMac).MapScan(dict) @@ -32,11 +33,17 @@ func (c *CassandraClient) GetRootDocument(cpeMac string) (*common.RootDocument, defer func() { <-c.concurrentQueries }() var rd common.RootDocument - stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params FROM root_document WHERE cpe_mac=?" - err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams) + var tobj time.Time + stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till FROM root_document WHERE cpe_mac=?" + err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj) if err != nil { return nil, common.NewError(err) } + if tobj.IsZero() { + rd.LockedTill = 0 + } else { + rd.LockedTill = int(tobj.UnixMilli()) + } return &rd, nil } diff --git a/db/cassandra/root_document_test.go b/db/cassandra/root_document_test.go index 04ceb1f..98ffe38 100644 --- a/db/cassandra/root_document_test.go +++ b/db/cassandra/root_document_test.go @@ -14,11 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" @@ -169,3 +170,30 @@ func TestRootDocumentUpdate(t *testing.T) { assert.NilError(t, err) assert.DeepEqual(t, tgtRootdoc3, rootdoc3) } + +func TestRootDocumentLocked(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + + rootdoc := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + + err := tdbclient.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + fetched, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.DeepEqual(t, rootdoc, fetched) + assert.Assert(t, fetched.Locked()) + + time.Sleep(time.Duration(1) * time.Second) + + assert.Assert(t, !fetched.Locked()) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index b013804..6953e84 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -39,8 +39,10 @@ var ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + query_params text, route text, schema_version text, version text diff --git a/db/service.go b/db/service.go index 00d4532..a307c42 100644 --- a/db/service.go +++ b/db/service.go @@ -220,6 +220,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } } + // eval if the root_document is locked + if cloudRootDocument.Locked() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } + switch rootCmpEnum { case common.RootDocumentEquals: // create an empty "document" diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 3e4c132..81c368a 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -45,8 +45,10 @@ var ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + query_params text, route text, schema_version, version text diff --git a/http/multipart.go b/http/multipart.go index fb4e8d0..d6c29b5 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -89,9 +89,12 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) - // REMINDER 404 use standard response - if status == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) + switch status { + case http.StatusNotFound: + Error(w, status, nil) + return + case http.StatusConflict: + w.WriteHeader(status) return } @@ -132,6 +135,12 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { s.ForwardSuccessKafkaMessages(messages, fields) } + + // root_document locked + if errors.Is(err, common.ErrRootDocumentLocked) { + return http.StatusConflict, respHeader, nil, common.NewError(err) + } + if uconn == nil { if err != nil { if !s.IsDbNotFound(err) { diff --git a/http/multipart_test.go b/http/multipart_test.go index 4111af3..ce73135 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -26,6 +26,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db/cassandra" @@ -168,6 +169,28 @@ func TestMultipartConfigHandler(t *testing.T) { assert.NilError(t, err) parameters = response.Parameters assert.Equal(t, len(parameters), 1) + + // test root_document lock + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + rootdoc.LockedTill = int(time.Now().UnixMilli()) + 1000 + err = server.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + // get document again + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusConflict) + + time.Sleep(time.Duration(1) * time.Second) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) } func TestCpeMiddleware(t *testing.T) { From a01d3bb4cae777a7a4c84e4b163409e9f20d702a Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 30 Oct 2024 16:39:25 -0700 Subject: [PATCH 110/215] change the updated_time as usual even if state=1 --- db/service.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/db/service.go b/db/service.go index 20dfbad..ea754e1 100644 --- a/db/service.go +++ b/db/service.go @@ -427,10 +427,6 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage var oldState int if subdoc.State() != nil { oldState = *subdoc.State() - if oldState == common.Deployed { - log.WithFields(fields).Warn("state=1 already") - return updatedSubdocIds, nil - } if oldState < common.Deployed || oldState > common.Failure { err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), From c75288f940ae99d531d795dcda0654a6e492f7fa Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 4 Nov 2024 11:35:17 -0800 Subject: [PATCH 111/215] add a control flag for root_document locking --- config/sample_webconfig.conf | 3 ++ db/cassandra/cassandra_client.go | 43 +++++++++++++++++---------- db/cassandra/cassandra_client_test.go | 10 ++++++- db/database_client.go | 7 +++-- db/service.go | 2 +- db/sqlite/sqlite_client.go | 31 ++++++++++++------- db/sqlite/sqlite_client_test.go | 10 ++++++- http/multipart_test.go | 11 ++++++- 8 files changed, 85 insertions(+), 32 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index d460eef..22dacea 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -253,4 +253,7 @@ webconfig { brokers = "localhost:9092" topic = "webconfig_downstream" } + + // this allows the root document locked if needed + lock_root_document_enabled = false } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 7855b9f..3804f22 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -23,12 +23,12 @@ import ( "os" "time" + "github.com/go-akka/configuration" + "github.com/gocql/gocql" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gocql/gocql" ) const ( @@ -48,11 +48,12 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string - stateCorrectionEnabled bool + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool } /* @@ -161,16 +162,18 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, }, nil } @@ -227,6 +230,14 @@ func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { c.stateCorrectionEnabled = enabled } +func (c *CassandraClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *CassandraClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 355b701..77c882e 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -46,6 +46,14 @@ func TestCassandraClient(t *testing.T) { enabled = false tdbclient.SetStateCorrectionEnabled(enabled) assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/database_client.go b/db/database_client.go index fa1065f..cc325f1 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -14,12 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( - "github.com/rdkcentral/webconfig/common" "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" ) type DatabaseClient interface { @@ -68,4 +68,7 @@ type DatabaseClient interface { // enable state correction StateCorrectionEnabled() bool SetStateCorrectionEnabled(bool) + + LockRootDocumentEnabled() bool + SetLockRootDocumentEnabled(bool) } diff --git a/db/service.go b/db/service.go index a307c42..d776cea 100644 --- a/db/service.go +++ b/db/service.go @@ -221,7 +221,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } // eval if the root_document is locked - if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() && cloudRootDocument.Locked() { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) } diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 9ab3b12..56ee4f5 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -22,10 +22,10 @@ import ( "errors" "fmt" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/db" "github.com/go-akka/configuration" _ "github.com/mattn/go-sqlite3" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" ) const ( @@ -43,9 +43,10 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string - stateCorrectionEnabled bool + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -60,6 +61,7 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") db, err := sql.Open("sqlite3", dbfile) if err != nil { @@ -67,10 +69,11 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, }, nil } @@ -148,6 +151,14 @@ func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { c.stateCorrectionEnabled = enabled } +func (c *SqliteClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *SqliteClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go index 3a3f74f..f915f8e 100644 --- a/db/sqlite/sqlite_client_test.go +++ b/db/sqlite/sqlite_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -40,4 +40,12 @@ func TestSqliteClient(t *testing.T) { enabled = false tdbclient.SetStateCorrectionEnabled(enabled) assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } diff --git a/http/multipart_test.go b/http/multipart_test.go index ce73135..cc92820 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -177,7 +177,16 @@ func TestMultipartConfigHandler(t *testing.T) { err = server.SetRootDocument(cpeMac, rootdoc) assert.NilError(t, err) - // get document again + // get document again without the feature flag enabled + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get document again with the feature flag enabled + server.SetLockRootDocumentEnabled(true) + res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) assert.NilError(t, err) From f8a86a2ac2de30d79a77ca86aef558b5d54df741 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 25 Oct 2024 21:42:48 -0700 Subject: [PATCH 112/215] fix a bug that 500 was returned when the reference doc was not found --- db/service.go | 3 + http/refsubdocument_handler_test.go | 144 ++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/db/service.go b/db/service.go index ea754e1..00d4532 100644 --- a/db/service.go +++ b/db/service.go @@ -645,6 +645,9 @@ func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log if refId, ok := GetRefId(payload); ok { refsubdocument, err := c.GetRefSubDocument(refId) if err != nil { + if c.IsDbNotFound(err) { + continue + } return nil, common.NewError(err) } subDocument.SetPayload(refsubdocument.Payload()) diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index c0bd996..c9a1e2f 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -75,3 +75,147 @@ func TestRefSubDocumentHandler(t *testing.T) { assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) } + +func TestSubDocumentWithInvalidRefDoc(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + referenceIndicatorBytes := make([]byte, 4) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 setup refdoc1 and subdoc1 ==== + refId1 := uuid.New().String() + bbytes1 := util.RandomBytes(100, 150) + subdocId1 := "defaultrfc" + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId1) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes1)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes1) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId1) + xbytes := []byte(refId1) + tmpbytes := append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 2 setup refdoc2 and subdoc2 ==== + refId2 := uuid.New().String() + bbytes2 := util.RandomBytes(100, 150) + subdocId2 := "defaulttelemetry" + + // post + url = fmt.Sprintf("/api/v1/reference/%v/document", refId2) + req, err = http.NewRequest("POST", url, bytes.NewReader(bbytes2)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes2) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId2) + xbytes = []byte(refId2) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 3 GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok := mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) + + // ==== step 4 setup 3rd subdoc but link it to an non-existent refdoc3 ==== + refId3 := uuid.New().String() + subdocId3 := "defaultdcm" + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId3) + xbytes = []byte(refId3) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 5 GET /config returns 2 subdocs and no errors ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) +} From 93a333f5745d1e48a51e4aa8a20e3690a27188d8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 31 Oct 2024 23:37:31 -0700 Subject: [PATCH 113/215] return 409 if the root_document is locked --- common/error.go | 5 +++-- common/root_document.go | 11 ++++++++++- common/root_document_test.go | 19 ++++++++++++++++++- db/cassandra/root_document.go | 15 +++++++++++---- db/cassandra/root_document_test.go | 30 +++++++++++++++++++++++++++++- db/cassandra/schema.go | 4 +++- db/service.go | 5 +++++ db/sqlite/schema.go | 4 +++- http/multipart.go | 15 ++++++++++++--- http/multipart_test.go | 23 +++++++++++++++++++++++ 10 files changed, 117 insertions(+), 14 deletions(-) diff --git a/common/error.go b/common/error.go index e733968..18b207f 100644 --- a/common/error.go +++ b/common/error.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -31,7 +31,8 @@ const ( ) var ( - ErrNotOK = fmt.Errorf("!ok") + ErrNotOK = fmt.Errorf("!ok") + ErrRootDocumentLocked = fmt.Errorf("root document is locked") ) type Http400Error struct { diff --git a/common/root_document.go b/common/root_document.go index a3542fd..c662f09 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "encoding/json" "fmt" + "time" ) const ( @@ -36,6 +37,7 @@ type RootDocument struct { SchemaVersion string `json:"schema_version"` Version string `json:"version"` QueryParams string `json:"query_params"` + LockedTill int `json:"locked_till"` } // (bitmap, firmware_version, model_name, partner_id, schema_version, version), nil @@ -69,6 +71,9 @@ func (d *RootDocument) NonEmptyColumnMap() map[string]interface{} { if d.Bitmap > 0 { dict["bitmap"] = d.Bitmap } + if d.LockedTill > 0 { + dict["locked_till"] = int64(d.LockedTill) + } tempDict := map[string]string{ "firmware_version": d.FirmwareVersion, @@ -170,3 +175,7 @@ func (d *RootDocument) Clone() *RootDocument { obj := *d return &obj } + +func (d *RootDocument) Locked() bool { + return d.LockedTill > 0 && int(time.Now().UnixMilli()) < d.LockedTill +} diff --git a/common/root_document_test.go b/common/root_document_test.go index c2ed5e2..50c149f 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "strings" "testing" + "time" "gotest.tools/assert" ) @@ -97,3 +98,19 @@ func TestRootDocumentEquals(t *testing.T) { ok = rootdoc1.Equals(rootdoc3) assert.Assert(t, !ok) } + +func TestRootDocumentLocked(t *testing.T) { + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + assert.Assert(t, !rootdoc.Locked()) + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + assert.Assert(t, rootdoc.Locked()) + time.Sleep(time.Duration(1) * time.Second) + assert.Assert(t, !rootdoc.Locked()) +} diff --git a/db/cassandra/root_document.go b/db/cassandra/root_document.go index 6d3d554..ff9321a 100644 --- a/db/cassandra/root_document.go +++ b/db/cassandra/root_document.go @@ -14,15 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "fmt" + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" - "github.com/prometheus/client_golang/prometheus" ) // shared.go: err := c.Query(stmt, cpeMac).MapScan(dict) @@ -32,11 +33,17 @@ func (c *CassandraClient) GetRootDocument(cpeMac string) (*common.RootDocument, defer func() { <-c.concurrentQueries }() var rd common.RootDocument - stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params FROM root_document WHERE cpe_mac=?" - err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams) + var tobj time.Time + stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till FROM root_document WHERE cpe_mac=?" + err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj) if err != nil { return nil, common.NewError(err) } + if tobj.IsZero() { + rd.LockedTill = 0 + } else { + rd.LockedTill = int(tobj.UnixMilli()) + } return &rd, nil } diff --git a/db/cassandra/root_document_test.go b/db/cassandra/root_document_test.go index 04ceb1f..98ffe38 100644 --- a/db/cassandra/root_document_test.go +++ b/db/cassandra/root_document_test.go @@ -14,11 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" @@ -169,3 +170,30 @@ func TestRootDocumentUpdate(t *testing.T) { assert.NilError(t, err) assert.DeepEqual(t, tgtRootdoc3, rootdoc3) } + +func TestRootDocumentLocked(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + + rootdoc := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + + err := tdbclient.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + fetched, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.DeepEqual(t, rootdoc, fetched) + assert.Assert(t, fetched.Locked()) + + time.Sleep(time.Duration(1) * time.Second) + + assert.Assert(t, !fetched.Locked()) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index b013804..6953e84 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -39,8 +39,10 @@ var ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + query_params text, route text, schema_version text, version text diff --git a/db/service.go b/db/service.go index 00d4532..a307c42 100644 --- a/db/service.go +++ b/db/service.go @@ -220,6 +220,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } } + // eval if the root_document is locked + if cloudRootDocument.Locked() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } + switch rootCmpEnum { case common.RootDocumentEquals: // create an empty "document" diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 3e4c132..81c368a 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -45,8 +45,10 @@ var ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + query_params text, route text, schema_version, version text diff --git a/http/multipart.go b/http/multipart.go index fb4e8d0..d6c29b5 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -89,9 +89,12 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) - // REMINDER 404 use standard response - if status == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) + switch status { + case http.StatusNotFound: + Error(w, status, nil) + return + case http.StatusConflict: + w.WriteHeader(status) return } @@ -132,6 +135,12 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { s.ForwardSuccessKafkaMessages(messages, fields) } + + // root_document locked + if errors.Is(err, common.ErrRootDocumentLocked) { + return http.StatusConflict, respHeader, nil, common.NewError(err) + } + if uconn == nil { if err != nil { if !s.IsDbNotFound(err) { diff --git a/http/multipart_test.go b/http/multipart_test.go index 4111af3..ce73135 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -26,6 +26,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db/cassandra" @@ -168,6 +169,28 @@ func TestMultipartConfigHandler(t *testing.T) { assert.NilError(t, err) parameters = response.Parameters assert.Equal(t, len(parameters), 1) + + // test root_document lock + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + rootdoc.LockedTill = int(time.Now().UnixMilli()) + 1000 + err = server.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + // get document again + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusConflict) + + time.Sleep(time.Duration(1) * time.Second) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) } func TestCpeMiddleware(t *testing.T) { From 6297aa056370fdca192459e2f21b6baa00569af8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 4 Nov 2024 11:35:17 -0800 Subject: [PATCH 114/215] add a control flag for root_document locking --- config/sample_webconfig.conf | 3 ++ db/cassandra/cassandra_client.go | 43 +++++++++++++++++---------- db/cassandra/cassandra_client_test.go | 10 ++++++- db/database_client.go | 7 +++-- db/service.go | 2 +- db/sqlite/sqlite_client.go | 31 ++++++++++++------- db/sqlite/sqlite_client_test.go | 10 ++++++- http/multipart_test.go | 11 ++++++- 8 files changed, 85 insertions(+), 32 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index d460eef..22dacea 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -253,4 +253,7 @@ webconfig { brokers = "localhost:9092" topic = "webconfig_downstream" } + + // this allows the root document locked if needed + lock_root_document_enabled = false } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 7855b9f..3804f22 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -23,12 +23,12 @@ import ( "os" "time" + "github.com/go-akka/configuration" + "github.com/gocql/gocql" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gocql/gocql" ) const ( @@ -48,11 +48,12 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string - stateCorrectionEnabled bool + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool } /* @@ -161,16 +162,18 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, }, nil } @@ -227,6 +230,14 @@ func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { c.stateCorrectionEnabled = enabled } +func (c *CassandraClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *CassandraClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 355b701..77c882e 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -46,6 +46,14 @@ func TestCassandraClient(t *testing.T) { enabled = false tdbclient.SetStateCorrectionEnabled(enabled) assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/database_client.go b/db/database_client.go index fa1065f..cc325f1 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -14,12 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( - "github.com/rdkcentral/webconfig/common" "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" ) type DatabaseClient interface { @@ -68,4 +68,7 @@ type DatabaseClient interface { // enable state correction StateCorrectionEnabled() bool SetStateCorrectionEnabled(bool) + + LockRootDocumentEnabled() bool + SetLockRootDocumentEnabled(bool) } diff --git a/db/service.go b/db/service.go index a307c42..d776cea 100644 --- a/db/service.go +++ b/db/service.go @@ -221,7 +221,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } // eval if the root_document is locked - if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() && cloudRootDocument.Locked() { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) } diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 9ab3b12..56ee4f5 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -22,10 +22,10 @@ import ( "errors" "fmt" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/db" "github.com/go-akka/configuration" _ "github.com/mattn/go-sqlite3" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" ) const ( @@ -43,9 +43,10 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string - stateCorrectionEnabled bool + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -60,6 +61,7 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") db, err := sql.Open("sqlite3", dbfile) if err != nil { @@ -67,10 +69,11 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, }, nil } @@ -148,6 +151,14 @@ func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { c.stateCorrectionEnabled = enabled } +func (c *SqliteClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *SqliteClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go index 3a3f74f..f915f8e 100644 --- a/db/sqlite/sqlite_client_test.go +++ b/db/sqlite/sqlite_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -40,4 +40,12 @@ func TestSqliteClient(t *testing.T) { enabled = false tdbclient.SetStateCorrectionEnabled(enabled) assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } diff --git a/http/multipart_test.go b/http/multipart_test.go index ce73135..cc92820 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -177,7 +177,16 @@ func TestMultipartConfigHandler(t *testing.T) { err = server.SetRootDocument(cpeMac, rootdoc) assert.NilError(t, err) - // get document again + // get document again without the feature flag enabled + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get document again with the feature flag enabled + server.SetLockRootDocumentEnabled(true) + res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) assert.NilError(t, err) From b5b428cf5a2687e1ec6e93695d71872fe0588360 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 22 Oct 2024 15:37:40 -0700 Subject: [PATCH 115/215] add bitmap supprot for subdoc webui --- common/bitmap.go | 4 +++- util/firmware_bitmap_test.go | 42 +++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/common/bitmap.go b/common/bitmap.go index dd9c777..89492cc 100644 --- a/common/bitmap.go +++ b/common/bitmap.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common // header X-System-Supported-Docs @@ -38,6 +38,7 @@ var ( {6, 6}, {7, 29}, // connectedbuilding {8, 35}, // xmspeedboost + {9, 40}, // webui }, 2: { {1, 7}, @@ -146,6 +147,7 @@ var ( "wifistatsconfig": 37, "mwoconfigs": 38, "interference": 39, + "webui": 40, } ) diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 8572f68..cf3d5a5 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -547,3 +547,43 @@ func TestParseSupportedDocsHeaderHcm(t *testing.T) { supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } + +func TestParseSupportedDocsHeaderWebui(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} From 33f6466c89bf8d4fe2c49f05447255c642359821 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 Nov 2024 17:32:40 -0800 Subject: [PATCH 116/215] add an option to read profiles from upstream --- config/sample_webconfig.conf | 3 + http/supplementary_handler.go | 75 +++++++-- http/supplementary_handler_test.go | 241 ++++++++++++++++++++++++++++- http/upstream_connector.go | 37 ++++- http/webconfig_server.go | 12 ++ http/webconfig_server_test.go | 10 +- util/string.go | 43 ++++- util/string_test.go | 17 +- 8 files changed, 419 insertions(+), 19 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 22dacea..9ad5f2f 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -256,4 +256,7 @@ webconfig { // this allows the root document locked if needed lock_root_document_enabled = false + + // get extra profiles from upstream + upstream_profiles_enabled = false } diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 82f0743..20dc8fd 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -29,6 +29,10 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + notFoundProfileText = `{"profiles":[]}` +) + func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { // ==== data integrity check ==== params := mux.Vars(r) @@ -50,18 +54,17 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } // append the extra query_params if any + var rootdoc *common.RootDocument var queryParams string - if s.SupplementaryAppendingEnabled() { - rootdoc, err := s.GetRootDocument(mac) + var err error + if s.SupplementaryAppendingEnabled() || s.UpstreamProfilesEnabled() { + rootdoc, err = s.GetRootDocument(mac) if err != nil { if !s.IsDbNotFound(err) { Error(w, http.StatusInternalServerError, common.NewError(err)) return } } - if rootdoc != nil { - queryParams = rootdoc.QueryParams - } } // partner handling @@ -70,21 +73,73 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r partnerId = "" } + if s.SupplementaryAppendingEnabled() && rootdoc != nil { + queryParams = rootdoc.QueryParams + } + urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - rbytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + xconfNotFound := false if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - Error(w, rherr.StatusCode, rherr) + if rherr.StatusCode == http.StatusNotFound { + if s.UpstreamProfilesEnabled() { + xconfNotFound = true + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if !xconfNotFound { + Error(w, http.StatusInternalServerError, common.NewError(err)) return } - Error(w, http.StatusInternalServerError, common.NewError(err)) - return } - mpart, err := util.TelemetryBytesToMultipart(rbytes) + var profileBytes []byte + if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { + // Get profiles from the second source + extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) + if err != nil { + exitNow := true + var rherr common.RemoteHttpError + if errors.As(err, &rherr) { + if rherr.StatusCode == http.StatusNotFound { + exitNow = false + extraProfileBytes = nil + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if exitNow { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + } + + if xconfNotFound { + baseProfileBytes = []byte(notFoundProfileText) + } + + // append profiles stored at webconfig + profileBytes, err = util.AppendProfiles(baseProfileBytes, extraProfileBytes) + if err != nil { + Error(w, http.StatusInternalServerError, err) + return + } + } else { + profileBytes = baseProfileBytes + } + + mpart, err := util.TelemetryBytesToMultipart(profileBytes) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) return diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 54e2642..1328b9c 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -890,3 +890,242 @@ func TestSupplementaryXconfReadAllErr(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) } + +var ( + mockUpstreamProfileResponse1 = []byte(`[ + { + "name": "subname1", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.1.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "977e16c4" + }, + { + "name": "subname2", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.2.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "4f207ebd" + } +]`) +) + +func TestSupplementaryUpstreamProfiles(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + cxbytes, err := util.CompactJson([]byte(mockProfileResponse)) + assert.NilError(t, err) + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cxbytes) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + cubytes, err := util.CompactJson(mockUpstreamProfileResponse1) + assert.NilError(t, err) + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cubytes) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok := mparts["telemetry"] + assert.Assert(t, ok) + + output := common.TR181Output{} + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes := []byte(output.Parameters[0].Value) + + var itf util.Dict + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok := itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok := profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf := profilesJs[0] + + profile1, ok := profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok := profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok := coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + var srcItf map[string]interface{} + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 4 enable the feature flag but no query_param expect 200 with 1 mpart ==== + server.SetUpstreamProfilesEnabled(true) + defer server.SetUpstreamProfilesEnabled(false) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok = profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok = coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + srcItf = make(map[string]interface{}) + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 5 set query_param in the root_document expect 200 with 1 mpart ==== + rdoc := new(common.RootDocument) + rdoc.QueryParams = "key1=val1" + rdoc.ModelName = modelName + rdoc.PartnerId = partnerID + rdoc.FirmwareVersion = firmwareVersion + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + t.Logf("%s\n", rbytes) + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 3) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + profile2Itf := profilesJs[1] + profile2, ok := profile2Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile2["name"].(string), "subname1") + + profile3Itf := profilesJs[2] + profile3, ok := profile3Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile3["name"].(string), "subname2") +} diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 165aac6..d006bc5 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,15 +22,16 @@ import ( "fmt" "net/http" + "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" owcommon "github.com/rdkcentral/webconfig/common" - "github.com/go-akka/configuration" log "github.com/sirupsen/logrus" ) const ( upstreamHostDefault = "http://localhost:1234" - upstreamUrlTemplateDefault = "/api/v1/device/%v/upstream" + defaultUpstreamUrlTemplate = "/api/v1/device/%v/upstream" + defaultProfileUrlTemplate = "/api/v1/device/%v/profile?%v" ) type UpstreamConnector struct { @@ -38,6 +39,7 @@ type UpstreamConnector struct { host string serviceName string upstreamUrlTemplate string + profileUrlTemplate string } func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *UpstreamConnector { @@ -45,13 +47,16 @@ func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *Up confKey := fmt.Sprintf("webconfig.%v.host", serviceName) host := conf.GetString(confKey, upstreamHostDefault) confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) - upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) + upstreamUrlTemplate := conf.GetString(confKey, defaultUpstreamUrlTemplate) + confKey = fmt.Sprintf("webconfig.%v.profile_url_template", serviceName) + profileUrlTemplate := conf.GetString(confKey, defaultProfileUrlTemplate) return &UpstreamConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, + profileUrlTemplate: profileUrlTemplate, } } @@ -90,3 +95,27 @@ func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes } return rbytes, header, nil } + +func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { + url := c.UpstreamHost() + fmt.Sprintf(c.profileUrlTemplate, mac, queryParams) + + if itf, ok := fields["audit_id"]; ok { + auditId := itf.(string) + if len(auditId) > 0 { + header.Set(common.HeaderAuditid, auditId) + } + } + + if itf, ok := fields["app_name"]; ok { + appName := itf.(string) + if len(appName) > 0 { + header.Set(common.HeaderSourceAppName, appName) + } + } + + rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) + if err != nil { + return rbytes, header, owcommon.NewError(err) + } + return rbytes, header, nil +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index a7e32c8..63d7d7c 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -118,6 +118,7 @@ type WebconfigServer struct { webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string + upstreamProfilesEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -284,6 +285,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } } + upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -318,6 +321,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, + upstreamProfilesEnabled: upstreamProfilesEnabled, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -634,6 +638,14 @@ func (s *WebconfigServer) SetKafkaProducerTopic(x string) { s.kafkaProducerTopic = x } +func (s *WebconfigServer) UpstreamProfilesEnabled() bool { + return s.upstreamProfilesEnabled +} + +func (s *WebconfigServer) SetUpstreamProfilesEnabled(enabled bool) { + s.upstreamProfilesEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 1699106..6f92f75 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -97,4 +97,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { validPartners = []string{"name3", "name4", "name5"} server.SetValidPartners(validPartners) assert.DeepEqual(t, server.ValidPartners(), validPartners) + + // get profiles from upstream + enabled = true + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + enabled = false + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) } diff --git a/util/string.go b/util/string.go index 439b0bb..05eab77 100644 --- a/util/string.go +++ b/util/string.go @@ -14,18 +14,26 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "fmt" "net/http" "net/url" "strconv" "strings" - "github.com/rdkcentral/webconfig/common" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" +) + +const ( + Comma = ',' + RightSquareBracket = ']' + RightCurlBracket = '}' + baseProfileStr = `{"profiles":[` ) var ( @@ -184,3 +192,34 @@ func IsValidUTF8(bbytes []byte) bool { str2 := strings.ToValidUTF8(str1, "#") return str1 == str2 } + +func AppendProfiles(pbytes, sbytes []byte) ([]byte, error) { + var builder strings.Builder + if len(pbytes) < 3 { + builder.WriteString(baseProfileStr) + } else { + s := strings.TrimSpace(string(pbytes)) + builder.WriteString(s[:len(s)-2]) + } + if len(sbytes) > 2 { + if len(pbytes) > 20 { + builder.WriteRune(',') + } + builder.Write(sbytes[1 : len(sbytes)-1]) + } + builder.WriteRune(RightSquareBracket) + builder.WriteRune(RightCurlBracket) + return []byte(builder.String()), nil +} + +func CompactJson(sbytes []byte) ([]byte, error) { + var itf interface{} + if err := json.Unmarshal(sbytes, &itf); err != nil { + return nil, common.NewError(err) + } + jsbytes, err := json.Marshal(itf) + if err != nil { + return nil, common.NewError(err) + } + return jsbytes, nil +} diff --git a/util/string_test.go b/util/string_test.go index cf6a894..73640bc 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -14,10 +14,11 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "net/http" "net/url" "testing" @@ -159,3 +160,17 @@ func TestTelemetryQueryWithWanMac(t *testing.T) { expected := "env=PROD&partnerId=comcast&version=2.0&model=TG1682G&accountId=1234567890&firmwareVersion=TG1682_3.14p9s6_PROD_sey&estbMacAddress=567890ABCDEF" assert.Equal(t, qstr, expected) } + +func TestAppendProfiles(t *testing.T) { + mockedBaseProfilesResponse := `{"profiles":[{"name":"XfinityWIFI_SYNC","value":{"Description":"XfinityWIFI_SYNC to capture XWIFI info every 12 hours","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://stbrtl.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"name":"cpe_passpoint_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingServiceEnable","type":"dataModel"},{"name":"cpe_passpoint_inter_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingService.Parameters","type":"dataModel"},{"name":"cpe_passpoint_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Parameters","type":"dataModel"},{"name":"cpe_passpoint_rdk_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Enable","type":"dataModel"},{"name":"open5_bss_active","reference":"Device.WiFi.SSID.6.Enable","type":"dataModel"},{"name":"secure5_bss_active","reference":"Device.WiFi.SSID.10.Enable","type":"dataModel"},{"name":"secure5_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"wifi_enabled","reference":"Device.DeviceInfo.X_COMCAST_COM_xfinitywifiEnable","type":"dataModel"},{"name":"primary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.PrimaryRemoteEndpoint","type":"dataModel"},{"name":"secondary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.SecondaryRemoteEndpoint","type":"dataModel"},{"name":"open5_advertisement_str","reference":"Device.WiFi.AccessPoint.6.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"secure5_advertisement_str","reference":"Device.WiFi.AccessPoint.10.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"device_vlan_2","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.Interface.2.VLANID","type":"dataModel"},{"name":"open5_ssid_str","reference":"Device.WiFi.SSID.6.SSID","type":"dataModel"},{"name":"secure5_ssid_str","reference":"Device.WiFi.SSID.10.SSID","type":"dataModel"},{"name":"secure5_securitymode_str","reference":"Device.WiFi.AccessPoint.10.Security.ModeEnabled","type":"dataModel"},{"name":"secure5_pri_port","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerPort","type":"dataModel"},{"name":"dscp_marker","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.DSCPMarkPolicy","type":"dataModel"},{"name":"WIFI_CH_2_split","reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"name":"WIFI_CW_2_split","reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"name":"open5_bssid_str","reference":"Device.WiFi.SSID.6.BSSID","type":"dataModel"},{"name":"secure5_bssid_str","reference":"Device.WiFi.SSID.10.BSSID","type":"dataModel"},{"name":"open5_status_str","reference":"Device.WiFi.SSID.6.Status","type":"dataModel"},{"name":"secure5_status_str","reference":"Device.WiFi.SSID.10.Status","type":"dataModel"},{"name":"open5_beaconpower_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"open5_beaconrate_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"secure5_beaconpower_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"secure5_beaconrate_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"radio5_beaconinterval","reference":"Device.WiFi.Radio.2.X_COMCAST-COM_BeaconInterval","type":"dataModel"},{"name":"secure5_encryption","reference":"Device.WiFi.AccessPoint.10.Security.X_CISCO_COM_EncryptionMethod","type":"dataModel"},{"name":"secure5_sec_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerIPAddr","type":"dataModel"},{"name":"secure5_sec_port","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerPort","type":"dataModel"},{"name":"UPTIME_split","reference":"Device.DeviceInfo.UpTime","type":"dataModel"},{"name":"open5_radius_server_ip","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"open5_pri_port","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerPort","type":"dataModel"},{"name":"open5_isolation_enable","reference":"Device.WiFi.AccessPoint.6.IsolationEnable","type":"dataModel"},{"name":"secure5_isolation_enable","reference":"Device.WiFi.AccessPoint.10.IsolationEnable","type":"dataModel"},{"name":"secure24_bss_active","reference":"Device.WiFi.SSID.9.Enable","type":"dataModel"},{"name":"open24_bss_active","reference":"Device.WiFi.SSID.5.Enable","type":"dataModel"}],"Protocol":"HTTP","ReportingAdjustments":{"FirstReportingInterval":300},"ReportingInterval":86400,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.6"},"versionHash":"d9c6f386"},{"name":"WIFI_MOTION_Telemetry","value":{"Description":"CSCWFM_Telemetry","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"}],"URL":"https://stbrtl-oi.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"name":"Profile_Name","reference":"Profile.Name","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMrbussub_fail","name":"SYS_ERROR_WFMrbussub_fail","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_CSIpipe_restart","name":"SYS_SH_WFMpipe_restart","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_ctrlifcreate_fail","name":"SYS_ERROR_WFMifcreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSC_RXMrbusinit_fail","name":"SYS_ERROR_WFM_rbusinit_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionacquire_fail","name":"SYS_ERROR_WFMSessionAcq_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMeventscreate_fail","name":"SYS_ERROR_WFMeventscreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionenable_fail","name":"SYS_ERROR_WFMsessionenable_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXM_restart","name":"SYS_SH_WFMRXM_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_borg_restart","name":"SYS_SH_WFMborg_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_mqtt_restart","name":"SYS_SH_WFMmqtt_restart","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_sounding_state","name":"WFMsoundingstate_split","type":"event","use":"absolute"},{"name":"WFMEnable_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.CognitiveMotionDetection.Enable","type":"dataModel"},{"name":"WFMStatus_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_XHFW.WiFiMotionStatus","type":"dataModel"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentCommFailCnt_split","search":"wfmCogAgentCommFailCnt:","type":"grep","use":"count"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentConnected_split","search":"wfmCogAgentConnected:","type":"grep","use":"absolute"}],"Protocol":"HTTP","ReportingInterval":900,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"bf86fd16"}]}` + mockedExtraProfilesResponse := `[{"name":"james_test_profile_001","value":{"ActivationTimeout":600,"Description":"Telemetry 2.0 HSD Gateway WiFi Radio","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://rdkrtldev.stb.r53.xcal.tv/"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Description","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"}],"Protocol":"HTTP","ReportingInterval":60,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"ed0de6ef"}]` + + appendedBytes, err := AppendProfiles([]byte(mockedBaseProfilesResponse), []byte(mockedExtraProfilesResponse)) + assert.NilError(t, err) + + var itf interface{} + err = json.Unmarshal(appendedBytes, &itf) + assert.NilError(t, err) + _, err = json.MarshalIndent(itf, "", " ") + assert.NilError(t, err) +} From b289cbe1967f09bade86dc807e97f728eb3ea813 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 Nov 2024 22:20:23 -0800 Subject: [PATCH 117/215] fix a bug that error_code and error_details were not cleanup during state correction --- db/service.go | 10 ++++++++++ http/multipart_test.go | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/db/service.go b/db/service.go index d776cea..0db393f 100644 --- a/db/service.go +++ b/db/service.go @@ -194,6 +194,8 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if len(cloudVersion) == 0 { continue } + cloudErrorCode := *subdocument.ErrorCode() + cloudErrorDetails := *subdocument.ErrorDetails() deviceVersion := deviceVersionMap[subdocId] if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { labels := prometheus.Labels{ @@ -203,6 +205,14 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // update state newState := common.Deployed subdocument.SetState(&newState) + if cloudErrorCode > 0 { + var newErrorCode int + subdocument.SetErrorCode(&newErrorCode) + } + if len(cloudErrorDetails) > 0 { + var newErrorDetails string + subdocument.SetErrorDetails(&newErrorDetails) + } if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } diff --git a/http/multipart_test.go b/http/multipart_test.go index cc92820..abfed54 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1065,7 +1065,11 @@ func TestStateCorrectionEnabled(t *testing.T) { assert.NilError(t, err) meshState := common.Failure + meshErrorCode := 307 + meshErrorDetails := "NACK:OneWifi," meshSubdocument.SetState(&meshState) + meshSubdocument.SetErrorCode(&meshErrorCode) + meshSubdocument.SetErrorDetails(&meshErrorDetails) err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) assert.NilError(t, err) @@ -1100,6 +1104,8 @@ func TestStateCorrectionEnabled(t *testing.T) { meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Failure) + assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) + assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) @@ -1132,6 +1138,8 @@ func TestStateCorrectionEnabled(t *testing.T) { meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + assert.Equal(t, *meshSubdocument.ErrorCode(), 0) + assert.Equal(t, *meshSubdocument.ErrorDetails(), "") mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) From e037db01b16de22b749c15418afa6d9434aa3e43 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 Nov 2024 22:20:23 -0800 Subject: [PATCH 118/215] fix a bug that error_code and error_details were not cleanup during state correction --- db/service.go | 10 ++++++++++ http/multipart_test.go | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/db/service.go b/db/service.go index d776cea..0db393f 100644 --- a/db/service.go +++ b/db/service.go @@ -194,6 +194,8 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if len(cloudVersion) == 0 { continue } + cloudErrorCode := *subdocument.ErrorCode() + cloudErrorDetails := *subdocument.ErrorDetails() deviceVersion := deviceVersionMap[subdocId] if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { labels := prometheus.Labels{ @@ -203,6 +205,14 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // update state newState := common.Deployed subdocument.SetState(&newState) + if cloudErrorCode > 0 { + var newErrorCode int + subdocument.SetErrorCode(&newErrorCode) + } + if len(cloudErrorDetails) > 0 { + var newErrorDetails string + subdocument.SetErrorDetails(&newErrorDetails) + } if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } diff --git a/http/multipart_test.go b/http/multipart_test.go index cc92820..abfed54 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1065,7 +1065,11 @@ func TestStateCorrectionEnabled(t *testing.T) { assert.NilError(t, err) meshState := common.Failure + meshErrorCode := 307 + meshErrorDetails := "NACK:OneWifi," meshSubdocument.SetState(&meshState) + meshSubdocument.SetErrorCode(&meshErrorCode) + meshSubdocument.SetErrorDetails(&meshErrorDetails) err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) assert.NilError(t, err) @@ -1100,6 +1104,8 @@ func TestStateCorrectionEnabled(t *testing.T) { meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Failure) + assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) + assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) @@ -1132,6 +1138,8 @@ func TestStateCorrectionEnabled(t *testing.T) { meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + assert.Equal(t, *meshSubdocument.ErrorCode(), 0) + assert.Equal(t, *meshSubdocument.ErrorDetails(), "") mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) From e00d70e744f5b1d6e27683003b91249491e5a2ec Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 16 Nov 2024 10:42:36 -0800 Subject: [PATCH 119/215] enforce strict format in device GET route --- common/error.go | 1 + config/sample_webconfig.conf | 3 + db/service.go | 10 +- http/multipart.go | 15 ++- http/multipart_test.go | 171 +++++++++++++++++++++++++++++++++ http/webconfig_server.go | 11 +++ http/webconfig_server_test.go | 8 ++ util/string.go | 61 ------------ util/string_test.go | 80 ---------------- util/validator.go | 124 ++++++++++++++++++++++++ util/validator_test.go | 174 ++++++++++++++++++++++++++++++++++ 11 files changed, 511 insertions(+), 147 deletions(-) create mode 100644 util/validator.go create mode 100644 util/validator_test.go diff --git a/common/error.go b/common/error.go index 18b207f..fa22ddf 100644 --- a/common/error.go +++ b/common/error.go @@ -33,6 +33,7 @@ const ( var ( ErrNotOK = fmt.Errorf("!ok") ErrRootDocumentLocked = fmt.Errorf("root document is locked") + ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") ) type Http400Error struct { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 9ad5f2f..d32fcff 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -259,4 +259,7 @@ webconfig { // get extra profiles from upstream upstream_profiles_enabled = false + + // only valid query_params formats are accepted + query_params_validation_enabled = false } diff --git a/db/service.go b/db/service.go index 0db393f..22c9b9b 100644 --- a/db/service.go +++ b/db/service.go @@ -231,8 +231,14 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } // eval if the root_document is locked - if c.LockRootDocumentEnabled() && cloudRootDocument.Locked() { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } else { + tfields := common.FilterLogFields(fields) + tfields["logger"] = "rootdoc" + log.WithFields(tfields).Warn("dryrun409") + } } switch rootCmpEnum { diff --git a/http/multipart.go b/http/multipart.go index d6c29b5..09ed8c1 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -75,11 +75,18 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. return } fields := xw.Audit() - fields["cpe_mac"] = mac - if qGroupIds, ok := r.URL.Query()["group_id"]; ok { - fields["group_id"] = qGroupIds[0] - r.Header.Set(common.HeaderDocName, qGroupIds[0]) + + // enforce strict query parameters check + err := util.ValidateQueryParams(r, fields) + if err != nil && s.QueryParamsValidationEnabled() { + if errors.Is(err, common.ErrInvalidQueryParams) { + Error(w, http.StatusBadRequest, nil) + log.WithFields(fields).Error(err) + return + } + Error(w, http.StatusInternalServerError, err) + return } // handle empty schema version header diff --git a/http/multipart_test.go b/http/multipart_test.go index abfed54..ce3b84d 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1308,3 +1308,174 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { _, ok = mpartMap["privatessid"] assert.Assert(t, !ok) } + +func TestValidateQueryParams(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetQueryParamsValidationEnabled(true) + assert.Assert(t, server.QueryParamsValidationEnabled()) + defer server.SetQueryParamsValidationEnabled(false) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123") + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + etag := res.Header.Get(common.HeaderEtag) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanMpartVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanMpartVersion := mpart.Version + matchedIfNoneMatch := fmt.Sprintf("%v,%v,%v", etag, lanMpartVersion, wanMpartVersion) + + // case 9 versions matched 304 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, matchedIfNoneMatch) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 63d7d7c..b1875b9 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -119,6 +119,7 @@ type WebconfigServer struct { kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool + queryParamsValidationEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -286,6 +287,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") + queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") ws := &WebconfigServer{ Server: &http.Server{ @@ -322,6 +324,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, upstreamProfilesEnabled: upstreamProfilesEnabled, + queryParamsValidationEnabled: queryParamsValidationEnabled, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -646,6 +649,14 @@ func (s *WebconfigServer) SetUpstreamProfilesEnabled(enabled bool) { s.upstreamProfilesEnabled = enabled } +func (s *WebconfigServer) QueryParamsValidationEnabled() bool { + return s.queryParamsValidationEnabled +} + +func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { + s.queryParamsValidationEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 6f92f75..404c3eb 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -105,4 +105,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { enabled = false server.SetUpstreamProfilesEnabled(enabled) assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + + // enforce strict query parameters validation + enabled = true + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + enabled = false + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) } diff --git a/util/string.go b/util/string.go index 05eab77..064cc8e 100644 --- a/util/string.go +++ b/util/string.go @@ -61,18 +61,6 @@ func GenerateRandomCpeMac() string { return strings.ToUpper(u[len(u)-12:]) } -func ValidateMac(mac string) bool { - if len(mac) != 12 { - return false - } - for _, r := range mac { - if r < 48 || r > 70 || (r > 57 && r < 65) { - return false - } - } - return true -} - func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId string) string { // build the query parameters in a fixed order params := []string{} @@ -129,55 +117,6 @@ func GetMacDiff(wanMac, mac string) int { return wanMacVal - macVal } -func ValidatePokeQuery(values url.Values) (string, error) { - // handle ?doc=xxx - if docQueryParamStrs, ok := values["doc"]; ok { - if len(docQueryParamStrs) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(docQueryParamStrs[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeDocs, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // handle ?route=xxx - if qparams, ok := values["route"]; ok { - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(qparams[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeRoutes, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // return default - return "primary", nil -} - func GetEstbMacAddress(mac string) string { // if the mac cannot be parsed, then return back the input i, err := strconv.ParseInt(mac, 16, 64) diff --git a/util/string_test.go b/util/string_test.go index 73640bc..3c6df1a 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -20,7 +20,6 @@ package util import ( "encoding/json" "net/http" - "net/url" "testing" "github.com/rdkcentral/webconfig/common" @@ -34,28 +33,6 @@ func TestString(t *testing.T) { assert.Equal(t, c, expected) } -func TestValidateMac(t *testing.T) { - mac := "001122334455" - assert.Assert(t, ValidateMac(mac)) - - mac = "4444ABCDEF01" - assert.Assert(t, ValidateMac(mac)) - - mac = "00112233445Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "001122334455Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "0H1122334455" - assert.Assert(t, !ValidateMac(mac)) - - for i := 0; i < 10; i++ { - mac := GenerateRandomCpeMac() - assert.Assert(t, ValidateMac(mac)) - } -} - func TestGetAuditId(t *testing.T) { auditId := GetAuditId() assert.Equal(t, len(auditId), 32) @@ -81,63 +58,6 @@ func TestTelemetryQuery(t *testing.T) { assert.Equal(t, qstr, expected) } -func TestValidatePokeQuery(t *testing.T) { - values := url.Values{} - - values["doc"] = []string{ - "primary,telemetry", - "hello,world", - } - _, err := ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,hello,world", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,telemetry", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary", - } - s, err := ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "telemetry", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "telemetry") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "primary", - } - values["route"] = []string{ - "mqtt", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "mqtt") -} - func TestIsValidUTF8(t *testing.T) { b1 := []byte(`{"foo":"bar","hello":123,"world":true}`) assert.Assert(t, IsValidUTF8(b1)) diff --git a/util/validator.go b/util/validator.go new file mode 100644 index 0000000..4ac602c --- /dev/null +++ b/util/validator.go @@ -0,0 +1,124 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package util + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" +) + +func ValidateMac(mac string) bool { + if len(mac) != 12 { + return false + } + for _, r := range mac { + if r < 48 || r > 70 || (r > 57 && r < 65) { + return false + } + } + return true +} + +func ValidatePokeQuery(values url.Values) (string, error) { + // handle ?doc=xxx + if docQueryParamStrs, ok := values["doc"]; ok { + if len(docQueryParamStrs) > 1 { + err := fmt.Errorf("multiple doc parameter is not allowed") + return "", common.NewError(err) + } + + qparams := strings.Split(docQueryParamStrs[0], ",") + if len(qparams) > 1 { + err := fmt.Errorf("multiple doc parameter is not allowed") + return "", common.NewError(err) + } + + queryStr := qparams[0] + if !Contains(common.SupportedPokeDocs, queryStr) { + err := fmt.Errorf("invalid query parameter: %v", queryStr) + return "", common.NewError(err) + + } + return queryStr, nil + } + + // handle ?route=xxx + if qparams, ok := values["route"]; ok { + if len(qparams) > 1 { + err := fmt.Errorf("multiple route parameter is not allowed") + return "", common.NewError(err) + } + + qparams := strings.Split(qparams[0], ",") + if len(qparams) > 1 { + err := fmt.Errorf("multiple route parameter is not allowed") + return "", common.NewError(err) + } + + queryStr := qparams[0] + if !Contains(common.SupportedPokeRoutes, queryStr) { + err := fmt.Errorf("invalid query parameter: %v", queryStr) + return "", common.NewError(err) + + } + return queryStr, nil + } + + // return default + return "primary", nil +} + +func ValidateQueryParams(r *http.Request, fields log.Fields) error { + groupIdValues, ok := r.URL.Query()["group_id"] + if !ok || len(groupIdValues) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + fields["group_id"] = groupIdValues[0] + r.Header.Set(common.HeaderDocName, groupIdValues[0]) + + subdocIds := strings.Split(groupIdValues[0], ",") + if len(subdocIds) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + if len(subdocIds) > 0 && subdocIds[0] != "root" { + return common.NewError(common.ErrInvalidQueryParams) + } + + for _, subdocId := range subdocIds[1:] { + if _, ok := common.SubdocBitIndexMap[subdocId]; !ok { + return common.NewError(common.ErrInvalidQueryParams) + } + } + + ifNoneMatch := r.Header.Get(common.HeaderIfNoneMatch) + if len(ifNoneMatch) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + versions := strings.Split(ifNoneMatch, ",") + if len(versions) != len(subdocIds) { + return common.NewError(common.ErrInvalidQueryParams) + } + return nil +} diff --git a/util/validator_test.go b/util/validator_test.go new file mode 100644 index 0000000..9109b82 --- /dev/null +++ b/util/validator_test.go @@ -0,0 +1,174 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package util + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestValidateMac(t *testing.T) { + mac := "001122334455" + assert.Assert(t, ValidateMac(mac)) + + mac = "4444ABCDEF01" + assert.Assert(t, ValidateMac(mac)) + + mac = "00112233445Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "001122334455Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "0H1122334455" + assert.Assert(t, !ValidateMac(mac)) + + for i := 0; i < 10; i++ { + mac := GenerateRandomCpeMac() + assert.Assert(t, ValidateMac(mac)) + } +} + +func TestValidatePokeQuery(t *testing.T) { + values := url.Values{} + + values["doc"] = []string{ + "primary,telemetry", + "hello,world", + } + _, err := ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,hello,world", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,telemetry", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary", + } + s, err := ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "telemetry", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "telemetry") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "primary", + } + values["route"] = []string{ + "mqtt", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "mqtt") +} + +func TestValidateQueryParams(t *testing.T) { + cpeMac := GenerateRandomCpeMac() + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + fields := make(log.Fields) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,lan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, fields) + assert.NilError(t, err) +} From f4b217e0a05153fdda395bf7d310f8e476311fff Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 17 Nov 2024 11:28:50 -0800 Subject: [PATCH 120/215] add token trust validation --- common/error.go | 1 + config/sample_webconfig.conf | 3 ++ http/multipart_test.go | 29 +++++++++++++-- http/webconfig_server.go | 30 +++++++++++++--- http/webconfig_server_test.go | 8 +++++ security/main_test.go | 6 ++-- security/token.go | 68 +++++++++++++++++++---------------- security/token_test.go | 22 +++++++----- 8 files changed, 119 insertions(+), 48 deletions(-) diff --git a/common/error.go b/common/error.go index fa22ddf..9a1f4d1 100644 --- a/common/error.go +++ b/common/error.go @@ -34,6 +34,7 @@ var ( ErrNotOK = fmt.Errorf("!ok") ErrRootDocumentLocked = fmt.Errorf("root document is locked") ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") + ErrLowTrust = fmt.Errorf("token trust is lower than threshold") ) type Http400Error struct { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index d32fcff..e7bcdea 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -262,4 +262,7 @@ webconfig { // only valid query_params formats are accepted query_params_validation_enabled = false + + // only devices using tokens with trust higher or equal can GET the full document + min_trust = 0 } diff --git a/http/multipart_test.go b/http/multipart_test.go index ce3b84d..3d26e81 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -253,7 +253,7 @@ func TestCpeMiddleware(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router1).Result() - rbytes, err = io.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusForbidden) @@ -264,7 +264,32 @@ func TestCpeMiddleware(t *testing.T) { assert.NilError(t, err) req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) res = ExecuteRequest(req, router1).Result() - rbytes, err = io.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // change the min trust to 1000 + server1.SetMinTrust(1000) + assert.Equal(t, 1000, server1.MinTrust()) + zeroToken := server1.Generate(cpeMac, 86400, 0) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + + // change the min trust back to 0 + server1.SetMinTrust(0) + assert.Equal(t, 0, server1.MinTrust()) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index b1875b9..9cc3aca 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -120,6 +120,7 @@ type WebconfigServer struct { kafkaProducerTopic string upstreamProfilesEnabled bool queryParamsValidationEnabled bool + minTrust int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -204,12 +205,14 @@ func GetDatabaseClient(sc *common.ServerConfig) db.DatabaseClient { func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer { conf := sc.Config var dbclient db.DatabaseClient + var tokenManager *security.TokenManager // setup up database client if testOnly { dbclient = GetTestDatabaseClient(sc) } else { dbclient = GetDatabaseClient(sc) + tokenManager = security.NewTokenManager(conf) } // setup jwks manager @@ -288,6 +291,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + minTrust := int(conf.GetInt32("webconfig.min_trust")) ws := &WebconfigServer{ Server: &http.Server{ @@ -296,7 +300,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer WriteTimeout: time.Duration(conf.GetInt32("webconfig.server.write_timeout_in_secs", 3)) * time.Second, }, DatabaseClient: dbclient, - TokenManager: security.NewTokenManager(conf), + TokenManager: tokenManager, JwksManager: jwksManager, ServerConfig: sc, WebpaConnector: NewWebpaConnector(conf, tlsConfig), @@ -325,6 +329,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerTopic: kafkaProducerTopic, upstreamProfilesEnabled: upstreamProfilesEnabled, queryParamsValidationEnabled: queryParamsValidationEnabled, + minTrust: minTrust, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -390,14 +395,21 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { return } - if ok, partnerId, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true + fields := xw.Audit() + fields["src_partner"] = partnerId + fields["trust"] = trust + if err := s.ValidatePartner(partnerId); err != nil { - fields := xw.Audit() - fields["src_partner"] = partnerId + // isValid = false partnerId = "unknown" tokenErr = common.NewError(err) } + if trust < s.MinTrust() { + isValid = false + tokenErr = common.NewError(common.ErrLowTrust) + } xw.SetPartnerId(partnerId) } else { tokenErr = common.NewError(err) @@ -475,7 +487,7 @@ func (s *WebconfigServer) TestingCpeMiddleware(next http.Handler) http.Handler { return } - if ok, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, _, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true } } @@ -657,6 +669,14 @@ func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { s.queryParamsValidationEnabled = enabled } +func (s *WebconfigServer) MinTrust() int { + return s.minTrust +} + +func (s *WebconfigServer) SetMinTrust(trust int) { + s.minTrust = trust +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 404c3eb..fd4c138 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -113,4 +113,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { enabled = false server.SetQueryParamsValidationEnabled(enabled) assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + + //configure trust level + trust := 1000 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + trust = 500 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) } diff --git a/security/main_test.go b/security/main_test.go index b1223c8..d822e54 100644 --- a/security/main_test.go +++ b/security/main_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -39,7 +39,9 @@ func TestMain(m *testing.M) { NewTestCodec(sc.Config) - tokenManager = NewTokenManager(sc.Config) + if sc.Config.GetBoolean("webconfig.jwt.enabled", false) || os.Getenv("TOKEN_TEST") == "1" { + tokenManager = NewTokenManager(sc.Config) + } log.SetOutput(io.Discard) returnCode := m.Run() diff --git a/security/token.go b/security/token.go index 169cd0f..47270a4 100644 --- a/security/token.go +++ b/security/token.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -26,11 +26,11 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -42,13 +42,13 @@ type ThemisClaims struct { Mac string `json:"mac"` PartnerId string `json:"partner-id"` Serial string `json:"serial"` - Trust string `json:"trust"` + Trust int `json:"trust"` Uuid string `json:"uuid"` Capabilities []string `json:"capabilities"` jwt.RegisteredClaims } -type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, error) +type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, int, error) type TokenManager struct { encodeKey *rsa.PrivateKey @@ -61,11 +61,6 @@ type TokenManager struct { } func NewTokenManager(conf *configuration.Config) *TokenManager { - jwtEnabled := conf.GetBoolean("webconfig.jwt.enabled", false) - if !jwtEnabled { - return nil - } - panicExitEnabled := conf.GetBoolean("webconfig.panic_exit_enabled", false) // prepare args for TokenManager @@ -134,19 +129,25 @@ func loadEncodeKey(keyfile string) (*rsa.PrivateKey, error) { } // TODO this is not an officially supported function. -func (m *TokenManager) Generate(mac string, ttl int64, vargs ...string) string { +func (m *TokenManager) Generate(mac string, ttl int64, itfs ...interface{}) string { // %% NOTE mac should be lowercase to be consistent with reference doc // static themis fields copied from examples in the webconfig confluence kid := "webconfig_key" serial := "ABCNDGE" - trust := "1000" + trust := 1000 capUuid := "1234567891234" capabilities := []string{"x1:issuer:test:.*:all"} - partner := "comcast" - if len(vargs) > 0 { - partner = vargs[0] + + for _, itf := range itfs { + switch ty := itf.(type) { + case string: + partner = ty + case int: + trust = ty + } } + utcnow := time.Now() claims := ThemisClaims{ @@ -224,19 +225,19 @@ func ParseKidFromTokenHeader(tokenString string) (string, error) { } func (m *TokenManager) VerifyApiToken(token string) (bool, error) { - ok, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) + ok, _, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) if err != nil { return ok, common.NewError(err) } return ok, err } -func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, error) { - ok, partner, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) +func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, int, error) { + ok, partner, trust, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) if err != nil { - return ok, "", common.NewError(err) + return ok, "", trust, common.NewError(err) } - return ok, partner, nil + return ok, partner, trust, nil } func (m *TokenManager) SetVerifyFunc(fn VerifyFunc) { @@ -293,9 +294,10 @@ func (m *TokenManager) ParseCpeToken(tokenStr string) (map[string]string, error) return data, nil } -func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, error) { +func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, int, error) { tokenString := vargs[0] var kid string + var trust int parser := &jwt.Parser{} @@ -305,11 +307,11 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi // check kid rawkid, ok := token.Header["kid"] if !ok { - return false, "", common.NewError(fmt.Errorf("missing kid in token")) + return false, "", trust, common.NewError(fmt.Errorf("missing kid in token")) } kid, ok = rawkid.(string) if !ok { - return false, "", common.NewError(fmt.Errorf("error in reading kid from header")) + return false, "", trust, common.NewError(fmt.Errorf("error in reading kid from header")) } ok = false @@ -320,7 +322,7 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !ok { - return false, "", common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) + return false, "", trust, common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) } // check capabilities, if requiredCapabilities is nonempty @@ -343,21 +345,21 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !isCapable { - return false, "", common.NewError(fmt.Errorf("token without proper capabilities")) + return false, "", trust, common.NewError(fmt.Errorf("token without proper capabilities")) } } } else { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } decodeKey, ok := decodeKeys[kid] if !ok { - return false, "", common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) + return false, "", trust, common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) } claims := jwt.MapClaims{} if _, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) { return decodeKey, nil }); err != nil { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } if len(vargs) > 1 { @@ -369,12 +371,12 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi if strings.ToLower(mac) == strings.ToLower(macstr) { isMatched = true } else { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) } } } if !isMatched { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) } } @@ -384,5 +386,9 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi partner = itf.(string) } - return true, partner, nil + if itf, ok := claims["trust"]; ok { + trust = util.ToInt(itf) + } + + return true, partner, trust, nil } diff --git a/security/token_test.go b/security/token_test.go index 5f6bab5..6f73aa6 100644 --- a/security/token_test.go +++ b/security/token_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -55,11 +55,7 @@ func TestLoadingKeyFiles(t *testing.T) { } func TestTokenValidation(t *testing.T) { - sc, err := common.GetTestServerConfig() - if err != nil { - panic(err) - } - if !sc.GetBoolean("webconfig.jwt.enabled") { + if tokenManager == nil { t.Skip("webconfig.jwt.enabled = false") } @@ -67,16 +63,26 @@ func TestTokenValidation(t *testing.T) { token := tokenManager.Generate(strings.ToLower(cpeMac), 86400) // default comcast - ok, parsedPartner, err := tokenManager.VerifyCpeToken(token, cpeMac) + ok, parsedPartner, trust, err := tokenManager.VerifyCpeToken(token, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, "comcast") + assert.Equal(t, trust, 1000) // create a partner token partner1 := "cox" token1 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1) - ok, parsedPartner, err = tokenManager.VerifyCpeToken(token1, cpeMac) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token1, cpeMac) + assert.NilError(t, err) + assert.Assert(t, ok) + assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 1000) + + // create a partner token with non-default trust + token2 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1, 500) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token2, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 500) } From 07e4e16dcea444ad7cf0fd54700fa55f5ea8f7a6 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 16 Nov 2024 10:42:36 -0800 Subject: [PATCH 121/215] cherrypick strict format check and resolve conflicts --- common/error.go | 1 + config/sample_webconfig.conf | 3 + db/service.go | 10 +- http/multipart.go | 15 ++- http/multipart_test.go | 171 +++++++++++++++++++++++++++++++++ http/webconfig_server.go | 12 +++ http/webconfig_server_test.go | 10 +- util/string.go | 61 ------------ util/string_test.go | 80 ---------------- util/validator.go | 124 ++++++++++++++++++++++++ util/validator_test.go | 174 ++++++++++++++++++++++++++++++++++ 11 files changed, 513 insertions(+), 148 deletions(-) create mode 100644 util/validator.go create mode 100644 util/validator_test.go diff --git a/common/error.go b/common/error.go index 18b207f..fa22ddf 100644 --- a/common/error.go +++ b/common/error.go @@ -33,6 +33,7 @@ const ( var ( ErrNotOK = fmt.Errorf("!ok") ErrRootDocumentLocked = fmt.Errorf("root document is locked") + ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") ) type Http400Error struct { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 22dacea..00b2f03 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -256,4 +256,7 @@ webconfig { // this allows the root document locked if needed lock_root_document_enabled = false + + // only valid query_params formats are accepted + query_params_validation_enabled = false } diff --git a/db/service.go b/db/service.go index 0db393f..22c9b9b 100644 --- a/db/service.go +++ b/db/service.go @@ -231,8 +231,14 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } // eval if the root_document is locked - if c.LockRootDocumentEnabled() && cloudRootDocument.Locked() { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } else { + tfields := common.FilterLogFields(fields) + tfields["logger"] = "rootdoc" + log.WithFields(tfields).Warn("dryrun409") + } } switch rootCmpEnum { diff --git a/http/multipart.go b/http/multipart.go index d6c29b5..09ed8c1 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -75,11 +75,18 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. return } fields := xw.Audit() - fields["cpe_mac"] = mac - if qGroupIds, ok := r.URL.Query()["group_id"]; ok { - fields["group_id"] = qGroupIds[0] - r.Header.Set(common.HeaderDocName, qGroupIds[0]) + + // enforce strict query parameters check + err := util.ValidateQueryParams(r, fields) + if err != nil && s.QueryParamsValidationEnabled() { + if errors.Is(err, common.ErrInvalidQueryParams) { + Error(w, http.StatusBadRequest, nil) + log.WithFields(fields).Error(err) + return + } + Error(w, http.StatusInternalServerError, err) + return } // handle empty schema version header diff --git a/http/multipart_test.go b/http/multipart_test.go index abfed54..ce3b84d 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1308,3 +1308,174 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { _, ok = mpartMap["privatessid"] assert.Assert(t, !ok) } + +func TestValidateQueryParams(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetQueryParamsValidationEnabled(true) + assert.Assert(t, server.QueryParamsValidationEnabled()) + defer server.SetQueryParamsValidationEnabled(false) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123") + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + etag := res.Header.Get(common.HeaderEtag) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanMpartVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanMpartVersion := mpart.Version + matchedIfNoneMatch := fmt.Sprintf("%v,%v,%v", etag, lanMpartVersion, wanMpartVersion) + + // case 9 versions matched 304 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, matchedIfNoneMatch) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index a7e32c8..06dfe11 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -118,6 +118,7 @@ type WebconfigServer struct { webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string + queryParamsValidationEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -284,6 +285,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } } + queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -318,6 +321,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, + queryParamsValidationEnabled: queryParamsValidationEnabled, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -634,6 +638,14 @@ func (s *WebconfigServer) SetKafkaProducerTopic(x string) { s.kafkaProducerTopic = x } +func (s *WebconfigServer) QueryParamsValidationEnabled() bool { + return s.queryParamsValidationEnabled +} + +func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { + s.queryParamsValidationEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 1699106..62224a6 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -97,4 +97,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { validPartners = []string{"name3", "name4", "name5"} server.SetValidPartners(validPartners) assert.DeepEqual(t, server.ValidPartners(), validPartners) + + // enforce strict query parameters validation + enabled = true + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + enabled = false + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) } diff --git a/util/string.go b/util/string.go index 439b0bb..bd1eac4 100644 --- a/util/string.go +++ b/util/string.go @@ -53,18 +53,6 @@ func GenerateRandomCpeMac() string { return strings.ToUpper(u[len(u)-12:]) } -func ValidateMac(mac string) bool { - if len(mac) != 12 { - return false - } - for _, r := range mac { - if r < 48 || r > 70 || (r > 57 && r < 65) { - return false - } - } - return true -} - func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId string) string { // build the query parameters in a fixed order params := []string{} @@ -121,55 +109,6 @@ func GetMacDiff(wanMac, mac string) int { return wanMacVal - macVal } -func ValidatePokeQuery(values url.Values) (string, error) { - // handle ?doc=xxx - if docQueryParamStrs, ok := values["doc"]; ok { - if len(docQueryParamStrs) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(docQueryParamStrs[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeDocs, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // handle ?route=xxx - if qparams, ok := values["route"]; ok { - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(qparams[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeRoutes, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // return default - return "primary", nil -} - func GetEstbMacAddress(mac string) string { // if the mac cannot be parsed, then return back the input i, err := strconv.ParseInt(mac, 16, 64) diff --git a/util/string_test.go b/util/string_test.go index cf6a894..8fb5288 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -19,7 +19,6 @@ package util import ( "net/http" - "net/url" "testing" "github.com/rdkcentral/webconfig/common" @@ -33,28 +32,6 @@ func TestString(t *testing.T) { assert.Equal(t, c, expected) } -func TestValidateMac(t *testing.T) { - mac := "001122334455" - assert.Assert(t, ValidateMac(mac)) - - mac = "4444ABCDEF01" - assert.Assert(t, ValidateMac(mac)) - - mac = "00112233445Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "001122334455Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "0H1122334455" - assert.Assert(t, !ValidateMac(mac)) - - for i := 0; i < 10; i++ { - mac := GenerateRandomCpeMac() - assert.Assert(t, ValidateMac(mac)) - } -} - func TestGetAuditId(t *testing.T) { auditId := GetAuditId() assert.Equal(t, len(auditId), 32) @@ -80,63 +57,6 @@ func TestTelemetryQuery(t *testing.T) { assert.Equal(t, qstr, expected) } -func TestValidatePokeQuery(t *testing.T) { - values := url.Values{} - - values["doc"] = []string{ - "primary,telemetry", - "hello,world", - } - _, err := ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,hello,world", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,telemetry", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary", - } - s, err := ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "telemetry", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "telemetry") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "primary", - } - values["route"] = []string{ - "mqtt", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "mqtt") -} - func TestIsValidUTF8(t *testing.T) { b1 := []byte(`{"foo":"bar","hello":123,"world":true}`) assert.Assert(t, IsValidUTF8(b1)) diff --git a/util/validator.go b/util/validator.go new file mode 100644 index 0000000..4ac602c --- /dev/null +++ b/util/validator.go @@ -0,0 +1,124 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package util + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" +) + +func ValidateMac(mac string) bool { + if len(mac) != 12 { + return false + } + for _, r := range mac { + if r < 48 || r > 70 || (r > 57 && r < 65) { + return false + } + } + return true +} + +func ValidatePokeQuery(values url.Values) (string, error) { + // handle ?doc=xxx + if docQueryParamStrs, ok := values["doc"]; ok { + if len(docQueryParamStrs) > 1 { + err := fmt.Errorf("multiple doc parameter is not allowed") + return "", common.NewError(err) + } + + qparams := strings.Split(docQueryParamStrs[0], ",") + if len(qparams) > 1 { + err := fmt.Errorf("multiple doc parameter is not allowed") + return "", common.NewError(err) + } + + queryStr := qparams[0] + if !Contains(common.SupportedPokeDocs, queryStr) { + err := fmt.Errorf("invalid query parameter: %v", queryStr) + return "", common.NewError(err) + + } + return queryStr, nil + } + + // handle ?route=xxx + if qparams, ok := values["route"]; ok { + if len(qparams) > 1 { + err := fmt.Errorf("multiple route parameter is not allowed") + return "", common.NewError(err) + } + + qparams := strings.Split(qparams[0], ",") + if len(qparams) > 1 { + err := fmt.Errorf("multiple route parameter is not allowed") + return "", common.NewError(err) + } + + queryStr := qparams[0] + if !Contains(common.SupportedPokeRoutes, queryStr) { + err := fmt.Errorf("invalid query parameter: %v", queryStr) + return "", common.NewError(err) + + } + return queryStr, nil + } + + // return default + return "primary", nil +} + +func ValidateQueryParams(r *http.Request, fields log.Fields) error { + groupIdValues, ok := r.URL.Query()["group_id"] + if !ok || len(groupIdValues) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + fields["group_id"] = groupIdValues[0] + r.Header.Set(common.HeaderDocName, groupIdValues[0]) + + subdocIds := strings.Split(groupIdValues[0], ",") + if len(subdocIds) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + if len(subdocIds) > 0 && subdocIds[0] != "root" { + return common.NewError(common.ErrInvalidQueryParams) + } + + for _, subdocId := range subdocIds[1:] { + if _, ok := common.SubdocBitIndexMap[subdocId]; !ok { + return common.NewError(common.ErrInvalidQueryParams) + } + } + + ifNoneMatch := r.Header.Get(common.HeaderIfNoneMatch) + if len(ifNoneMatch) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + versions := strings.Split(ifNoneMatch, ",") + if len(versions) != len(subdocIds) { + return common.NewError(common.ErrInvalidQueryParams) + } + return nil +} diff --git a/util/validator_test.go b/util/validator_test.go new file mode 100644 index 0000000..9109b82 --- /dev/null +++ b/util/validator_test.go @@ -0,0 +1,174 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package util + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestValidateMac(t *testing.T) { + mac := "001122334455" + assert.Assert(t, ValidateMac(mac)) + + mac = "4444ABCDEF01" + assert.Assert(t, ValidateMac(mac)) + + mac = "00112233445Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "001122334455Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "0H1122334455" + assert.Assert(t, !ValidateMac(mac)) + + for i := 0; i < 10; i++ { + mac := GenerateRandomCpeMac() + assert.Assert(t, ValidateMac(mac)) + } +} + +func TestValidatePokeQuery(t *testing.T) { + values := url.Values{} + + values["doc"] = []string{ + "primary,telemetry", + "hello,world", + } + _, err := ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,hello,world", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,telemetry", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary", + } + s, err := ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "telemetry", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "telemetry") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "primary", + } + values["route"] = []string{ + "mqtt", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "mqtt") +} + +func TestValidateQueryParams(t *testing.T) { + cpeMac := GenerateRandomCpeMac() + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + fields := make(log.Fields) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,lan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, fields) + assert.NilError(t, err) +} From ddbbb08dc410d9ded8daa4e7b70b85b61c35d020 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 19 Nov 2024 22:25:55 -0800 Subject: [PATCH 122/215] accept non bitmap subdocs when the validator is enabled during device get route --- config/sample_webconfig.conf | 3 +++ http/multipart.go | 2 +- http/webconfig_server.go | 16 ++++++++++++++++ http/webconfig_server_test.go | 10 ++++++++++ util/validator.go | 6 +++--- util/validator_test.go | 30 +++++++++++++++++++++--------- 6 files changed, 54 insertions(+), 13 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index e7bcdea..6d2b232 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -265,4 +265,7 @@ webconfig { // only devices using tokens with trust higher or equal can GET the full document min_trust = 0 + + // subdoc ids accepted by the validator even if they are without bitmap definition + valid_subdoc_ids = [] } diff --git a/http/multipart.go b/http/multipart.go index 09ed8c1..5e39531 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -78,7 +78,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. fields["cpe_mac"] = mac // enforce strict query parameters check - err := util.ValidateQueryParams(r, fields) + err := util.ValidateQueryParams(r, s.ValidSubdocIdMap(), fields) if err != nil && s.QueryParamsValidationEnabled() { if errors.Is(err, common.ErrInvalidQueryParams) { Error(w, http.StatusBadRequest, nil) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 9cc3aca..4cb161a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -26,6 +26,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "os" "strings" @@ -121,6 +122,7 @@ type WebconfigServer struct { upstreamProfilesEnabled bool queryParamsValidationEnabled bool minTrust int + validSubdocIdMap map[string]int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -292,6 +294,11 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") minTrust := int(conf.GetInt32("webconfig.min_trust")) + validSubdocIds := conf.GetStringList("webconfig.valid_subdoc_ids") + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + for _, x := range validSubdocIds { + validSubdocIdMap[x] = 1 + } ws := &WebconfigServer{ Server: &http.Server{ @@ -330,6 +337,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer upstreamProfilesEnabled: upstreamProfilesEnabled, queryParamsValidationEnabled: queryParamsValidationEnabled, minTrust: minTrust, + validSubdocIdMap: validSubdocIdMap, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -677,6 +685,14 @@ func (s *WebconfigServer) SetMinTrust(trust int) { s.minTrust = trust } +func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { + return s.validSubdocIdMap +} + +func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { + s.validSubdocIdMap = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index fd4c138..0c028aa 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -121,4 +121,14 @@ func TestWebconfigServerSetterGetter(t *testing.T) { trust = 500 server.SetMinTrust(trust) assert.Equal(t, server.MinTrust(), trust) + + validSubdocIdMap := map[string]int{ + "red": 1, + "orange": 2, + "yellow": 3, + "green": 4, + } + server.SetValidSubdocIdMap(validSubdocIdMap) + assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) + } diff --git a/util/validator.go b/util/validator.go index 4ac602c..754044f 100644 --- a/util/validator.go +++ b/util/validator.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -88,7 +88,7 @@ func ValidatePokeQuery(values url.Values) (string, error) { return "primary", nil } -func ValidateQueryParams(r *http.Request, fields log.Fields) error { +func ValidateQueryParams(r *http.Request, validSubdocIdMap map[string]int, fields log.Fields) error { groupIdValues, ok := r.URL.Query()["group_id"] if !ok || len(groupIdValues) == 0 { return common.NewError(common.ErrInvalidQueryParams) @@ -106,7 +106,7 @@ func ValidateQueryParams(r *http.Request, fields log.Fields) error { } for _, subdocId := range subdocIds[1:] { - if _, ok := common.SubdocBitIndexMap[subdocId]; !ok { + if _, ok := validSubdocIdMap[subdocId]; !ok { return common.NewError(common.ErrInvalidQueryParams) } } diff --git a/util/validator_test.go b/util/validator_test.go index 9109b82..4313f03 100644 --- a/util/validator_test.go +++ b/util/validator_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( "errors" "fmt" + "maps" "net/http" "net/url" "testing" @@ -110,34 +111,37 @@ func TestValidatePokeQuery(t *testing.T) { func TestValidateQueryParams(t *testing.T) { cpeMac := GenerateRandomCpeMac() + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + validSubdocIdMap["red"] = 1 + validSubdocIdMap["orange"] = 1 // case 1 deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) req, err := http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) fields := make(log.Fields) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 2 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 3 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 4 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 5 @@ -145,7 +149,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 6 @@ -153,7 +157,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 7 @@ -161,7 +165,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 8 @@ -169,6 +173,14 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.NilError(t, err) + + // case 9 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,red,orange", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456,678") + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.NilError(t, err) } From 7ccaf8cdea3ec402b7a21100514154ee57458150 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 19 Nov 2024 22:25:55 -0800 Subject: [PATCH 123/215] cherrypick non bitmap validation and resolve conflicts --- config/sample_webconfig.conf | 3 +++ http/multipart.go | 2 +- http/webconfig_server.go | 16 ++++++++++++++++ http/webconfig_server_test.go | 9 +++++++++ util/validator.go | 6 +++--- util/validator_test.go | 30 +++++++++++++++++++++--------- 6 files changed, 53 insertions(+), 13 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 00b2f03..013f180 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -259,4 +259,7 @@ webconfig { // only valid query_params formats are accepted query_params_validation_enabled = false + + // subdoc ids accepted by the validator even if they are without bitmap definition + valid_subdoc_ids = [] } diff --git a/http/multipart.go b/http/multipart.go index 09ed8c1..5e39531 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -78,7 +78,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. fields["cpe_mac"] = mac // enforce strict query parameters check - err := util.ValidateQueryParams(r, fields) + err := util.ValidateQueryParams(r, s.ValidSubdocIdMap(), fields) if err != nil && s.QueryParamsValidationEnabled() { if errors.Is(err, common.ErrInvalidQueryParams) { Error(w, http.StatusBadRequest, nil) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 06dfe11..7397acd 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -26,6 +26,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "os" "strings" @@ -119,6 +120,7 @@ type WebconfigServer struct { kafkaProducerEnabled bool kafkaProducerTopic string queryParamsValidationEnabled bool + validSubdocIdMap map[string]int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -286,6 +288,11 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + validSubdocIds := conf.GetStringList("webconfig.valid_subdoc_ids") + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + for _, x := range validSubdocIds { + validSubdocIdMap[x] = 1 + } ws := &WebconfigServer{ Server: &http.Server{ @@ -322,6 +329,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, queryParamsValidationEnabled: queryParamsValidationEnabled, + validSubdocIdMap: validSubdocIdMap, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -646,6 +654,14 @@ func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { s.queryParamsValidationEnabled = enabled } +func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { + return s.validSubdocIdMap +} + +func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { + s.validSubdocIdMap = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 62224a6..74ee523 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -105,4 +105,13 @@ func TestWebconfigServerSetterGetter(t *testing.T) { enabled = false server.SetQueryParamsValidationEnabled(enabled) assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + + validSubdocIdMap := map[string]int{ + "red": 1, + "orange": 2, + "yellow": 3, + "green": 4, + } + server.SetValidSubdocIdMap(validSubdocIdMap) + assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) } diff --git a/util/validator.go b/util/validator.go index 4ac602c..754044f 100644 --- a/util/validator.go +++ b/util/validator.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -88,7 +88,7 @@ func ValidatePokeQuery(values url.Values) (string, error) { return "primary", nil } -func ValidateQueryParams(r *http.Request, fields log.Fields) error { +func ValidateQueryParams(r *http.Request, validSubdocIdMap map[string]int, fields log.Fields) error { groupIdValues, ok := r.URL.Query()["group_id"] if !ok || len(groupIdValues) == 0 { return common.NewError(common.ErrInvalidQueryParams) @@ -106,7 +106,7 @@ func ValidateQueryParams(r *http.Request, fields log.Fields) error { } for _, subdocId := range subdocIds[1:] { - if _, ok := common.SubdocBitIndexMap[subdocId]; !ok { + if _, ok := validSubdocIdMap[subdocId]; !ok { return common.NewError(common.ErrInvalidQueryParams) } } diff --git a/util/validator_test.go b/util/validator_test.go index 9109b82..4313f03 100644 --- a/util/validator_test.go +++ b/util/validator_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( "errors" "fmt" + "maps" "net/http" "net/url" "testing" @@ -110,34 +111,37 @@ func TestValidatePokeQuery(t *testing.T) { func TestValidateQueryParams(t *testing.T) { cpeMac := GenerateRandomCpeMac() + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + validSubdocIdMap["red"] = 1 + validSubdocIdMap["orange"] = 1 // case 1 deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) req, err := http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) fields := make(log.Fields) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 2 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 3 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 4 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 5 @@ -145,7 +149,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 6 @@ -153,7 +157,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 7 @@ -161,7 +165,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 8 @@ -169,6 +173,14 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.NilError(t, err) + + // case 9 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,red,orange", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456,678") + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.NilError(t, err) } From 9a1c893a53b9fc636af1c29a5c04ecd2f70d7da0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 17 Nov 2024 11:28:50 -0800 Subject: [PATCH 124/215] cherrypick the feature to validate token trust value and resolve conflicts --- common/error.go | 1 + config/sample_webconfig.conf | 3 ++ http/multipart_test.go | 29 +++++++++++++-- http/webconfig_server.go | 30 +++++++++++++--- http/webconfig_server_test.go | 8 +++++ security/main_test.go | 6 ++-- security/token.go | 68 +++++++++++++++++++---------------- security/token_test.go | 22 +++++++----- 8 files changed, 119 insertions(+), 48 deletions(-) diff --git a/common/error.go b/common/error.go index fa22ddf..9a1f4d1 100644 --- a/common/error.go +++ b/common/error.go @@ -34,6 +34,7 @@ var ( ErrNotOK = fmt.Errorf("!ok") ErrRootDocumentLocked = fmt.Errorf("root document is locked") ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") + ErrLowTrust = fmt.Errorf("token trust is lower than threshold") ) type Http400Error struct { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 013f180..0d0b4ed 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -262,4 +262,7 @@ webconfig { // subdoc ids accepted by the validator even if they are without bitmap definition valid_subdoc_ids = [] + + // only devices using tokens with trust higher or equal can GET the full document + min_trust = 0 } diff --git a/http/multipart_test.go b/http/multipart_test.go index ce3b84d..3d26e81 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -253,7 +253,7 @@ func TestCpeMiddleware(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router1).Result() - rbytes, err = io.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusForbidden) @@ -264,7 +264,32 @@ func TestCpeMiddleware(t *testing.T) { assert.NilError(t, err) req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) res = ExecuteRequest(req, router1).Result() - rbytes, err = io.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // change the min trust to 1000 + server1.SetMinTrust(1000) + assert.Equal(t, 1000, server1.MinTrust()) + zeroToken := server1.Generate(cpeMac, 86400, 0) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + + // change the min trust back to 0 + server1.SetMinTrust(0) + assert.Equal(t, 0, server1.MinTrust()) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 7397acd..01b4ed0 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -121,6 +121,7 @@ type WebconfigServer struct { kafkaProducerTopic string queryParamsValidationEnabled bool validSubdocIdMap map[string]int + minTrust int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -205,12 +206,14 @@ func GetDatabaseClient(sc *common.ServerConfig) db.DatabaseClient { func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer { conf := sc.Config var dbclient db.DatabaseClient + var tokenManager *security.TokenManager // setup up database client if testOnly { dbclient = GetTestDatabaseClient(sc) } else { dbclient = GetDatabaseClient(sc) + tokenManager = security.NewTokenManager(conf) } // setup jwks manager @@ -293,6 +296,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer for _, x := range validSubdocIds { validSubdocIdMap[x] = 1 } + minTrust := int(conf.GetInt32("webconfig.min_trust")) ws := &WebconfigServer{ Server: &http.Server{ @@ -301,7 +305,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer WriteTimeout: time.Duration(conf.GetInt32("webconfig.server.write_timeout_in_secs", 3)) * time.Second, }, DatabaseClient: dbclient, - TokenManager: security.NewTokenManager(conf), + TokenManager: tokenManager, JwksManager: jwksManager, ServerConfig: sc, WebpaConnector: NewWebpaConnector(conf, tlsConfig), @@ -330,6 +334,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerTopic: kafkaProducerTopic, queryParamsValidationEnabled: queryParamsValidationEnabled, validSubdocIdMap: validSubdocIdMap, + minTrust: minTrust, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -395,14 +400,21 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { return } - if ok, partnerId, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true + fields := xw.Audit() + fields["src_partner"] = partnerId + fields["trust"] = trust + if err := s.ValidatePartner(partnerId); err != nil { - fields := xw.Audit() - fields["src_partner"] = partnerId + // isValid = false partnerId = "unknown" tokenErr = common.NewError(err) } + if trust < s.MinTrust() { + isValid = false + tokenErr = common.NewError(common.ErrLowTrust) + } xw.SetPartnerId(partnerId) } else { tokenErr = common.NewError(err) @@ -480,7 +492,7 @@ func (s *WebconfigServer) TestingCpeMiddleware(next http.Handler) http.Handler { return } - if ok, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, _, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true } } @@ -662,6 +674,14 @@ func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { s.validSubdocIdMap = x } +func (s *WebconfigServer) MinTrust() int { + return s.minTrust +} + +func (s *WebconfigServer) SetMinTrust(trust int) { + s.minTrust = trust +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 74ee523..37a7a5f 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -114,4 +114,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { } server.SetValidSubdocIdMap(validSubdocIdMap) assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) + + //configure trust level + trust := 1000 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + trust = 500 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) } diff --git a/security/main_test.go b/security/main_test.go index b1223c8..d822e54 100644 --- a/security/main_test.go +++ b/security/main_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -39,7 +39,9 @@ func TestMain(m *testing.M) { NewTestCodec(sc.Config) - tokenManager = NewTokenManager(sc.Config) + if sc.Config.GetBoolean("webconfig.jwt.enabled", false) || os.Getenv("TOKEN_TEST") == "1" { + tokenManager = NewTokenManager(sc.Config) + } log.SetOutput(io.Discard) returnCode := m.Run() diff --git a/security/token.go b/security/token.go index 169cd0f..47270a4 100644 --- a/security/token.go +++ b/security/token.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -26,11 +26,11 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -42,13 +42,13 @@ type ThemisClaims struct { Mac string `json:"mac"` PartnerId string `json:"partner-id"` Serial string `json:"serial"` - Trust string `json:"trust"` + Trust int `json:"trust"` Uuid string `json:"uuid"` Capabilities []string `json:"capabilities"` jwt.RegisteredClaims } -type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, error) +type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, int, error) type TokenManager struct { encodeKey *rsa.PrivateKey @@ -61,11 +61,6 @@ type TokenManager struct { } func NewTokenManager(conf *configuration.Config) *TokenManager { - jwtEnabled := conf.GetBoolean("webconfig.jwt.enabled", false) - if !jwtEnabled { - return nil - } - panicExitEnabled := conf.GetBoolean("webconfig.panic_exit_enabled", false) // prepare args for TokenManager @@ -134,19 +129,25 @@ func loadEncodeKey(keyfile string) (*rsa.PrivateKey, error) { } // TODO this is not an officially supported function. -func (m *TokenManager) Generate(mac string, ttl int64, vargs ...string) string { +func (m *TokenManager) Generate(mac string, ttl int64, itfs ...interface{}) string { // %% NOTE mac should be lowercase to be consistent with reference doc // static themis fields copied from examples in the webconfig confluence kid := "webconfig_key" serial := "ABCNDGE" - trust := "1000" + trust := 1000 capUuid := "1234567891234" capabilities := []string{"x1:issuer:test:.*:all"} - partner := "comcast" - if len(vargs) > 0 { - partner = vargs[0] + + for _, itf := range itfs { + switch ty := itf.(type) { + case string: + partner = ty + case int: + trust = ty + } } + utcnow := time.Now() claims := ThemisClaims{ @@ -224,19 +225,19 @@ func ParseKidFromTokenHeader(tokenString string) (string, error) { } func (m *TokenManager) VerifyApiToken(token string) (bool, error) { - ok, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) + ok, _, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) if err != nil { return ok, common.NewError(err) } return ok, err } -func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, error) { - ok, partner, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) +func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, int, error) { + ok, partner, trust, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) if err != nil { - return ok, "", common.NewError(err) + return ok, "", trust, common.NewError(err) } - return ok, partner, nil + return ok, partner, trust, nil } func (m *TokenManager) SetVerifyFunc(fn VerifyFunc) { @@ -293,9 +294,10 @@ func (m *TokenManager) ParseCpeToken(tokenStr string) (map[string]string, error) return data, nil } -func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, error) { +func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, int, error) { tokenString := vargs[0] var kid string + var trust int parser := &jwt.Parser{} @@ -305,11 +307,11 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi // check kid rawkid, ok := token.Header["kid"] if !ok { - return false, "", common.NewError(fmt.Errorf("missing kid in token")) + return false, "", trust, common.NewError(fmt.Errorf("missing kid in token")) } kid, ok = rawkid.(string) if !ok { - return false, "", common.NewError(fmt.Errorf("error in reading kid from header")) + return false, "", trust, common.NewError(fmt.Errorf("error in reading kid from header")) } ok = false @@ -320,7 +322,7 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !ok { - return false, "", common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) + return false, "", trust, common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) } // check capabilities, if requiredCapabilities is nonempty @@ -343,21 +345,21 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !isCapable { - return false, "", common.NewError(fmt.Errorf("token without proper capabilities")) + return false, "", trust, common.NewError(fmt.Errorf("token without proper capabilities")) } } } else { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } decodeKey, ok := decodeKeys[kid] if !ok { - return false, "", common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) + return false, "", trust, common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) } claims := jwt.MapClaims{} if _, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) { return decodeKey, nil }); err != nil { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } if len(vargs) > 1 { @@ -369,12 +371,12 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi if strings.ToLower(mac) == strings.ToLower(macstr) { isMatched = true } else { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) } } } if !isMatched { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) } } @@ -384,5 +386,9 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi partner = itf.(string) } - return true, partner, nil + if itf, ok := claims["trust"]; ok { + trust = util.ToInt(itf) + } + + return true, partner, trust, nil } diff --git a/security/token_test.go b/security/token_test.go index 5f6bab5..6f73aa6 100644 --- a/security/token_test.go +++ b/security/token_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -55,11 +55,7 @@ func TestLoadingKeyFiles(t *testing.T) { } func TestTokenValidation(t *testing.T) { - sc, err := common.GetTestServerConfig() - if err != nil { - panic(err) - } - if !sc.GetBoolean("webconfig.jwt.enabled") { + if tokenManager == nil { t.Skip("webconfig.jwt.enabled = false") } @@ -67,16 +63,26 @@ func TestTokenValidation(t *testing.T) { token := tokenManager.Generate(strings.ToLower(cpeMac), 86400) // default comcast - ok, parsedPartner, err := tokenManager.VerifyCpeToken(token, cpeMac) + ok, parsedPartner, trust, err := tokenManager.VerifyCpeToken(token, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, "comcast") + assert.Equal(t, trust, 1000) // create a partner token partner1 := "cox" token1 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1) - ok, parsedPartner, err = tokenManager.VerifyCpeToken(token1, cpeMac) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token1, cpeMac) + assert.NilError(t, err) + assert.Assert(t, ok) + assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 1000) + + // create a partner token with non-default trust + token2 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1, 500) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token2, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 500) } From 969249e182a5949784e3bc4ae8379babbf4f7e61 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 26 Nov 2024 15:55:42 -0800 Subject: [PATCH 125/215] log details for token errors --- http/webconfig_server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 4cb161a..9b4cb53 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -393,6 +393,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + fields := xw.Audit() authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { @@ -405,7 +406,6 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true - fields := xw.Audit() fields["src_partner"] = partnerId fields["trust"] = trust @@ -426,6 +426,10 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) } + if tokenErr != nil { + fields["error"] = tokenErr + } + if isValid { next.ServeHTTP(xw, r) } else { From 8200e8d3c829d1cae5f87de74dcb7a65ca988a42 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 26 Nov 2024 15:55:42 -0800 Subject: [PATCH 126/215] log details for token errors --- http/webconfig_server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 01b4ed0..dcd7e1e 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -390,6 +390,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + fields := xw.Audit() authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { @@ -402,7 +403,6 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true - fields := xw.Audit() fields["src_partner"] = partnerId fields["trust"] = trust @@ -423,6 +423,10 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) } + if tokenErr != nil { + fields["error"] = tokenErr + } + if isValid { next.ServeHTTP(xw, r) } else { From 884fff2d8bbac6f697fd024edc9894c29661b049 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 Nov 2024 17:32:40 -0800 Subject: [PATCH 127/215] cherrypick upstream profiles and merge conflicts --- config/sample_webconfig.conf | 9 +- http/supplementary_handler.go | 75 +++++++-- http/supplementary_handler_test.go | 241 ++++++++++++++++++++++++++++- http/upstream_connector.go | 37 ++++- http/webconfig_server.go | 33 ++-- http/webconfig_server_test.go | 23 ++- util/string.go | 43 ++++- util/string_test.go | 17 +- 8 files changed, 439 insertions(+), 39 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 0d0b4ed..6d2b232 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -257,12 +257,15 @@ webconfig { // this allows the root document locked if needed lock_root_document_enabled = false + // get extra profiles from upstream + upstream_profiles_enabled = false + // only valid query_params formats are accepted query_params_validation_enabled = false - // subdoc ids accepted by the validator even if they are without bitmap definition - valid_subdoc_ids = [] - // only devices using tokens with trust higher or equal can GET the full document min_trust = 0 + + // subdoc ids accepted by the validator even if they are without bitmap definition + valid_subdoc_ids = [] } diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 82f0743..20dc8fd 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -29,6 +29,10 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + notFoundProfileText = `{"profiles":[]}` +) + func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { // ==== data integrity check ==== params := mux.Vars(r) @@ -50,18 +54,17 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } // append the extra query_params if any + var rootdoc *common.RootDocument var queryParams string - if s.SupplementaryAppendingEnabled() { - rootdoc, err := s.GetRootDocument(mac) + var err error + if s.SupplementaryAppendingEnabled() || s.UpstreamProfilesEnabled() { + rootdoc, err = s.GetRootDocument(mac) if err != nil { if !s.IsDbNotFound(err) { Error(w, http.StatusInternalServerError, common.NewError(err)) return } } - if rootdoc != nil { - queryParams = rootdoc.QueryParams - } } // partner handling @@ -70,21 +73,73 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r partnerId = "" } + if s.SupplementaryAppendingEnabled() && rootdoc != nil { + queryParams = rootdoc.QueryParams + } + urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - rbytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + xconfNotFound := false if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - Error(w, rherr.StatusCode, rherr) + if rherr.StatusCode == http.StatusNotFound { + if s.UpstreamProfilesEnabled() { + xconfNotFound = true + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if !xconfNotFound { + Error(w, http.StatusInternalServerError, common.NewError(err)) return } - Error(w, http.StatusInternalServerError, common.NewError(err)) - return } - mpart, err := util.TelemetryBytesToMultipart(rbytes) + var profileBytes []byte + if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { + // Get profiles from the second source + extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) + if err != nil { + exitNow := true + var rherr common.RemoteHttpError + if errors.As(err, &rherr) { + if rherr.StatusCode == http.StatusNotFound { + exitNow = false + extraProfileBytes = nil + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if exitNow { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + } + + if xconfNotFound { + baseProfileBytes = []byte(notFoundProfileText) + } + + // append profiles stored at webconfig + profileBytes, err = util.AppendProfiles(baseProfileBytes, extraProfileBytes) + if err != nil { + Error(w, http.StatusInternalServerError, err) + return + } + } else { + profileBytes = baseProfileBytes + } + + mpart, err := util.TelemetryBytesToMultipart(profileBytes) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) return diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 54e2642..1328b9c 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -890,3 +890,242 @@ func TestSupplementaryXconfReadAllErr(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) } + +var ( + mockUpstreamProfileResponse1 = []byte(`[ + { + "name": "subname1", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.1.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "977e16c4" + }, + { + "name": "subname2", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.2.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "4f207ebd" + } +]`) +) + +func TestSupplementaryUpstreamProfiles(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + cxbytes, err := util.CompactJson([]byte(mockProfileResponse)) + assert.NilError(t, err) + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cxbytes) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + cubytes, err := util.CompactJson(mockUpstreamProfileResponse1) + assert.NilError(t, err) + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cubytes) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok := mparts["telemetry"] + assert.Assert(t, ok) + + output := common.TR181Output{} + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes := []byte(output.Parameters[0].Value) + + var itf util.Dict + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok := itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok := profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf := profilesJs[0] + + profile1, ok := profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok := profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok := coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + var srcItf map[string]interface{} + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 4 enable the feature flag but no query_param expect 200 with 1 mpart ==== + server.SetUpstreamProfilesEnabled(true) + defer server.SetUpstreamProfilesEnabled(false) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok = profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok = coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + srcItf = make(map[string]interface{}) + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 5 set query_param in the root_document expect 200 with 1 mpart ==== + rdoc := new(common.RootDocument) + rdoc.QueryParams = "key1=val1" + rdoc.ModelName = modelName + rdoc.PartnerId = partnerID + rdoc.FirmwareVersion = firmwareVersion + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + t.Logf("%s\n", rbytes) + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 3) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + profile2Itf := profilesJs[1] + profile2, ok := profile2Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile2["name"].(string), "subname1") + + profile3Itf := profilesJs[2] + profile3, ok := profile3Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile3["name"].(string), "subname2") +} diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 165aac6..d006bc5 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,15 +22,16 @@ import ( "fmt" "net/http" + "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" owcommon "github.com/rdkcentral/webconfig/common" - "github.com/go-akka/configuration" log "github.com/sirupsen/logrus" ) const ( upstreamHostDefault = "http://localhost:1234" - upstreamUrlTemplateDefault = "/api/v1/device/%v/upstream" + defaultUpstreamUrlTemplate = "/api/v1/device/%v/upstream" + defaultProfileUrlTemplate = "/api/v1/device/%v/profile?%v" ) type UpstreamConnector struct { @@ -38,6 +39,7 @@ type UpstreamConnector struct { host string serviceName string upstreamUrlTemplate string + profileUrlTemplate string } func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *UpstreamConnector { @@ -45,13 +47,16 @@ func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *Up confKey := fmt.Sprintf("webconfig.%v.host", serviceName) host := conf.GetString(confKey, upstreamHostDefault) confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) - upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) + upstreamUrlTemplate := conf.GetString(confKey, defaultUpstreamUrlTemplate) + confKey = fmt.Sprintf("webconfig.%v.profile_url_template", serviceName) + profileUrlTemplate := conf.GetString(confKey, defaultProfileUrlTemplate) return &UpstreamConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, + profileUrlTemplate: profileUrlTemplate, } } @@ -90,3 +95,27 @@ func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes } return rbytes, header, nil } + +func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { + url := c.UpstreamHost() + fmt.Sprintf(c.profileUrlTemplate, mac, queryParams) + + if itf, ok := fields["audit_id"]; ok { + auditId := itf.(string) + if len(auditId) > 0 { + header.Set(common.HeaderAuditid, auditId) + } + } + + if itf, ok := fields["app_name"]; ok { + appName := itf.(string) + if len(appName) > 0 { + header.Set(common.HeaderSourceAppName, appName) + } + } + + rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) + if err != nil { + return rbytes, header, owcommon.NewError(err) + } + return rbytes, header, nil +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index dcd7e1e..9b4cb53 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -119,9 +119,10 @@ type WebconfigServer struct { webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string + upstreamProfilesEnabled bool queryParamsValidationEnabled bool - validSubdocIdMap map[string]int minTrust int + validSubdocIdMap map[string]int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -290,13 +291,14 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } } + upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + minTrust := int(conf.GetInt32("webconfig.min_trust")) validSubdocIds := conf.GetStringList("webconfig.valid_subdoc_ids") validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) for _, x := range validSubdocIds { validSubdocIdMap[x] = 1 } - minTrust := int(conf.GetInt32("webconfig.min_trust")) ws := &WebconfigServer{ Server: &http.Server{ @@ -332,9 +334,10 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, + upstreamProfilesEnabled: upstreamProfilesEnabled, queryParamsValidationEnabled: queryParamsValidationEnabled, - validSubdocIdMap: validSubdocIdMap, minTrust: minTrust, + validSubdocIdMap: validSubdocIdMap, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -662,20 +665,20 @@ func (s *WebconfigServer) SetKafkaProducerTopic(x string) { s.kafkaProducerTopic = x } -func (s *WebconfigServer) QueryParamsValidationEnabled() bool { - return s.queryParamsValidationEnabled +func (s *WebconfigServer) UpstreamProfilesEnabled() bool { + return s.upstreamProfilesEnabled } -func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { - s.queryParamsValidationEnabled = enabled +func (s *WebconfigServer) SetUpstreamProfilesEnabled(enabled bool) { + s.upstreamProfilesEnabled = enabled } -func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { - return s.validSubdocIdMap +func (s *WebconfigServer) QueryParamsValidationEnabled() bool { + return s.queryParamsValidationEnabled } -func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { - s.validSubdocIdMap = x +func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { + s.queryParamsValidationEnabled = enabled } func (s *WebconfigServer) MinTrust() int { @@ -686,6 +689,14 @@ func (s *WebconfigServer) SetMinTrust(trust int) { s.minTrust = trust } +func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { + return s.validSubdocIdMap +} + +func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { + s.validSubdocIdMap = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 37a7a5f..0c028aa 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -98,6 +98,14 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetValidPartners(validPartners) assert.DeepEqual(t, server.ValidPartners(), validPartners) + // get profiles from upstream + enabled = true + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + enabled = false + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + // enforce strict query parameters validation enabled = true server.SetQueryParamsValidationEnabled(enabled) @@ -106,6 +114,14 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetQueryParamsValidationEnabled(enabled) assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + //configure trust level + trust := 1000 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + trust = 500 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + validSubdocIdMap := map[string]int{ "red": 1, "orange": 2, @@ -115,11 +131,4 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetValidSubdocIdMap(validSubdocIdMap) assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) - //configure trust level - trust := 1000 - server.SetMinTrust(trust) - assert.Equal(t, server.MinTrust(), trust) - trust = 500 - server.SetMinTrust(trust) - assert.Equal(t, server.MinTrust(), trust) } diff --git a/util/string.go b/util/string.go index bd1eac4..064cc8e 100644 --- a/util/string.go +++ b/util/string.go @@ -14,18 +14,26 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "fmt" "net/http" "net/url" "strconv" "strings" - "github.com/rdkcentral/webconfig/common" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" +) + +const ( + Comma = ',' + RightSquareBracket = ']' + RightCurlBracket = '}' + baseProfileStr = `{"profiles":[` ) var ( @@ -123,3 +131,34 @@ func IsValidUTF8(bbytes []byte) bool { str2 := strings.ToValidUTF8(str1, "#") return str1 == str2 } + +func AppendProfiles(pbytes, sbytes []byte) ([]byte, error) { + var builder strings.Builder + if len(pbytes) < 3 { + builder.WriteString(baseProfileStr) + } else { + s := strings.TrimSpace(string(pbytes)) + builder.WriteString(s[:len(s)-2]) + } + if len(sbytes) > 2 { + if len(pbytes) > 20 { + builder.WriteRune(',') + } + builder.Write(sbytes[1 : len(sbytes)-1]) + } + builder.WriteRune(RightSquareBracket) + builder.WriteRune(RightCurlBracket) + return []byte(builder.String()), nil +} + +func CompactJson(sbytes []byte) ([]byte, error) { + var itf interface{} + if err := json.Unmarshal(sbytes, &itf); err != nil { + return nil, common.NewError(err) + } + jsbytes, err := json.Marshal(itf) + if err != nil { + return nil, common.NewError(err) + } + return jsbytes, nil +} diff --git a/util/string_test.go b/util/string_test.go index 8fb5288..3c6df1a 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -14,10 +14,11 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "net/http" "testing" @@ -79,3 +80,17 @@ func TestTelemetryQueryWithWanMac(t *testing.T) { expected := "env=PROD&partnerId=comcast&version=2.0&model=TG1682G&accountId=1234567890&firmwareVersion=TG1682_3.14p9s6_PROD_sey&estbMacAddress=567890ABCDEF" assert.Equal(t, qstr, expected) } + +func TestAppendProfiles(t *testing.T) { + mockedBaseProfilesResponse := `{"profiles":[{"name":"XfinityWIFI_SYNC","value":{"Description":"XfinityWIFI_SYNC to capture XWIFI info every 12 hours","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://stbrtl.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"name":"cpe_passpoint_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingServiceEnable","type":"dataModel"},{"name":"cpe_passpoint_inter_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingService.Parameters","type":"dataModel"},{"name":"cpe_passpoint_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Parameters","type":"dataModel"},{"name":"cpe_passpoint_rdk_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Enable","type":"dataModel"},{"name":"open5_bss_active","reference":"Device.WiFi.SSID.6.Enable","type":"dataModel"},{"name":"secure5_bss_active","reference":"Device.WiFi.SSID.10.Enable","type":"dataModel"},{"name":"secure5_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"wifi_enabled","reference":"Device.DeviceInfo.X_COMCAST_COM_xfinitywifiEnable","type":"dataModel"},{"name":"primary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.PrimaryRemoteEndpoint","type":"dataModel"},{"name":"secondary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.SecondaryRemoteEndpoint","type":"dataModel"},{"name":"open5_advertisement_str","reference":"Device.WiFi.AccessPoint.6.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"secure5_advertisement_str","reference":"Device.WiFi.AccessPoint.10.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"device_vlan_2","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.Interface.2.VLANID","type":"dataModel"},{"name":"open5_ssid_str","reference":"Device.WiFi.SSID.6.SSID","type":"dataModel"},{"name":"secure5_ssid_str","reference":"Device.WiFi.SSID.10.SSID","type":"dataModel"},{"name":"secure5_securitymode_str","reference":"Device.WiFi.AccessPoint.10.Security.ModeEnabled","type":"dataModel"},{"name":"secure5_pri_port","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerPort","type":"dataModel"},{"name":"dscp_marker","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.DSCPMarkPolicy","type":"dataModel"},{"name":"WIFI_CH_2_split","reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"name":"WIFI_CW_2_split","reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"name":"open5_bssid_str","reference":"Device.WiFi.SSID.6.BSSID","type":"dataModel"},{"name":"secure5_bssid_str","reference":"Device.WiFi.SSID.10.BSSID","type":"dataModel"},{"name":"open5_status_str","reference":"Device.WiFi.SSID.6.Status","type":"dataModel"},{"name":"secure5_status_str","reference":"Device.WiFi.SSID.10.Status","type":"dataModel"},{"name":"open5_beaconpower_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"open5_beaconrate_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"secure5_beaconpower_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"secure5_beaconrate_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"radio5_beaconinterval","reference":"Device.WiFi.Radio.2.X_COMCAST-COM_BeaconInterval","type":"dataModel"},{"name":"secure5_encryption","reference":"Device.WiFi.AccessPoint.10.Security.X_CISCO_COM_EncryptionMethod","type":"dataModel"},{"name":"secure5_sec_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerIPAddr","type":"dataModel"},{"name":"secure5_sec_port","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerPort","type":"dataModel"},{"name":"UPTIME_split","reference":"Device.DeviceInfo.UpTime","type":"dataModel"},{"name":"open5_radius_server_ip","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"open5_pri_port","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerPort","type":"dataModel"},{"name":"open5_isolation_enable","reference":"Device.WiFi.AccessPoint.6.IsolationEnable","type":"dataModel"},{"name":"secure5_isolation_enable","reference":"Device.WiFi.AccessPoint.10.IsolationEnable","type":"dataModel"},{"name":"secure24_bss_active","reference":"Device.WiFi.SSID.9.Enable","type":"dataModel"},{"name":"open24_bss_active","reference":"Device.WiFi.SSID.5.Enable","type":"dataModel"}],"Protocol":"HTTP","ReportingAdjustments":{"FirstReportingInterval":300},"ReportingInterval":86400,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.6"},"versionHash":"d9c6f386"},{"name":"WIFI_MOTION_Telemetry","value":{"Description":"CSCWFM_Telemetry","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"}],"URL":"https://stbrtl-oi.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"name":"Profile_Name","reference":"Profile.Name","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMrbussub_fail","name":"SYS_ERROR_WFMrbussub_fail","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_CSIpipe_restart","name":"SYS_SH_WFMpipe_restart","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_ctrlifcreate_fail","name":"SYS_ERROR_WFMifcreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSC_RXMrbusinit_fail","name":"SYS_ERROR_WFM_rbusinit_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionacquire_fail","name":"SYS_ERROR_WFMSessionAcq_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMeventscreate_fail","name":"SYS_ERROR_WFMeventscreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionenable_fail","name":"SYS_ERROR_WFMsessionenable_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXM_restart","name":"SYS_SH_WFMRXM_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_borg_restart","name":"SYS_SH_WFMborg_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_mqtt_restart","name":"SYS_SH_WFMmqtt_restart","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_sounding_state","name":"WFMsoundingstate_split","type":"event","use":"absolute"},{"name":"WFMEnable_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.CognitiveMotionDetection.Enable","type":"dataModel"},{"name":"WFMStatus_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_XHFW.WiFiMotionStatus","type":"dataModel"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentCommFailCnt_split","search":"wfmCogAgentCommFailCnt:","type":"grep","use":"count"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentConnected_split","search":"wfmCogAgentConnected:","type":"grep","use":"absolute"}],"Protocol":"HTTP","ReportingInterval":900,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"bf86fd16"}]}` + mockedExtraProfilesResponse := `[{"name":"james_test_profile_001","value":{"ActivationTimeout":600,"Description":"Telemetry 2.0 HSD Gateway WiFi Radio","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://rdkrtldev.stb.r53.xcal.tv/"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Description","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"}],"Protocol":"HTTP","ReportingInterval":60,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"ed0de6ef"}]` + + appendedBytes, err := AppendProfiles([]byte(mockedBaseProfilesResponse), []byte(mockedExtraProfilesResponse)) + assert.NilError(t, err) + + var itf interface{} + err = json.Unmarshal(appendedBytes, &itf) + assert.NilError(t, err) + _, err = json.MarshalIndent(itf, "", " ") + assert.NilError(t, err) +} From eb7872f79b776dd388617f6857e853df9d58b088 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Dec 2024 22:16:26 -0800 Subject: [PATCH 128/215] Add support for poke root and telemetry together --- common/const_var.go | 2 +- util/validator.go | 50 +++++++++++++----------------------------- util/validator_test.go | 4 ++-- 3 files changed, 18 insertions(+), 38 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 8e56f84..b8b8ec3 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -119,7 +119,7 @@ const ( ) var ( - SupportedPokeDocs = []string{"primary", "telemetry"} + SupportedPokeDocs = []string{"primary", "telemetry", "root"} SupportedPokeRoutes = []string{"mqtt"} ) diff --git a/util/validator.go b/util/validator.go index 754044f..2bab589 100644 --- a/util/validator.go +++ b/util/validator.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "net/url" + "slices" "strings" "github.com/rdkcentral/webconfig/common" @@ -41,50 +42,29 @@ func ValidateMac(mac string) bool { func ValidatePokeQuery(values url.Values) (string, error) { // handle ?doc=xxx - if docQueryParamStrs, ok := values["doc"]; ok { - if len(docQueryParamStrs) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(docQueryParamStrs[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeDocs, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - + docStr := values.Get("doc") + if len(docStr) > 0 { + docNames := strings.Split(docStr, ",") + for _, n := range docNames { + if !slices.Contains(common.SupportedPokeDocs, n) { + err := fmt.Errorf("invalid query parameter: %v", n) + return "", common.NewError(err) + } } - return queryStr, nil + return docStr, nil } // handle ?route=xxx - if qparams, ok := values["route"]; ok { - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(qparams[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeRoutes, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) + routeStr := values.Get("route") + if len(routeStr) > 0 { + if !slices.Contains(common.SupportedPokeRoutes, routeStr) { + err := fmt.Errorf("invalid query parameter: %v", routeStr) return "", common.NewError(err) } - return queryStr, nil + return routeStr, nil } - // return default return "primary", nil } diff --git a/util/validator_test.go b/util/validator_test.go index 4313f03..4cfb97a 100644 --- a/util/validator_test.go +++ b/util/validator_test.go @@ -56,8 +56,8 @@ func TestValidatePokeQuery(t *testing.T) { values := url.Values{} values["doc"] = []string{ - "primary,telemetry", "hello,world", + "primary,telemetry", } _, err := ValidatePokeQuery(values) assert.Assert(t, err != nil) @@ -72,7 +72,7 @@ func TestValidatePokeQuery(t *testing.T) { "primary,telemetry", } _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) + assert.NilError(t, err) values["doc"] = []string{ "primary", From 7e32cdd615df583b944200de43d6092e15c27418 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 9 Dec 2024 16:17:13 -0800 Subject: [PATCH 129/215] set updated_time in the case of state correction --- db/service.go | 2 ++ http/multipart_test.go | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/db/service.go b/db/service.go index 22c9b9b..169f2a0 100644 --- a/db/service.go +++ b/db/service.go @@ -188,6 +188,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } } } + updatedTime := int(time.Now().UnixMilli()) for subdocId, subdocument := range document.Items() { cloudVersion := subdocument.GetVersion() cloudState := subdocument.GetState() @@ -205,6 +206,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // update state newState := common.Deployed subdocument.SetState(&newState) + subdocument.SetUpdatedTime(&updatedTime) if cloudErrorCode > 0 { var newErrorCode int subdocument.SetErrorCode(&newErrorCode) diff --git a/http/multipart_test.go b/http/multipart_test.go index 3d26e81..ca35726 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1121,20 +1121,24 @@ func TestStateCorrectionEnabled(t *testing.T) { lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") assert.NilError(t, err) assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + oldLanUpdatedTime := *lanSubdocument.UpdatedTime() wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") assert.NilError(t, err) assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + oldWanUpdatedTime := *wanSubdocument.UpdatedTime() meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Failure) assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) + oldMeshUpdatedTime := *meshSubdocument.UpdatedTime() mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + oldMocaUpdatedTime := *mocaSubdocument.UpdatedTime() // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== server.SetStateCorrectionEnabled(true) @@ -1142,6 +1146,8 @@ func TestStateCorrectionEnabled(t *testing.T) { server.SetStateCorrectionEnabled(false) }() + time.Sleep(time.Duration(100) * time.Millisecond) + req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, header1) @@ -1155,20 +1161,24 @@ func TestStateCorrectionEnabled(t *testing.T) { lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") assert.NilError(t, err) assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *lanSubdocument.UpdatedTime() > oldLanUpdatedTime) wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") assert.NilError(t, err) assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *wanSubdocument.UpdatedTime() > oldWanUpdatedTime) meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Deployed) assert.Equal(t, *meshSubdocument.ErrorCode(), 0) assert.Equal(t, *meshSubdocument.ErrorDetails(), "") + assert.Assert(t, *meshSubdocument.UpdatedTime() > oldMeshUpdatedTime) mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + assert.Assert(t, *mocaSubdocument.UpdatedTime() == oldMocaUpdatedTime) } func TestCorruptedEncryptedDocumentHandler(t *testing.T) { From bb27227418720fe163f365b90a75dbbb281791cb Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 9 Dec 2024 16:17:13 -0800 Subject: [PATCH 130/215] set updated_time in the case of state correction --- db/service.go | 2 ++ http/multipart_test.go | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/db/service.go b/db/service.go index 22c9b9b..169f2a0 100644 --- a/db/service.go +++ b/db/service.go @@ -188,6 +188,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } } } + updatedTime := int(time.Now().UnixMilli()) for subdocId, subdocument := range document.Items() { cloudVersion := subdocument.GetVersion() cloudState := subdocument.GetState() @@ -205,6 +206,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // update state newState := common.Deployed subdocument.SetState(&newState) + subdocument.SetUpdatedTime(&updatedTime) if cloudErrorCode > 0 { var newErrorCode int subdocument.SetErrorCode(&newErrorCode) diff --git a/http/multipart_test.go b/http/multipart_test.go index 3d26e81..ca35726 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1121,20 +1121,24 @@ func TestStateCorrectionEnabled(t *testing.T) { lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") assert.NilError(t, err) assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + oldLanUpdatedTime := *lanSubdocument.UpdatedTime() wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") assert.NilError(t, err) assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + oldWanUpdatedTime := *wanSubdocument.UpdatedTime() meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Failure) assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) + oldMeshUpdatedTime := *meshSubdocument.UpdatedTime() mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + oldMocaUpdatedTime := *mocaSubdocument.UpdatedTime() // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== server.SetStateCorrectionEnabled(true) @@ -1142,6 +1146,8 @@ func TestStateCorrectionEnabled(t *testing.T) { server.SetStateCorrectionEnabled(false) }() + time.Sleep(time.Duration(100) * time.Millisecond) + req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, header1) @@ -1155,20 +1161,24 @@ func TestStateCorrectionEnabled(t *testing.T) { lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") assert.NilError(t, err) assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *lanSubdocument.UpdatedTime() > oldLanUpdatedTime) wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") assert.NilError(t, err) assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *wanSubdocument.UpdatedTime() > oldWanUpdatedTime) meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Deployed) assert.Equal(t, *meshSubdocument.ErrorCode(), 0) assert.Equal(t, *meshSubdocument.ErrorDetails(), "") + assert.Assert(t, *meshSubdocument.UpdatedTime() > oldMeshUpdatedTime) mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + assert.Assert(t, *mocaSubdocument.UpdatedTime() == oldMocaUpdatedTime) } func TestCorruptedEncryptedDocumentHandler(t *testing.T) { From 6b6de0bd83517b02f7a6d723e2f473cf058b3468 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 11 Dec 2024 22:48:58 -0800 Subject: [PATCH 131/215] propagate headers for tracing --- common/log_fields.go | 4 +--- common/log_fields_test.go | 20 ++++++++--------- http/poke_handler.go | 6 +++--- http/webconfig_server.go | 45 ++++++++++++++------------------------- http/webpa_connector.go | 17 +++------------ 5 files changed, 32 insertions(+), 60 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index 5ce267b..c14a280 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -27,8 +27,6 @@ var ( unloggedFields = []string{ "moneytrace", "token", - "out_traceparent", - "out_tracestate", } coreFields = []string{ "app_name", diff --git a/common/log_fields_test.go b/common/log_fields_test.go index 960af77..e2298cd 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -49,16 +49,14 @@ func TestFilterLogFields(t *testing.T) { assert.DeepEqual(t, expected, c2) src3 := log.Fields{ - "red": "maroon", - "orange": "auburn", - "yellow": "amber", - "green": "viridian", - "blue": "turquoise", - "indigo": "sapphire", - "violet": "purple", - "out_traceparent": "foo", - "out_tracestate": "cyan", - "token": "bar", + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "token": "bar", } c3 := FilterLogFields(src3) assert.DeepEqual(t, src, c3) diff --git a/http/poke_handler.go b/http/poke_handler.go index 4552769..cc48d30 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,10 +25,10 @@ import ( "sort" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" "go.opentelemetry.io/otel/attribute" ) @@ -172,7 +172,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { span.End() }() - transactionId, err := s.Poke(mac, token, pokeStr, fields) + transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 9b4cb53..fffb233 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -712,9 +712,9 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) - transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) + transactionId, err := c.Patch(rHeader, cpeMac, token, []byte(body), fields) if err != nil { return "", common.NewError(err) } @@ -742,7 +742,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques token = elements[1] } - var xmTraceId, traceId, outTraceparent, outTracestate string + var xmTraceId, traceId string // extract moneytrace from the header tracePart := strings.Split(r.Header.Get("X-Moneytrace"), ";")[0] @@ -756,14 +756,11 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques traceparent := r.Header.Get(common.HeaderTraceparent) if len(traceparent) == 55 { traceId = traceparent[3:35] - outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] } // extract tracestate from the header tracestate := r.Header.Get(common.HeaderTracestate) - if len(tracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) - } + // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { @@ -771,17 +768,17 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } headerMap := util.HeaderToMap(header) fields := log.Fields{ - "path": r.URL.String(), - "method": r.Method, - "audit_id": auditId, - "remote_ip": remoteIp, - "host_name": host, - "header": headerMap, - "logger": "request", - "trace_id": traceId, - "app_name": s.AppName(), - "out_traceparent": outTraceparent, - "out_tracestate": outTracestate, + "path": r.URL.String(), + "method": r.Method, + "audit_id": auditId, + "remote_ip": remoteIp, + "host_name": host, + "header": headerMap, + "logger": "request", + "trace_id": traceId, + "app_name": s.AppName(), + "traceparent": traceparent, + "tracestate": tracestate, } userAgent := r.UserAgent() @@ -992,7 +989,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) sc = sc.WithTraceFlags(traceFlags) } - tracestateStr := s.getTracestate(r) + tracestateStr := r.Header.Get(common.HeaderTracestate) if tracestateStr != "" { tracestate, _ := trace.ParseTraceState(tracestateStr) sc = sc.WithTraceState(tracestate) @@ -1017,16 +1014,6 @@ func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, tra return } -// extract tracestate from the header -func (s *WebconfigServer) getTracestate(r *http.Request) string { - inTracestate := r.Header.Get(common.HeaderTracestate) - var outTracestate string - if len(inTracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, s.TracestateVendorID(), s.TraceparentParentID()) - } - return outTracestate -} - func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { var span trace.Span diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 52378d9..41bbf9b 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -195,10 +195,10 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { c.asyncPokeEnabled = enabled } -func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { +func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { url := fmt.Sprintf(webpaUrlTemplate, c.WebpaHost(), c.ApiVersion(), cpeMac) - var traceId, xmTraceId, outTraceparent, outTracestate string + var traceId, xmTraceId string if itf, ok := fields["trace_id"]; ok { traceId = itf.(string) } @@ -213,22 +213,11 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field if len(traceId) == 0 { traceId = xmTraceId } - if itf, ok := fields["out_traceparent"]; ok { - outTraceparent = itf.(string) - } - if itf, ok := fields["out_tracestate"]; ok { - outTracestate = itf.(string) - } t := time.Now().UnixNano() / 1000 transactionId := fmt.Sprintf("%s_____%015x", xmTraceId, t) - xmoney := fmt.Sprintf("trace-id=%s;parent-id=0;span-id=0;span-name=%s", xmTraceId, webpaServiceName) - header := make(http.Header) - header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + header := rHeader.Clone() header.Set("X-Webpa-Transaction-Id", transactionId) - header.Set("X-Moneytrace", xmoney) - header.Set(common.HeaderTraceparent, outTraceparent) - header.Set(common.HeaderTracestate, outTracestate) method := "PATCH" _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) From 9b693066e5e8ac6009b64451b365941f3a170bde Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 12 Dec 2024 16:51:29 -0800 Subject: [PATCH 132/215] remove otel span codes --- go.mod | 19 +--- go.sum | 36 +------ http/otel.go | 208 --------------------------------------- http/poke_handler.go | 18 ---- http/router.go | 3 +- http/webconfig_server.go | 105 -------------------- http/webpa_connector.go | 10 -- main.go | 1 - 8 files changed, 7 insertions(+), 393 deletions(-) delete mode 100644 http/otel.go diff --git a/go.mod b/go.mod index 95108f9..a7cd24a 100644 --- a/go.mod +++ b/go.mod @@ -17,11 +17,6 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/otel v1.27.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 - go.opentelemetry.io/otel/sdk v1.27.0 - go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 golang.org/x/sync v0.6.0 @@ -31,18 +26,14 @@ require ( require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -53,24 +44,20 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 74c2010..8d8c331 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,6 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -64,6 +62,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -92,11 +91,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= @@ -167,8 +161,6 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -228,6 +220,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -264,6 +257,7 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -303,22 +297,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= @@ -478,8 +456,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -581,10 +557,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -597,8 +569,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/http/otel.go b/http/otel.go deleted file mode 100644 index 468035f..0000000 --- a/http/otel.go +++ /dev/null @@ -1,208 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 -*/ -package http - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" - "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/trace" - "go.opentelemetry.io/otel/trace/noop" - - "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" -) - -// Tracing contains the core dependencies to make tracing possible across an application. -type otelTracing struct { - providerName string - envName string - appName string - opName string - tracerProvider trace.TracerProvider - propagator propagation.TextMapPropagator - tracer trace.Tracer -} - -type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) - -var ( - ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") - ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") - providersConfig = map[string]providerConstructor{ - "http": httpTraceProvider, - "stdout": stdoutTraceProvider, - "noop": noopTraceProvider, - } - - otelTracer otelTracing -) - -// DefaultTracerProvider is used when no provider is given. -// The Noop tracer provider turns all tracing related operations into -// noops essentially disabling tracing. -const defaultTracerProvider = "noop" - -// newOtel creates a structure with components that apps can use to initialize OpenTelemetry -// tracing instrumentation code. -func newOtel(conf *configuration.Config) (*otelTracing, error) { - if IsNoopTracing(conf) { - log.Debug("open telemetry tracing disabled (noop)") - } else { - log.Debug("opentelemetry tracing enabled") - } - - otelTracer.appName = conf.GetString("webconfig.app_name") - otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") - otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") - tracerProvider, err := newTracerProvider(conf) - if err != nil { - return &otelTracer, err - } - otelTracer.tracerProvider = tracerProvider - otel.SetTracerProvider(tracerProvider) - - // Set up propagator. - prop := newPropagator() - otelTracer.propagator = prop - otel.SetTextMapPropagator(prop) - - otelTracer.tracer = otel.Tracer(otelTracer.appName) - return &otelTracer, nil -} - -// IsNoopTracing returns true if the provider is set to "noop" -func IsNoopTracing(conf *configuration.Config) bool { - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - return strings.EqualFold(providerName, "noop") -} - -// TracerProvider returns the tracer provider component. By default, the noop -// tracer provider is returned. -func (t otelTracing) TracerProvider() trace.TracerProvider { - if t.tracerProvider == nil { - return noop.NewTracerProvider() - } - return t.tracerProvider -} - -// Propagator returns the component that helps propagate trace context across -// API boundaries. By default, a W3C Trace Context format propagator is returned. -func (t otelTracing) Propagator() propagation.TextMapPropagator { - if t.propagator == nil { - return propagation.TraceContext{} - } - return t.propagator -} - -func newPropagator() propagation.TextMapPropagator { - return propagation.NewCompositeTextMapPropagator( - propagation.TraceContext{}, - propagation.Baggage{}, - ) -} - -// newTracerProvider creates the TracerProvider based on config setting -// If no config setting, a noop tracerProvider will be returned. -func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - if len(providerName) == 0 { - providerName = defaultTracerProvider - } - // Handling camelcase of provider. - providerName = strings.ToLower(providerName) - providerConfig := providersConfig[providerName] - if providerConfig == nil { - return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) - } - - traceProvider, err := providerConfig(conf) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) - } - return traceProvider, nil -} - -func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - return noop.NewTracerProvider(), nil -} - -func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - option := stdouttrace.WithPrettyPrint() - exporter, err := stdouttrace.New(option) - if err != nil { - return nil, err - } - tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), - sdktrace.WithBatcher(exporter, - // Default is 5s. Set to 1s for demonstrative purposes. - sdktrace.WithBatchTimeout(time.Second)), - sdktrace.WithResource( - resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(otelTracer.appName), - semconv.ServiceNamespaceKey.String(otelTracer.envName), - ), - ), - ) - return tp, nil -} - -func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - // Send traces over HTTP - endpoint := conf.GetString("webconfig.opentelemetry.endpoint") - if endpoint == "" { - return nil, ErrTracerProviderBuildFailed - } - exporter, err := otlptracehttp.New(context.Background(), - otlptracehttp.WithEndpoint(endpoint), - otlptracehttp.WithInsecure(), - ) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) - } - - return sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), - sdktrace.WithResource( - resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(otelTracer.appName), - semconv.ServiceNamespaceKey.String(otelTracer.envName), - ), - ), - ), nil -} - -func (s *WebconfigServer) OtelShutdown() { - sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) - if ok && sdkTraceProvider != nil { - sdkTraceProvider.Shutdown(context.TODO()) - } -} diff --git a/http/poke_handler.go b/http/poke_handler.go index cc48d30..205fea6 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -29,7 +29,6 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "go.opentelemetry.io/otel/attribute" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -157,28 +156,12 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - statusCode := http.StatusOK - pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) - // We are not passing any qparams to webpa, so fullPath = path - fullPath := pokeSpanPath - _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") - - // endSpan should reflect the real status of the webpa patch call - // not the transformed custom status - // e.g 404 from webpa patch is converted to 521, but we want to show 404 - defer func() { - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - span.End() - }() - transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 @@ -204,7 +187,6 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } - statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index f3e523f..97348c1 100644 --- a/http/router.go +++ b/http/router.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -100,7 +100,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } - sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index fffb233..c1ee3ef 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -32,11 +32,6 @@ import ( "strings" "time" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/trace" - "github.com/IBM/sarama" "github.com/go-akka/configuration" "github.com/google/uuid" @@ -115,8 +110,6 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool - otelTracer *otelTracing // For OpenTelemetry Tracing - webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool @@ -263,12 +256,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) - otelTracer, err := newOtel(conf) - if err != nil { - // Just log err and continue - log.Error("Could not initialize open telemetry for tracing, but continuing") - } - supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) // kafka producer @@ -330,7 +317,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, - otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, @@ -339,8 +325,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, } - // Init the child poke span name - ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -973,37 +957,6 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - spanContext := trace.SpanContextFromContext(ctx) - remote := spanContext.IsRemote() - sc := trace.SpanContext{}.WithRemote(remote) - - traceIDStr, traceFlagsStr := s.parseTraceparent(r) - if traceIDStr != "" { - traceID, _ := trace.TraceIDFromHex(traceIDStr) - sc = sc.WithTraceID(traceID) - } - if traceFlagsStr != "" { - traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) - sc = sc.WithTraceFlags(traceFlags) - } - tracestateStr := r.Header.Get(common.HeaderTracestate) - if tracestateStr != "" { - tracestate, _ := trace.ParseTraceState(tracestateStr) - sc = sc.WithTraceState(tracestate) - } - ctx = trace.ContextWithSpanContext(ctx, sc) - - ctx, span := s.newParentPokeSpan(ctx, r) - defer endSpan(span, w) - - // Pass the context with the span to the next handler - next.ServeHTTP(w, r.WithContext(ctx)) - }) -} - // extract traceparent from the header func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) @@ -1014,64 +967,6 @@ func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, tra return } -func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { - var span trace.Span - - ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) - // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) - - // Feedback: Better to use the "path"/API rather than a hard coded name - route := "oswebconfig_poke_handler" - if mux.CurrentRoute(r) != nil { // This can be nil in unit tests - route, _ = mux.CurrentRoute(r).GetPathTemplate() - } - s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) - return ctx, span -} - -func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { - var span trace.Span - ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) - // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) - - s.addAttributes(span, route, path, fullPath, method) - return ctx, span -} - -func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { - span.SetAttributes( - attribute.String("env", otelTracer.envName), - attribute.String("operation.name", otelTracer.opName), - attribute.String("http.url_details.path", path), - semconv.HTTPMethodKey.String(method), - semconv.HTTPRouteKey.String(route), - semconv.HTTPURLKey.String(fullPath), - ) - - log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) - log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) - log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) - log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) - log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) - log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) -} - -func endSpan(span trace.Span, w http.ResponseWriter) { - if xw, ok := w.(*XResponseWriter); ok { - statusCode := xw.Status() - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) - if statusCode >= http.StatusInternalServerError { - statusText := http.StatusText(statusCode) - span.SetStatus(codes.Error, statusText) - span.SetAttributes(attribute.String("http.response.error", statusText)) - log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) - } - } - span.End() -} - func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 41bbf9b..3ad378d 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,16 +168,6 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } -func (c *WebpaConnector) PokeSpanTemplate() string { - // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") -} - -// Base URL with the cpemac populated -func (c *WebpaConnector) PokeSpanPath(mac string) string { - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) -} - func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") diff --git a/main.go b/main.go index 8a47cb9..71de68c 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,6 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) - defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From e454080090cf93f6584e954c856643a03db02845 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 17 Dec 2024 23:56:19 -0800 Subject: [PATCH 133/215] restore otel codes --- go.mod | 19 +++++-- go.sum | 36 ++++++++++++-- http/poke_handler.go | 18 +++++++ http/router.go | 1 + http/webconfig_server.go | 105 +++++++++++++++++++++++++++++++++++++++ http/webpa_connector.go | 10 ++++ main.go | 1 + 7 files changed, 184 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a7cd24a..95108f9 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,11 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 golang.org/x/sync v0.6.0 @@ -26,14 +31,18 @@ require ( require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -44,20 +53,24 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/pretty v0.3.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 8d8c331..74c2010 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -62,7 +64,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -91,6 +92,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= @@ -161,6 +167,8 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -220,7 +228,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -257,7 +264,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -297,6 +303,22 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= @@ -456,6 +478,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -557,6 +581,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -569,6 +597,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/http/poke_handler.go b/http/poke_handler.go index 205fea6..cc48d30 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -29,6 +29,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" + "go.opentelemetry.io/otel/attribute" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -156,12 +157,28 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } + statusCode := http.StatusOK + pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) + // We are not passing any qparams to webpa, so fullPath = path + fullPath := pokeSpanPath + _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") + + // endSpan should reflect the real status of the webpa patch call + // not the transformed custom status + // e.g 404 from webpa patch is converted to 521, but we want to show 404 + defer func() { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + span.End() + }() + transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling + statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 @@ -187,6 +204,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } + statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index 97348c1..26d2dbe 100644 --- a/http/router.go +++ b/http/router.go @@ -100,6 +100,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } + sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index c1ee3ef..fffb233 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -32,6 +32,11 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/trace" + "github.com/IBM/sarama" "github.com/go-akka/configuration" "github.com/google/uuid" @@ -110,6 +115,8 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool + otelTracer *otelTracing // For OpenTelemetry Tracing + webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool @@ -256,6 +263,12 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + otelTracer, err := newOtel(conf) + if err != nil { + // Just log err and continue + log.Error("Could not initialize open telemetry for tracing, but continuing") + } + supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) // kafka producer @@ -317,6 +330,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, + otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, @@ -325,6 +339,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, } + // Init the child poke span name + ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -957,6 +973,37 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } +func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + spanContext := trace.SpanContextFromContext(ctx) + remote := spanContext.IsRemote() + sc := trace.SpanContext{}.WithRemote(remote) + + traceIDStr, traceFlagsStr := s.parseTraceparent(r) + if traceIDStr != "" { + traceID, _ := trace.TraceIDFromHex(traceIDStr) + sc = sc.WithTraceID(traceID) + } + if traceFlagsStr != "" { + traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) + sc = sc.WithTraceFlags(traceFlags) + } + tracestateStr := r.Header.Get(common.HeaderTracestate) + if tracestateStr != "" { + tracestate, _ := trace.ParseTraceState(tracestateStr) + sc = sc.WithTraceState(tracestate) + } + ctx = trace.ContextWithSpanContext(ctx, sc) + + ctx, span := s.newParentPokeSpan(ctx, r) + defer endSpan(span, w) + + // Pass the context with the span to the next handler + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + // extract traceparent from the header func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) @@ -967,6 +1014,64 @@ func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, tra return } +func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { + var span trace.Span + + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) + + // Feedback: Better to use the "path"/API rather than a hard coded name + route := "oswebconfig_poke_handler" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + route, _ = mux.CurrentRoute(r).GetPathTemplate() + } + s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) + return ctx, span +} + +func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) + + s.addAttributes(span, route, path, fullPath, method) + return ctx, span +} + +func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { + span.SetAttributes( + attribute.String("env", otelTracer.envName), + attribute.String("operation.name", otelTracer.opName), + attribute.String("http.url_details.path", path), + semconv.HTTPMethodKey.String(method), + semconv.HTTPRouteKey.String(route), + semconv.HTTPURLKey.String(fullPath), + ) + + log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) + log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) + log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) + log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) + log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) + log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) +} + +func endSpan(span trace.Span, w http.ResponseWriter) { + if xw, ok := w.(*XResponseWriter); ok { + statusCode := xw.Status() + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) + } + } + span.End() +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 3ad378d..41bbf9b 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,6 +168,16 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } +func (c *WebpaConnector) PokeSpanTemplate() string { + // By convention, span name won't have the host, but only the base template + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") +} + +// Base URL with the cpemac populated +func (c *WebpaConnector) PokeSpanPath(mac string) string { + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) +} + func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") diff --git a/main.go b/main.go index 71de68c..8a47cb9 100644 --- a/main.go +++ b/main.go @@ -61,6 +61,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 751143dfdd09b3cd1030482c9f22ee0bc144c445 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 18 Dec 2024 00:04:54 -0800 Subject: [PATCH 134/215] add back a missed file --- http/otel.go | 208 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 http/otel.go diff --git a/http/otel.go b/http/otel.go new file mode 100644 index 0000000..468035f --- /dev/null +++ b/http/otel.go @@ -0,0 +1,208 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" +) + +// Tracing contains the core dependencies to make tracing possible across an application. +type otelTracing struct { + providerName string + envName string + appName string + opName string + tracerProvider trace.TracerProvider + propagator propagation.TextMapPropagator + tracer trace.Tracer +} + +type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) + +var ( + ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") + ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, + "stdout": stdoutTraceProvider, + "noop": noopTraceProvider, + } + + otelTracer otelTracing +) + +// DefaultTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultTracerProvider = "noop" + +// newOtel creates a structure with components that apps can use to initialize OpenTelemetry +// tracing instrumentation code. +func newOtel(conf *configuration.Config) (*otelTracing, error) { + if IsNoopTracing(conf) { + log.Debug("open telemetry tracing disabled (noop)") + } else { + log.Debug("opentelemetry tracing enabled") + } + + otelTracer.appName = conf.GetString("webconfig.app_name") + otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") + otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") + tracerProvider, err := newTracerProvider(conf) + if err != nil { + return &otelTracer, err + } + otelTracer.tracerProvider = tracerProvider + otel.SetTracerProvider(tracerProvider) + + // Set up propagator. + prop := newPropagator() + otelTracer.propagator = prop + otel.SetTextMapPropagator(prop) + + otelTracer.tracer = otel.Tracer(otelTracer.appName) + return &otelTracer, nil +} + +// IsNoopTracing returns true if the provider is set to "noop" +func IsNoopTracing(conf *configuration.Config) bool { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + return strings.EqualFold(providerName, "noop") +} + +// TracerProvider returns the tracer provider component. By default, the noop +// tracer provider is returned. +func (t otelTracing) TracerProvider() trace.TracerProvider { + if t.tracerProvider == nil { + return noop.NewTracerProvider() + } + return t.tracerProvider +} + +// Propagator returns the component that helps propagate trace context across +// API boundaries. By default, a W3C Trace Context format propagator is returned. +func (t otelTracing) Propagator() propagation.TextMapPropagator { + if t.propagator == nil { + return propagation.TraceContext{} + } + return t.propagator +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +// newTracerProvider creates the TracerProvider based on config setting +// If no config setting, a noop tracerProvider will be returned. +func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + if len(providerName) == 0 { + providerName = defaultTracerProvider + } + // Handling camelcase of provider. + providerName = strings.ToLower(providerName) + providerConfig := providersConfig[providerName] + if providerConfig == nil { + return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) + } + + traceProvider, err := providerConfig(conf) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + return traceProvider, nil +} + +func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), + ) + return tp, nil +} + +func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + // Send traces over HTTP + endpoint := conf.GetString("webconfig.opentelemetry.endpoint") + if endpoint == "" { + return nil, ErrTracerProviderBuildFailed + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), + ), nil +} + +func (s *WebconfigServer) OtelShutdown() { + sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} From 6bea1ad46ba681d092440cd7af4c3c683ad280ca Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 19 Dec 2024 22:45:39 -0800 Subject: [PATCH 135/215] handle tracing propagation --- go.mod | 89 ++--- go.sum | 599 +++++--------------------------- http/http_client.go | 10 +- http/mqtt_connector.go | 9 +- http/multipart.go | 14 +- http/multipart_test.go | 19 +- http/poke_handler.go | 37 +- http/router.go | 1 - http/supplementary_handler.go | 6 +- http/upstream_connector.go | 9 +- http/upstream_connector_test.go | 6 +- http/webconfig_server.go | 101 +----- http/webpa_connector.go | 29 +- http/xconf_connector.go | 9 +- kafka/consumer.go | 7 +- main.go | 1 - 16 files changed, 224 insertions(+), 722 deletions(-) diff --git a/go.mod b/go.mod index 95108f9..6b05244 100644 --- a/go.mod +++ b/go.mod @@ -1,50 +1,52 @@ module github.com/rdkcentral/webconfig -go 1.21 +go 1.22.7 + +toolchain go1.22.10 require ( - github.com/IBM/sarama v1.42.1 + github.com/IBM/sarama v1.43.3 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 - github.com/gocql/gocql v1.6.0 - github.com/golang-jwt/jwt/v5 v5.0.0 + github.com/gocql/gocql v1.7.0 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.0 - github.com/mattn/go-sqlite3 v1.14.15 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 - github.com/sirupsen/logrus v1.9.0 - github.com/twmb/murmur3 v1.1.6 + github.com/gorilla/mux v1.8.1 + github.com/mattn/go-sqlite3 v1.14.24 + github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_model v0.6.1 + github.com/sirupsen/logrus v1.9.3 + github.com/twmb/murmur3 v1.1.8 github.com/vmihailenco/msgpack v4.0.4+incompatible - github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/otel v1.27.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 - go.opentelemetry.io/otel/sdk v1.27.0 - go.opentelemetry.io/otel/trace v1.27.0 - go.uber.org/automaxprocs v1.5.1 - go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.6.0 + github.com/vmihailenco/msgpack/v4 v4.3.13 + go.opentelemetry.io/otel v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 + go.opentelemetry.io/otel/sdk v1.33.0 + go.opentelemetry.io/otel/trace v1.33.0 + go.uber.org/automaxprocs v1.6.0 + go.uber.org/ratelimit v0.3.1 + golang.org/x/sync v0.10.0 gotest.tools v2.2.0+incompatible ) require ( - github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.4.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -52,25 +54,28 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.30.0 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect + google.golang.org/protobuf v1.35.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 74c2010..d588a29 100644 --- a/go.sum +++ b/go.sum @@ -1,51 +1,9 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= -github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= +github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= +github.com/IBM/sarama v1.43.3/go.mod h1:FVIRaLrhK3Cla/9FfRF5X9Zua2KpS3SYIXxhac1H+FQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= @@ -54,80 +12,33 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= -github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= +github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= -github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= +github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -135,52 +46,27 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -193,21 +79,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -215,431 +88,147 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= -github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= -github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNlKBGKKXKI= +github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= -go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= -go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 h1:W5AWUn/IVe8RFb5pZx1Uh9Laf/4+Qmm4kJL5zPuvR+0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0/go.mod h1:mzKxJywMNBdEX8TSJais3NnsVZUaJ+bAy6UxPTng2vk= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/http/http_client.go b/http/http_client.go index f67598c..9661cbf 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -19,6 +19,7 @@ package http import ( "bytes" + "context" "crypto/tls" "encoding/base64" "encoding/json" @@ -37,6 +38,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/propagation" ) const ( @@ -107,7 +109,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } } -func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { +func (c *HttpClient) Do(ctx context.Context, method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields) // verify a response is received @@ -206,6 +208,8 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] startTime := time.Now() // the core http call + propagator := propagation.TraceContext{} + propagator.Inject(ctx, propagation.HeaderCarrier(req.Header)) res, err := c.Client.Do(req) // err should be *url.Error @@ -354,7 +358,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] return rbytes, res.Header, false, nil } -func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { +func (c *HttpClient) DoWithRetries(ctx context.Context, method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { var respBytes []byte var respHeader http.Header var err error @@ -368,7 +372,7 @@ func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Heade if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(ctx, method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 07c086b..064f2e5 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -14,18 +14,19 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( + "context" "crypto/tls" "fmt" "net/http" "time" - "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) @@ -64,7 +65,7 @@ func (c *MqttConnector) ServiceName() string { return c.serviceName } -func (c *MqttConnector) PostMqtt(cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { +func (c *MqttConnector) PostMqtt(ctx context.Context, cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { url := fmt.Sprintf(mqttUrlTemplate, c.MqttHost(), cpeMac) var traceId, xmTraceId, outTraceparent, outTracestate string @@ -94,7 +95,7 @@ func (c *MqttConnector) PostMqtt(cpeMac string, bbytes []byte, fields log.Fields header.Set(common.HeaderTraceparent, outTraceparent) header.Set(common.HeaderTracestate, outTracestate) - rbytes, _, err := c.DoWithRetries("POST", url, header, bbytes, fields, c.ServiceName()) + rbytes, _, err := c.DoWithRetries(ctx, "POST", url, header, bbytes, fields, c.ServiceName()) if err != nil { return rbytes, common.NewError(err) } diff --git a/http/multipart.go b/http/multipart.go index 5e39531..a431552 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -18,6 +18,7 @@ package http import ( + "context" "errors" "fmt" "net/http" @@ -43,6 +44,7 @@ var ( ) func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() // check if this is a Supplementary service, if so, call a different handler if hd := r.Header.Get(common.HeaderSupplementaryService); len(hd) > 0 { s.MultipartSupplementaryHandler(w, r) @@ -94,7 +96,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. r.Header.Set(common.HeaderSchemaVersion, "none") } - status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(s, ctx, r.Header, common.RouteHttp, fields) switch status { case http.StatusNotFound: @@ -118,7 +120,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. _, _ = w.Write(respBytes) } -func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route string, fields log.Fields) (int, http.Header, []byte, error) { +func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader http.Header, route string, fields log.Fields) (int, http.Header, []byte, error) { fields["for_device"] = true fields["is_primary"] = true @@ -131,7 +133,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // factory reset handling ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) if ifNoneMatch == "NONE" || ifNoneMatch == "NONE-REBOOT" { - status, respHeader, rbytes, err := BuildFactoryResetResponse(s, rHeader, fields) + status, respHeader, rbytes, err := BuildFactoryResetResponse(s, ctx, rHeader, fields) if err != nil { return status, respHeader, rbytes, common.NewError(err) } @@ -285,7 +287,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin upstreamHeader.Set(common.HeaderUpstreamOldSchemaVersion, oldRootDocument.SchemaVersion) } - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, respBytes, fields) + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(ctx, mac, upstreamHeader, respBytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -337,7 +339,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusOK, upstreamRespHeader, finalFilteredBytes, nil } -func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { +func BuildFactoryResetResponse(s *WebconfigServer, ctx context.Context, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { c := s.DatabaseClient uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) @@ -408,7 +410,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } // call /upstream to handle factory reset - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(ctx, mac, upstreamHeader, oldDocBytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { diff --git a/http/multipart_test.go b/http/multipart_test.go index ca35726..b20bb45 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -19,6 +19,7 @@ package http import ( "bytes" + "context" "encoding/hex" "fmt" "io" @@ -298,6 +299,7 @@ func TestCpeMiddleware(t *testing.T) { func TestVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) + ctx := context.Background() cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== @@ -415,7 +417,7 @@ func TestVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") fields := make(log.Fields) - status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType := respHeader.Get(common.HeaderContentType) @@ -433,7 +435,7 @@ func TestVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDocName, "root,lan,wan") kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusNotModified) assert.Equal(t, len(respBytes), 0) @@ -617,6 +619,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { func TestMqttUpstreamVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) + ctx := context.Background() cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== @@ -677,7 +680,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader := make(http.Header) kHeader.Set(common.HeaderDeviceId, cpeMac) fields := make(log.Fields) - status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType := respHeader.Get(common.HeaderContentType) @@ -707,7 +710,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.Assert(t, err != nil) assert.Equal(t, status, http.StatusServiceUnavailable) @@ -715,7 +718,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader = make(http.Header) kHeader.Set(common.HeaderDeviceId, cpeMac) fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.Assert(t, err != nil) assert.Equal(t, status, http.StatusServiceUnavailable) @@ -741,7 +744,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType = respHeader.Get(common.HeaderContentType) @@ -765,7 +768,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusNotModified) @@ -776,7 +779,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType = respHeader.Get(common.HeaderContentType) diff --git a/http/poke_handler.go b/http/poke_handler.go index cc48d30..fb25745 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -29,10 +29,24 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { + // tracing propagation + ctx := r.Context() + + propagator := propagation.TraceContext{} + ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) + span := trace.SpanFromContext(ctx) + defer span.End() + tracer := otel.Tracer(s.otelTracer.appName) + ctx, childSpan := tracer.Start(ctx, s.otelTracer.opName) + defer childSpan.End() + + // handler params := mux.Vars(r) mac := params["mac"] mac = strings.ToUpper(mac) @@ -104,7 +118,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - _, err = s.PostMqtt(deviceId, mbytes, fields) + _, err = s.PostMqtt(ctx, deviceId, mbytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -157,28 +171,12 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - statusCode := http.StatusOK - pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) - // We are not passing any qparams to webpa, so fullPath = path - fullPath := pokeSpanPath - _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") - - // endSpan should reflect the real status of the webpa patch call - // not the transformed custom status - // e.g 404 from webpa patch is converted to 521, but we want to show 404 - defer func() { - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - span.End() - }() - - transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) + transactionId, err := s.Poke(ctx, r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 @@ -204,7 +202,6 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } - statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index 26d2dbe..97348c1 100644 --- a/http/router.go +++ b/http/router.go @@ -100,7 +100,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } - sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 20dc8fd..401c4b7 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -34,6 +34,8 @@ const ( ) func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + // ==== data integrity check ==== params := mux.Vars(r) mac, ok := params["mac"] @@ -80,7 +82,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(ctx, urlSuffix, fields) xconfNotFound := false if err != nil { var rherr common.RemoteHttpError @@ -106,7 +108,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r var profileBytes []byte if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { // Get profiles from the second source - extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) + extraProfileBytes, _, err := s.GetUpstreamProfiles(ctx, mac, queryParams, r.Header, fields) if err != nil { exitNow := true var rherr common.RemoteHttpError diff --git a/http/upstream_connector.go b/http/upstream_connector.go index d006bc5..303f2b6 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -18,6 +18,7 @@ package http import ( + "context" "crypto/tls" "fmt" "net/http" @@ -72,7 +73,7 @@ func (c *UpstreamConnector) ServiceName() string { return c.serviceName } -func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { +func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { url := c.UpstreamHost() + fmt.Sprintf(c.upstreamUrlTemplate, mac) if itf, ok := fields["audit_id"]; ok { @@ -89,14 +90,14 @@ func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes } } - rbytes, header, err := c.DoWithRetries("POST", url, header, bbytes, fields, c.ServiceName()) + rbytes, header, err := c.DoWithRetries(ctx, "POST", url, header, bbytes, fields, c.ServiceName()) if err != nil { return rbytes, header, owcommon.NewError(err) } return rbytes, header, nil } -func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { +func (c *UpstreamConnector) GetUpstreamProfiles(ctx context.Context, mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { url := c.UpstreamHost() + fmt.Sprintf(c.profileUrlTemplate, mac, queryParams) if itf, ok := fields["audit_id"]; ok { @@ -113,7 +114,7 @@ func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header } } - rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) + rbytes, header, err := c.DoWithRetries(ctx, "GET", url, header, nil, fields, c.ServiceName()) if err != nil { return rbytes, header, owcommon.NewError(err) } diff --git a/http/upstream_connector_test.go b/http/upstream_connector_test.go index e9dfccb..13fddbb 100644 --- a/http/upstream_connector_test.go +++ b/http/upstream_connector_test.go @@ -14,10 +14,11 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( + "context" "net/http" "net/http/httptest" "testing" @@ -51,6 +52,7 @@ func TestUpstreamConnector(t *testing.T) { bbytes := []byte("hello world") var err error fields := log.Fields{} - _, _, err = server.PostUpstream(mac, header, bbytes, fields) + ctx := context.Background() + _, _, err = server.PostUpstream(ctx, mac, header, bbytes, fields) assert.NilError(t, err) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index fffb233..daf331e 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -32,11 +32,6 @@ import ( "strings" "time" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/trace" - "github.com/IBM/sarama" "github.com/go-akka/configuration" "github.com/google/uuid" @@ -116,7 +111,6 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing - webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool @@ -339,8 +333,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, } - // Init the child poke span name - ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -712,9 +704,9 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(ctx context.Context, rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) - transactionId, err := c.Patch(rHeader, cpeMac, token, []byte(body), fields) + transactionId, err := c.Patch(ctx, rHeader, cpeMac, token, []byte(body), fields) if err != nil { return "", common.NewError(err) } @@ -973,37 +965,6 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - spanContext := trace.SpanContextFromContext(ctx) - remote := spanContext.IsRemote() - sc := trace.SpanContext{}.WithRemote(remote) - - traceIDStr, traceFlagsStr := s.parseTraceparent(r) - if traceIDStr != "" { - traceID, _ := trace.TraceIDFromHex(traceIDStr) - sc = sc.WithTraceID(traceID) - } - if traceFlagsStr != "" { - traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) - sc = sc.WithTraceFlags(traceFlags) - } - tracestateStr := r.Header.Get(common.HeaderTracestate) - if tracestateStr != "" { - tracestate, _ := trace.ParseTraceState(tracestateStr) - sc = sc.WithTraceState(tracestate) - } - ctx = trace.ContextWithSpanContext(ctx, sc) - - ctx, span := s.newParentPokeSpan(ctx, r) - defer endSpan(span, w) - - // Pass the context with the span to the next handler - next.ServeHTTP(w, r.WithContext(ctx)) - }) -} - // extract traceparent from the header func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) @@ -1014,64 +975,6 @@ func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, tra return } -func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { - var span trace.Span - - ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) - // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) - - // Feedback: Better to use the "path"/API rather than a hard coded name - route := "oswebconfig_poke_handler" - if mux.CurrentRoute(r) != nil { // This can be nil in unit tests - route, _ = mux.CurrentRoute(r).GetPathTemplate() - } - s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) - return ctx, span -} - -func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { - var span trace.Span - ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) - // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) - - s.addAttributes(span, route, path, fullPath, method) - return ctx, span -} - -func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { - span.SetAttributes( - attribute.String("env", otelTracer.envName), - attribute.String("operation.name", otelTracer.opName), - attribute.String("http.url_details.path", path), - semconv.HTTPMethodKey.String(method), - semconv.HTTPRouteKey.String(route), - semconv.HTTPURLKey.String(fullPath), - ) - - log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) - log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) - log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) - log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) - log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) - log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) -} - -func endSpan(span trace.Span, w http.ResponseWriter) { - if xw, ok := w.(*XResponseWriter); ok { - statusCode := xw.Status() - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) - if statusCode >= http.StatusInternalServerError { - statusText := http.StatusText(statusCode) - span.SetStatus(codes.Error, statusText) - span.SetAttributes(attribute.String("http.response.error", statusText)) - log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) - } - } - span.End() -} - func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 41bbf9b..f184e1f 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -18,6 +18,7 @@ package http import ( + "context" "crypto/tls" "encoding/json" "errors" @@ -168,16 +169,6 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } -func (c *WebpaConnector) PokeSpanTemplate() string { - // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") -} - -// Base URL with the cpemac populated -func (c *WebpaConnector) PokeSpanPath(mac string) string { - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) -} - func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") @@ -195,7 +186,7 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { c.asyncPokeEnabled = enabled } -func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { +func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { url := fmt.Sprintf(webpaUrlTemplate, c.WebpaHost(), c.ApiVersion(), cpeMac) var traceId, xmTraceId string @@ -220,16 +211,16 @@ func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, header.Set("X-Webpa-Transaction-Id", transactionId) method := "PATCH" - _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(ctx, method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { if rherr.StatusCode == 524 { if c.asyncPokeEnabled { c.queue <- struct{}{} - go c.AsyncDoWithRetries(method, url, header, bbytes, fields, asyncWebpaServiceName) + go c.AsyncDoWithRetries(ctx, method, url, header, bbytes, fields, asyncWebpaServiceName) } else { - _, err := c.SyncDoWithRetries(method, url, header, bbytes, fields, webpaServiceName) + _, err := c.SyncDoWithRetries(ctx, method, url, header, bbytes, fields, webpaServiceName) if err != nil { return transactionId, common.NewError(err) } @@ -238,7 +229,7 @@ func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, } } if cont { - _, _, err := c.syncClient.DoWithRetries("PATCH", url, header, bbytes, fields, webpaServiceName) + _, _, err := c.syncClient.DoWithRetries(ctx, "PATCH", url, header, bbytes, fields, webpaServiceName) if err != nil { return transactionId, common.NewError(err) } @@ -249,7 +240,7 @@ func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, return transactionId, nil } -func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { +func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { tfields := common.FilterLogFields(fields) tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { @@ -258,7 +249,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header ht if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(ctx, method, url, header, cbytes, fields, loggerName, i) if !cont { msg := fmt.Sprintf("finished success after 1 retry") if i > 1 { @@ -275,7 +266,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header ht } // this has 1 less retries compared to the standard DoWithRetries() -func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, error) { +func (c *WebpaConnector) SyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, error) { var rbytes []byte var err error var cont bool @@ -286,7 +277,7 @@ func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header htt if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(ctx, method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError diff --git a/http/xconf_connector.go b/http/xconf_connector.go index d40550a..7a39974 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -14,16 +14,17 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( + "context" "crypto/tls" "fmt" "net/http" - owcommon "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" + owcommon "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) @@ -62,9 +63,9 @@ func (c *XconfConnector) ServiceName() string { return c.serviceName } -func (c *XconfConnector) GetProfiles(urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { +func (c *XconfConnector) GetProfiles(ctx context.Context, urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(xconfUrlTemplate, c.XconfHost(), urlSuffix) - rbytes, resHeader, err := c.DoWithRetries("GET", url, nil, nil, fields, c.ServiceName()) + rbytes, resHeader, err := c.DoWithRetries(ctx, "GET", url, nil, nil, fields, c.ServiceName()) if err != nil { return rbytes, resHeader, owcommon.NewError(err) } diff --git a/kafka/consumer.go b/kafka/consumer.go index 40e8ec1..51cc318 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -18,6 +18,7 @@ package kafka import ( + "context" "encoding/base64" "encoding/json" "fmt" @@ -105,6 +106,8 @@ func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common // NOTE we choose to return an EventMessage object just to pass along the metricsAgent func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common.EventMessage, error) { + ctx := context.TODO() + rHeader, _ := util.ParseHttp(inbytes) params := rHeader.Get(common.HeaderDocName) cpeMac := rHeader.Get(common.HeaderDeviceId) @@ -148,7 +151,7 @@ func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common. rHeader.Set(common.HeaderSchemaVersion, "none") } - status, respHeader, respBytes, err := wchttp.BuildWebconfigResponse(c.WebconfigServer, rHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := wchttp.BuildWebconfigResponse(c.WebconfigServer, ctx, rHeader, common.RouteMqtt, fields) if err != nil && respBytes == nil { respBytes = []byte(err.Error()) } @@ -159,7 +162,7 @@ func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common. } mqttBytes := common.BuildPayloadAsHttp(status, respHeader, respBytes) - _, err = c.PostMqtt(cpeMac, mqttBytes, fields) + _, err = c.PostMqtt(ctx, cpeMac, mqttBytes, fields) if err != nil { return &m, common.NewError(err) } diff --git a/main.go b/main.go index 8a47cb9..71de68c 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,6 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) - defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 127cee1a2b5cd9144941210d621399f447aff0ea Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 19 Dec 2024 22:51:09 -0800 Subject: [PATCH 136/215] add back calling otelshutdown --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 71de68c..8a47cb9 100644 --- a/main.go +++ b/main.go @@ -61,6 +61,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 13a9c622476ff4237a69f758f5a178ed96175f87 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 19 Dec 2024 23:04:56 -0800 Subject: [PATCH 137/215] roll back go.mod to use 1.21 --- go.mod | 89 ++++----- go.sum | 599 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 547 insertions(+), 141 deletions(-) diff --git a/go.mod b/go.mod index 6b05244..95108f9 100644 --- a/go.mod +++ b/go.mod @@ -1,52 +1,50 @@ module github.com/rdkcentral/webconfig -go 1.22.7 - -toolchain go1.22.10 +go 1.21 require ( - github.com/IBM/sarama v1.43.3 + github.com/IBM/sarama v1.42.1 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 - github.com/gocql/gocql v1.7.0 - github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/gocql/gocql v1.6.0 + github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.1 - github.com/mattn/go-sqlite3 v1.14.24 - github.com/prometheus/client_golang v1.20.5 - github.com/prometheus/client_model v0.6.1 - github.com/sirupsen/logrus v1.9.3 - github.com/twmb/murmur3 v1.1.8 + github.com/gorilla/mux v1.8.0 + github.com/mattn/go-sqlite3 v1.14.15 + github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_model v0.2.0 + github.com/sirupsen/logrus v1.9.0 + github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible - github.com/vmihailenco/msgpack/v4 v4.3.13 - go.opentelemetry.io/otel v1.33.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 - go.opentelemetry.io/otel/sdk v1.33.0 - go.opentelemetry.io/otel/trace v1.33.0 - go.uber.org/automaxprocs v1.6.0 - go.uber.org/ratelimit v0.3.1 - golang.org/x/sync v0.10.0 + github.com/vmihailenco/msgpack/v4 v4.3.12 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 + go.uber.org/automaxprocs v1.5.1 + go.uber.org/ratelimit v0.2.0 + golang.org/x/sync v0.6.0 gotest.tools v2.2.0+incompatible ) require ( - github.com/benbjohnson/clock v1.3.0 // indirect + github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/eapache/go-resiliency v1.7.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -54,28 +52,25 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect - go.opentelemetry.io/otel/metric v1.33.0 // indirect - go.opentelemetry.io/proto/otlp v1.4.0 // indirect - go.uber.org/atomic v1.11.0 // indirect - golang.org/x/crypto v0.30.0 // indirect - golang.org/x/net v0.32.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.35.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index d588a29..74c2010 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,51 @@ -github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= -github.com/IBM/sarama v1.43.3/go.mod h1:FVIRaLrhK3Cla/9FfRF5X9Zua2KpS3SYIXxhac1H+FQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= +github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= @@ -12,33 +54,80 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= -github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= +github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= -github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= +github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -46,27 +135,52 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -79,8 +193,21 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -88,147 +215,431 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= -github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= -github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNlKBGKKXKI= -github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= -go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 h1:W5AWUn/IVe8RFb5pZx1Uh9Laf/4+Qmm4kJL5zPuvR+0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0/go.mod h1:mzKxJywMNBdEX8TSJais3NnsVZUaJ+bAy6UxPTng2vk= -go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= -go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= -go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= -go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= -go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= -go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= -go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= -go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= -go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= -go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= -go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= +go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= +go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From a5984fb5ce583ece505a9414798bf2c27da96acc Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 20 Dec 2024 22:48:24 -0800 Subject: [PATCH 138/215] change the poke keyword from primary to root --- util/validator.go | 2 +- util/validator_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/validator.go b/util/validator.go index 2bab589..bbd269e 100644 --- a/util/validator.go +++ b/util/validator.go @@ -65,7 +65,7 @@ func ValidatePokeQuery(values url.Values) (string, error) { return routeStr, nil } - return "primary", nil + return "root", nil } func ValidateQueryParams(r *http.Request, validSubdocIdMap map[string]int, fields log.Fields) error { diff --git a/util/validator_test.go b/util/validator_test.go index 4cfb97a..83ac7f2 100644 --- a/util/validator_test.go +++ b/util/validator_test.go @@ -91,7 +91,7 @@ func TestValidatePokeQuery(t *testing.T) { delete(values, "doc") s, err = ValidatePokeQuery(values) assert.NilError(t, err) - assert.Equal(t, s, "primary") + assert.Equal(t, s, "root") values["doc"] = []string{ "primary", From bf96308d50cb352dd44ddf85037a31561c78e0be Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Mon, 23 Dec 2024 00:48:59 -0800 Subject: [PATCH 139/215] Set Otel Span attribute for x-cl-expt --- http/poke_handler.go | 8 ++++++++ http/webconfig_server.go | 16 ---------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index fb25745..24d1d69 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -30,11 +30,18 @@ import ( "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" ) +const moracideTag = "X-Cl-Experiment" + func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { + moracideTagVal := r.Header.Get(moracideTag) + if moracideTagVal == "" { + moracideTagVal = "false" + } // tracing propagation ctx := r.Context() @@ -42,6 +49,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) span := trace.SpanFromContext(ctx) defer span.End() + span.SetAttributes(attribute.String(moracideTag, moracideTagVal)) tracer := otel.Tracer(s.otelTracer.appName) ctx, childSpan := tracer.Start(ctx, s.otelTracer.opName) defer childSpan.End() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index daf331e..cd33e81 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -21,7 +21,6 @@ import ( "context" "crypto/tls" "encoding/base64" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -965,21 +964,6 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -// extract traceparent from the header -func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { - inTraceparent := r.Header.Get(common.HeaderTraceparent) - if len(inTraceparent) == 55 { - traceID = inTraceparent[3:35] - traceFlags = inTraceparent[53:55] - } - return -} - -func hexStringToBytes(hexString string) []byte { - bytes, _ := hex.DecodeString(hexString) - return bytes -} - func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields) { tfields := common.CopyCoreLogFields(fields) From 228e9111d0cb89f772b3c960069fe0de3b565302 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Sun, 29 Dec 2024 21:45:46 -0800 Subject: [PATCH 140/215] Migrate/adapt/rewrite tracing --- config/sample_webconfig.conf | 16 ++- go.mod | 3 + http/http_client.go | 60 +++++++- http/otel.go | 208 ---------------------------- http/poke_handler.go | 19 --- http/response.go | 36 +++++ http/router.go | 5 +- http/webconfig_server.go | 58 ++++---- main.go | 2 +- tracing/ctx.go | 19 +++ tracing/homegrown.go | 230 +++++++++++++++++++++++++++++++ tracing/homegrown_test.go | 231 +++++++++++++++++++++++++++++++ tracing/otel.go | 256 +++++++++++++++++++++++++++++++++++ tracing/span.go | 190 ++++++++++++++++++++++++++ tracing/tracer.go | 99 ++++++++++++++ 15 files changed, 1168 insertions(+), 264 deletions(-) delete mode 100644 http/otel.go create mode 100644 tracing/ctx.go create mode 100644 tracing/homegrown.go create mode 100644 tracing/homegrown_test.go create mode 100644 tracing/otel.go create mode 100644 tracing/span.go create mode 100644 tracing/tracer.go diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 6d2b232..e5e93f8 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -4,17 +4,23 @@ webconfig { } panic_exit_enabled = false - traceparent_parent_id = "0000000000000001" - tracestate_vendor_id = "webconfig" - opentelemetry { + tracing { + moracide_tag_prefix = "X-Cl-Experiment" + homegrown_algorithm { + app_id = "0000000000000001" + xpc_trace_propagation = false + xpc_trace_generation = false + } + opentelemetry { + enabled = false endpoint = "127.0.0.1:4318" + operation_name = "http.request" // Allowed values: "noop", "stdout", "http" // "noop" will generate no trace // "stdout" will use stdoutTracer and output spans to stdout // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" - env_name = "dev" - operation_name = "http.request" + } } // build info diff --git a/go.mod b/go.mod index 95108f9..e085f6d 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/sirupsen/logrus v1.9.0 + github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 @@ -56,6 +57,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -73,4 +75,5 @@ require ( google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/http/http_client.go b/http/http_client.go index 9661cbf..5ec0e72 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -36,9 +36,9 @@ import ( "github.com/go-akka/configuration" "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/tracing" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel/propagation" ) const ( @@ -112,6 +112,13 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl func (c *HttpClient) Do(ctx context.Context, method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields) + var respMoracideTagsFound bool + defer func(found *bool) { + if !*found { + log.Debugf("http_client: no moracide tags in response") + } + }(&respMoracideTagsFound) + // verify a response is received var req *http.Request var err error @@ -152,6 +159,7 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h req.Header.Set(common.HeaderUserAgent, c.userAgent) } + c.addMoracideTags(header, auditFields) logHeader := header.Clone() auth := logHeader.Get("Authorization") if len(auth) > 0 { @@ -207,9 +215,6 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h startTime := time.Now() - // the core http call - propagator := propagation.TraceContext{} - propagator.Inject(ctx, propagation.HeaderCarrier(req.Header)) res, err := c.Client.Do(req) // err should be *url.Error @@ -219,6 +224,31 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h delete(fields, bodyKey) + // We want to capture any errs returned by server + // i.e. timeout, 503 etc. wouldn't have a valid resp, but possible that + // the err returned by http.Do actually includes an err returned by the backend + // In which case, resp would be non-nil + moracideTagPrefix := strings.ToLower(tracing.GetMoracideTagPrefix()) + if res != nil { + for headerKey, headerVals := range res.Header { + if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { + respMoracideTagsFound = true + log.Debugf("http_client: moracide tag %s = %s found in response", headerKey, headerVals[0]) + if len(headerVals) > 1 { + log.Debugf("Tracing: moracide tag key = %s, has multiple values = %+v", headerKey, headerVals) + } + val := "false" + for _, v := range headerVals { + if v == "true" { + val = v + break + } + } + auditFields["resp_"+headerKey] = headerVals[0] + log.Debugf("Tracing: found moracide tag in response key = %s, val = %s", headerKey, val) + } + } + } var endMessage string if retry > 0 { endMessage = fmt.Sprintf("%v retry=%v ends", loggerName, retry) @@ -394,3 +424,25 @@ func (c *HttpClient) StatusHandler(status int) StatusHandlerFunc { } return nil } + +// addMoracideTags - if ctx has a moracide tag as a header, add it to the headers +// Also add traceparent, tracestate headers +func (c *HttpClient) addMoracideTags(header http.Header, fields log.Fields) { + moracideTagPrefix := strings.ToLower("req_"+tracing.GetMoracideTagPrefix()) + for key, val := range fields { + if key == "out_traceparent" { + header.Set("traceparent", val.(string)) + } + if key == "out_tracestate" { + header.Set("tracestate", val.(string)) + } + if len(key) < 5 { + // Should be of the form "req_x-cl-experiment" + continue + } + if strings.HasPrefix(strings.ToLower(key), moracideTagPrefix) { + log.Debugf("Adding moracide tag %s = %s to outgoing req", key, val) + header.Set(key[4:], val.(string)) + } + } +} diff --git a/http/otel.go b/http/otel.go deleted file mode 100644 index 468035f..0000000 --- a/http/otel.go +++ /dev/null @@ -1,208 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 -*/ -package http - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" - "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/trace" - "go.opentelemetry.io/otel/trace/noop" - - "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" -) - -// Tracing contains the core dependencies to make tracing possible across an application. -type otelTracing struct { - providerName string - envName string - appName string - opName string - tracerProvider trace.TracerProvider - propagator propagation.TextMapPropagator - tracer trace.Tracer -} - -type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) - -var ( - ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") - ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") - providersConfig = map[string]providerConstructor{ - "http": httpTraceProvider, - "stdout": stdoutTraceProvider, - "noop": noopTraceProvider, - } - - otelTracer otelTracing -) - -// DefaultTracerProvider is used when no provider is given. -// The Noop tracer provider turns all tracing related operations into -// noops essentially disabling tracing. -const defaultTracerProvider = "noop" - -// newOtel creates a structure with components that apps can use to initialize OpenTelemetry -// tracing instrumentation code. -func newOtel(conf *configuration.Config) (*otelTracing, error) { - if IsNoopTracing(conf) { - log.Debug("open telemetry tracing disabled (noop)") - } else { - log.Debug("opentelemetry tracing enabled") - } - - otelTracer.appName = conf.GetString("webconfig.app_name") - otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") - otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") - tracerProvider, err := newTracerProvider(conf) - if err != nil { - return &otelTracer, err - } - otelTracer.tracerProvider = tracerProvider - otel.SetTracerProvider(tracerProvider) - - // Set up propagator. - prop := newPropagator() - otelTracer.propagator = prop - otel.SetTextMapPropagator(prop) - - otelTracer.tracer = otel.Tracer(otelTracer.appName) - return &otelTracer, nil -} - -// IsNoopTracing returns true if the provider is set to "noop" -func IsNoopTracing(conf *configuration.Config) bool { - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - return strings.EqualFold(providerName, "noop") -} - -// TracerProvider returns the tracer provider component. By default, the noop -// tracer provider is returned. -func (t otelTracing) TracerProvider() trace.TracerProvider { - if t.tracerProvider == nil { - return noop.NewTracerProvider() - } - return t.tracerProvider -} - -// Propagator returns the component that helps propagate trace context across -// API boundaries. By default, a W3C Trace Context format propagator is returned. -func (t otelTracing) Propagator() propagation.TextMapPropagator { - if t.propagator == nil { - return propagation.TraceContext{} - } - return t.propagator -} - -func newPropagator() propagation.TextMapPropagator { - return propagation.NewCompositeTextMapPropagator( - propagation.TraceContext{}, - propagation.Baggage{}, - ) -} - -// newTracerProvider creates the TracerProvider based on config setting -// If no config setting, a noop tracerProvider will be returned. -func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - if len(providerName) == 0 { - providerName = defaultTracerProvider - } - // Handling camelcase of provider. - providerName = strings.ToLower(providerName) - providerConfig := providersConfig[providerName] - if providerConfig == nil { - return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) - } - - traceProvider, err := providerConfig(conf) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) - } - return traceProvider, nil -} - -func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - return noop.NewTracerProvider(), nil -} - -func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - option := stdouttrace.WithPrettyPrint() - exporter, err := stdouttrace.New(option) - if err != nil { - return nil, err - } - tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), - sdktrace.WithBatcher(exporter, - // Default is 5s. Set to 1s for demonstrative purposes. - sdktrace.WithBatchTimeout(time.Second)), - sdktrace.WithResource( - resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(otelTracer.appName), - semconv.ServiceNamespaceKey.String(otelTracer.envName), - ), - ), - ) - return tp, nil -} - -func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - // Send traces over HTTP - endpoint := conf.GetString("webconfig.opentelemetry.endpoint") - if endpoint == "" { - return nil, ErrTracerProviderBuildFailed - } - exporter, err := otlptracehttp.New(context.Background(), - otlptracehttp.WithEndpoint(endpoint), - otlptracehttp.WithInsecure(), - ) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) - } - - return sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), - sdktrace.WithResource( - resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(otelTracer.appName), - semconv.ServiceNamespaceKey.String(otelTracer.envName), - ), - ), - ), nil -} - -func (s *WebconfigServer) OtelShutdown() { - sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) - if ok && sdkTraceProvider != nil { - sdkTraceProvider.Shutdown(context.TODO()) - } -} diff --git a/http/poke_handler.go b/http/poke_handler.go index 24d1d69..902441b 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -29,31 +29,12 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/trace" ) -const moracideTag = "X-Cl-Experiment" - func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { - moracideTagVal := r.Header.Get(moracideTag) - if moracideTagVal == "" { - moracideTagVal = "false" - } // tracing propagation ctx := r.Context() - propagator := propagation.TraceContext{} - ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) - span := trace.SpanFromContext(ctx) - defer span.End() - span.SetAttributes(attribute.String(moracideTag, moracideTagVal)) - tracer := otel.Tracer(s.otelTracer.appName) - ctx, childSpan := tracer.Start(ctx, s.otelTracer.opName) - defer childSpan.End() - // handler params := mux.Vars(r) mac := params["mac"] diff --git a/http/response.go b/http/response.go index 7107bfa..5fbd1bc 100644 --- a/http/response.go +++ b/http/response.go @@ -21,8 +21,11 @@ import ( "encoding/json" "fmt" "net/http" + "strings" + log "github.com/sirupsen/logrus" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/tracing" ) const ( @@ -54,6 +57,7 @@ func WriteByMarshal(w http.ResponseWriter, status int, o interface{}) { return } w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(status) w.Write(rbytes) } @@ -107,6 +111,7 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, rbytes = []byte(fmt.Sprintf(OkResponseTemplate, s, stateText, updatedTime)) } w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(http.StatusOK) w.Write(rbytes) } @@ -114,6 +119,7 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, // this is used to return default tr-181 payload while the cpe is not in the db func WriteContentTypeAndResponse(w http.ResponseWriter, rbytes []byte, version string, contentType string) { w.Header().Set(common.HeaderContentType, contentType) + addMoracideTagsAsResponseHeaders(w) w.Header().Set(common.HeaderEtag, version) w.WriteHeader(http.StatusOK) w.Write(rbytes) @@ -152,12 +158,42 @@ func WriteResponseBytes(w http.ResponseWriter, rbytes []byte, statusCode int, va if len(vargs) > 0 { w.Header().Set(common.HeaderContentType, vargs[0]) } + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(statusCode) w.Write(rbytes) } func WriteFactoryResetResponse(w http.ResponseWriter) { w.Header().Set(common.HeaderContentType, common.MultipartContentType) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(http.StatusOK) w.Write([]byte{}) } + +func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { + xw, ok := w.(*XResponseWriter) + if !ok { + return + } + + reqMoracideTagPrefix := strings.ToLower("req_"+tracing.GetMoracideTagPrefix()) + respMoracideTagPrefix := strings.ToLower("resp_"+tracing.GetMoracideTagPrefix()) + fields := xw.Audit() + moracideTags := make(map[string]string) + for key, val := range fields { + if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { + log.Debugf("Adding moracide tag from req %s = %s to response", key, val) + moracideTags[key[4:]] = val.(string) + } + if strings.HasPrefix(strings.ToLower(key), respMoracideTagPrefix) { + log.Debugf("Adding moracide tag from resp %s = %s to response", key, val) + realKey := key[5:] + if existingVal, ok := moracideTags[realKey]; !ok || (ok && existingVal != "true") { + moracideTags[realKey] = val.(string) + } + } + } + for key, val := range moracideTags { + w.Header().Set(key, val) + } +} diff --git a/http/router.go b/http/router.go index 97348c1..4969326 100644 --- a/http/router.go +++ b/http/router.go @@ -19,6 +19,7 @@ package http import ( "github.com/gorilla/mux" + "github.com/rdkcentral/webconfig/tracing" ) func (s *WebconfigServer) AddBaseRoutes(testOnly bool, router *mux.Router) { @@ -95,9 +96,9 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.TestingMiddleware) } else { if s.ServerApiTokenAuthEnabled() { - sub2.Use(s.ApiMiddleware) + sub2.Use(tracing.SpanMiddleware, s.ApiMiddleware) } else { - sub2.Use(s.NoAuthMiddleware) + sub2.Use(tracing.SpanMiddleware, s.NoAuthMiddleware) } } sub2.HandleFunc("", s.PokeHandler).Methods("POST") diff --git a/http/webconfig_server.go b/http/webconfig_server.go index cd33e81..fbc7d0c 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -40,6 +40,7 @@ import ( "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" + "github.com/rdkcentral/webconfig/tracing" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -93,6 +94,7 @@ type WebconfigServer struct { *MqttConnector *UpstreamConnector sarama.AsyncProducer + *tracing.XpcTracer tlsConfig *tls.Config notLoggedHeaders []string metricsEnabled bool @@ -109,7 +111,6 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool - otelTracer *otelTracing // For OpenTelemetry Tracing kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool @@ -254,13 +255,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validPartners = append(validPartners, strings.ToLower(p)) } - traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) - tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) - otelTracer, err := newOtel(conf) - if err != nil { - // Just log err and continue - log.Error("Could not initialize open telemetry for tracing, but continuing") - } + xpcTracer := tracing.NewXpcTracer(conf) supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) @@ -321,9 +316,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validateMacEnabled: validateMacEnabled, validPartners: validPartners, jwksEnabled: jwksEnabled, - traceparentParentID: traceparentParentID, - tracestateVendorID: tracestateVendorID, - otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, @@ -331,11 +323,16 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer queryParamsValidationEnabled: queryParamsValidationEnabled, minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, + XpcTracer: xpcTracer, } return ws } +func (s *WebconfigServer) Stop() { + tracing.StopXpcTracer() +} + func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { xw := NewXResponseWriter(w) @@ -749,27 +746,34 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques traceId = traceparent[3:35] } - // extract tracestate from the header - tracestate := r.Header.Get(common.HeaderTracestate) - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { auditId = util.GetAuditId() } + + // traceparent handling for E2E tracing + xpcTrace := tracing.NewXpcTrace(r) + headerMap := util.HeaderToMap(header) fields := log.Fields{ - "path": r.URL.String(), - "method": r.Method, - "audit_id": auditId, - "remote_ip": remoteIp, - "host_name": host, - "header": headerMap, - "logger": "request", - "trace_id": traceId, - "app_name": s.AppName(), - "traceparent": traceparent, - "tracestate": tracestate, + "path": r.URL.String(), + "method": r.Method, + "audit_id": auditId, + "remote_ip": remoteIp, + "host_name": host, + "header": headerMap, + "logger": "request", + "trace_id": traceId, + "app_name": s.AppName(), + "traceparent": xpcTrace.ReqTraceparent, + "tracestate": xpcTrace.ReqTracestate, + "out_traceparent": xpcTrace.OutTraceparent, + "out_tracestate": xpcTrace.OutTracestate, + "xpc_trace": xpcTrace, + } + for key, val := range xpcTrace.ReqMoracideTags { + fields["req_"+key] = val } userAgent := r.UserAgent() @@ -828,6 +832,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques log.WithFields(tfields).Info("request starts") } + xwriter.LogDebug(r, "tracing", fmt.Sprintf("Trace final out_traceparent %s out_traceState %s", xpcTrace.OutTraceparent, xpcTrace.OutTracestate)) return xwriter } @@ -897,6 +902,9 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["duration"] = duration fields["logger"] = "request" + tracing.SetSpanStatusCode(fields) + tracing.SetSpanMoracideTags(fields) + var userAgent string if itf, ok := fields["user_agent"]; ok { userAgent = itf.(string) diff --git a/main.go b/main.go index 8a47cb9..284012c 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) - defer server.OtelShutdown() + defer server.Stop() // setup logging logFile := server.GetString("webconfig.log.file") diff --git a/tracing/ctx.go b/tracing/ctx.go new file mode 100644 index 0000000..204e084 --- /dev/null +++ b/tracing/ctx.go @@ -0,0 +1,19 @@ +package tracing + +import "context" + +type contextKey string + +func (c contextKey) String() string { + return string(c) +} + +// SetContext - setting context for logging +func SetContext(ctx context.Context, ctxName string, ctxValue interface{}) context.Context { + return context.WithValue(ctx, contextKey(ctxName), ctxValue) +} + +// GetContext - getting context value from context +func GetContext(ctx context.Context, key string) interface{} { + return ctx.Value(contextKey(key)) +} diff --git a/tracing/homegrown.go b/tracing/homegrown.go new file mode 100644 index 0000000..40f4a86 --- /dev/null +++ b/tracing/homegrown.go @@ -0,0 +1,230 @@ +/** + * @license + * Copyright Comcast. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @author RV + +/* + W3C Doc: https://www.w3.org/TR/trace-context-1 + + Usage: + 1. AppID should be set in the config + Config Template (from HC): + ------------------------------------------- + "Tracing": { + // This identifies the microservice + "AppID": "0000000000000001" + }, + ------------------------------------------- + 2. Call GetTracingHeader with a http Request, and it will return a modified + traceparent/tracestate header + 3. Set it in req.Header + 4. If the generate flag is set in GetTracingHeader and if the input req doesn't + have a traceparent header, these two headers will be created + TODO: + Also pass a Kafka message as an input to do the above + + Spec: + traceparent header has 4 sections + "---" + version is 2 digits, traceID is 32 digits, callerIF is 16, and flags is 2 + if traceparent is present in headers, replace the callerID with app's ID + + Template - hc's ID defaults to "0000000000000001" and can be changed in config + + If tracestate header exists, add "=," to the beginning of the header + else set it to "=" + + As of now, we don't generate traceparent if the header is not passed in + Any trace flag setting that tells us to regenerate the traceID is ignored. + Version, TraceID, and trace flags are passed as-is. Only CallerID is modified + TraceState will be generated if it doesn't exist + + Epic: XPC-16688 + HC Ticket: ODP-25026 + Xapproxy Ticket: XPC-17946 + + Changes: + otel implements their own traceparent/tracestate propagation and this clashes with + our home-grown propagation. If otel is enabled, suppress the home-grown propagation. + TODO: Change homegrown propagation too to a per API basis instead of per app basis +*/ + +package tracing + +import ( + "fmt" + "math/rand" + "net/http" + "strings" + // "sync" + "time" + log "github.com/sirupsen/logrus" + "github.com/rdkcentral/webconfig/common" +) + +const ( + // These are two completely internal + traceVersion = "01" + traceFlags = "00" + + defaultAppID = "0000000000000001" + + traceAppIDNotSetErr = "Tracing: AppID not set" +) + +type TraceHeader struct { + Key, Value string +} + +func GetTraceHeaders(r *http.Request, generateTrace bool) ([]TraceHeader, error) { + // This is not concurrency safe (hat tip to Jay) + // Hence it is instantiated in the goroutine that handles the API req + // Google search says this is inexpensive and acceptable + // Prefer not to use a mutex as multiple Kafka reqs can wait on this lock + + randGen := rand.NewSource(time.Now().UnixNano()) + + headers := make([]TraceHeader, 0) + traceParent := r.Header.Get("traceparent") + traceComponents := strings.Split(traceParent, "-") + if len(traceComponents) != 4 { + // traceparent header is either incorrectly formatted or doesn't exist + if !generateTrace { + // Don't create traceparent header if flag is not set + // Don't bother about tracestate header + return headers, nil + } + + if xpcTracer.appID == "" { + return headers, fmt.Errorf(traceAppIDNotSetErr) + } + // Create a new traceparent header + traceParent = fmt.Sprintf("%s-%016x%016x-%s-%s", + traceVersion, + // genInt63(randGen), genInt63(randGen), + randGen.Int63(), randGen.Int63(), + xpcTracer.appID, + traceFlags) + } else { + // Replace the appID part + if xpcTracer.appID == "" { + return headers, fmt.Errorf(traceAppIDNotSetErr) + } + // Create a new traceparent header + traceParent = fmt.Sprintf("%s-%s-%s-%s", + traceComponents[0], + traceComponents[1], + xpcTracer.appID, + traceComponents[3]) + } + headers = append(headers, TraceHeader{ + Key: "traceparent", + Value: traceParent, + }) + + traceState := r.Header.Get("tracestate") + if traceState == "" { + // Create tracestate header if it doesn't exist + traceState = fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID) + } else { + // Add current app to traceState + traceState = fmt.Sprintf("%s=%s,%s", xpcTracer.appName, xpcTracer.appID, traceState) + } + headers = append(headers, TraceHeader{ + Key: "tracestate", + Value: traceState, + }) + return headers, nil +} + +func tryHomegrownTpTs(r *http.Request, xpcTrace *XpcTrace) { + // if homegrownTracePropagation is true, we modify existing traceparent/tracestate and forward them + // but we will not generate them if they don't exist + // if homegrownTraceGeneration is true, and there is no tp/ts, we generate it + + if !xpcTracer.homegrownTracePropagation { + return + } + // ctx := r.Context() + log.Debug("Tracing: using homegrown traceparent propagation/generation") + if traceHeaders, err := GetTraceHeaders(r, xpcTracer.homegrownTraceGeneration); err == nil { + // TODO: Change homegrown propagation too to a per API basis instead of per app basis + // Reverse engineer otel basically + for _, h := range traceHeaders { + log.Debugf("Tracing: %s set to %s", h.Key, h.Value) + if h.Key == common.HeaderTraceparent { + xpcTrace.OutTraceparent = h.Value + } + if h.Key == common.HeaderTracestate { + xpcTrace.OutTracestate = h.Value + } + } + } else { + log.Errorf("Error in homegrown traceparent handling %+v", err) + } +} + +/* + // Benchmarking expt to compare mutex based random vs one source per goroutine + var ( + // This is for a benchmarking expt + global bool + globalLock sync.Mutex + globalRandGen = rand.NewSource(time.Now().UnixNano()) + + // Sample Benchmarking Results + // BenchmarkSharedRandGen-16 352419 3052 ns/op + // BenchmarkIndependentRandGen-16 670383 2755 ns/op + ) + + func setGlobal(g bool) { + global = g + } + + func genInt63(randGen rand.Source) int64 { + if global { + globalLock.Lock() + i := globalRandGen.Int63() + globalLock.Unlock() + return i + } + return randGen.Int63() + } +*/ + +/* + traceparentParentID string + tracestateVendorID string + otelEnabled bool + traceparentParentID: traceparentParentID, + tracestateVendorID: tracestateVendorID, +func (s *WebconfigServer) TraceparentParentID() string { + return s.traceparentParentID +} + +func (s *WebconfigServer) SetTraceparentParentID(x string) { + s.traceparentParentID = x +} + +func (s *WebconfigServer) TracestateVendorID() string { + return s.tracestateVendorID +} + +func (s *WebconfigServer) SetTracestateVendorID(x string) { + s.tracestateVendorID = x +} +*/ diff --git a/tracing/homegrown_test.go b/tracing/homegrown_test.go new file mode 100644 index 0000000..6e36c36 --- /dev/null +++ b/tracing/homegrown_test.go @@ -0,0 +1,231 @@ +/** + * @license + * Copyright Comcast. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @author RV + +package tracing + +import ( + "bytes" + "fmt" + "net/http" + "strings" + // "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + testTraceID = "0123456789abcdef0123456789abcdef" + testCallerID = "fedcba9876543210" + testTraceParent = traceVersion + "-" + testTraceID + "-" + testCallerID + "-" + traceFlags + + testCallerName = "test" + testTraceState = testCallerName + "=" + testCallerID + + testMyName = "whatever" + testMyID = "0123456789abcdef" +) + +func TestAppIDNotSet(t *testing.T) { + // Mimic appName/appID not set by explicitly setting them to empty strs + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = "" + xpcTracer.appName = "" + + req := makeReq(t) + req.Header.Add("traceparent", testTraceParent) + _, err := GetTraceHeaders(req, false) + require.Equal(t, traceAppIDNotSetErr, err.Error()) + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestNoGenerate(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + req := makeReq(t) + headers, _ := GetTraceHeaders(req, false) + require.Equal(t, 0, len(headers)) + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestOnlyParentHeader(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + req := makeReq(t) + req.Header.Add("traceparent", testTraceParent) + + headers, _ := GetTraceHeaders(req, false) + require.Equal(t, 2, len(headers)) + validateTraceParentHeader(t, headers) + for _, h := range headers { + if h.Key == "tracestate" { + require.Equal(t, fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID), h.Value) + } + } + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestOnlyStateHeader(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + req := makeReq(t) + req.Header.Add("tracestate", testTraceState) + headers, _ := GetTraceHeaders(req, false) + require.Equal(t, 0, len(headers)) + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestParentPlusStateHeaders(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + req := makeReq(t) + req.Header.Add("traceparent", testTraceParent) + req.Header.Add("tracestate", testTraceState) + headers, _ := GetTraceHeaders(req, false) + require.Equal(t, 2, len(headers)) + validateTraceParentHeader(t, headers) + for _, h := range headers { + if h.Key == "tracestate" { + require.Equal(t, fmt.Sprintf("%s=%s,%s", xpcTracer.appName, xpcTracer.appID,testTraceState), h.Value) + } + } + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestTraceHeaderGeneration(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + testTraceHeaderGeneration(t) + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func validateTraceParentHeader(t *testing.T, headers []TraceHeader) { + for _, h := range headers { + if h.Key == "traceparent" { + traceComponents := strings.Split(h.Value, "-") + require.Equal(t, 4, len(traceComponents), 4) + require.Equal(t, traceVersion, traceComponents[0]) + require.Equal(t, testTraceID, traceComponents[1]) + require.Equal(t, xpcTracer.appID, traceComponents[2]) + require.Equal(t, traceFlags, traceComponents[3]) + } + } +} + +func makeReq(t *testing.T) *http.Request { + url := "/testurl" + var body []byte + req, err := http.NewRequest("GET", url, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + return req +} + +func testTraceHeaderGeneration(t *testing.T) { + req := makeReq(t) + headers, _ := GetTraceHeaders(req, true) + require.Equal(t, 2, len(headers)) + for _, h := range headers { + if h.Key == "traceparent" { + traceComponents := strings.Split(h.Value, "-") + require.Equal(t, 4, len(traceComponents)) + require.Equal(t, traceVersion, traceComponents[0]) + // We don't know the ID here, we can only check it is 32 digits long + require.Equal(t, 32, len(traceComponents[1])) + require.Equal(t, xpcTracer.appID, traceComponents[2]) + require.Equal(t, traceFlags, traceComponents[3]) + } + if h.Key == "tracestate" { + require.Equal(t, fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID), h.Value) + } + } +} + +/* + // These tests/benchmarks are to test mutex based random generation + // We are not going to use mutex as of now, as using one source + // per goroutine approach is faster + func TestTraceHeaderGenerationGlobal(t *testing.T) { + setGlobal(true) + testTraceHeaderGeneration(t) + setGlobal(false) + } + + func BenchmarkSharedRandGen(b *testing.B) { + setGlobal(true) + url := "/testurl" + var body []byte + req, _ := http.NewRequest("GET", url, bytes.NewReader(body)) + var wg sync.WaitGroup + for n := 0; n < b.N; n++ { + wg.Add(1) + go func() { + defer wg.Done() + GetTraceHeaders(req, true) + }() + } + wg.Wait() + setGlobal(false) + } + + func BenchmarkIndependentRandGen(b *testing.B) { + url := "/testurl" + var body []byte + req, _ := http.NewRequest("GET", url, bytes.NewReader(body)) + var wg sync.WaitGroup + + for n := 0; n < b.N; n++ { + wg.Add(1) + go func() { + defer wg.Done() + GetTraceHeaders(req, true) + }() + } + wg.Wait() + } +*/ diff --git a/tracing/otel.go b/tracing/otel.go new file mode 100644 index 0000000..22e4728 --- /dev/null +++ b/tracing/otel.go @@ -0,0 +1,256 @@ +package tracing + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/gorilla/mux" + "github.com/go-akka/configuration" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + log "github.com/sirupsen/logrus" +) + +type providerConstructor func() (oteltrace.TracerProvider, error) + +var ( + providerBuilders = map[string]providerConstructor{ + "http": otelHttpTraceProvider, + "stdout": otelStdoutTraceProvider, + "noop": otelNoopTraceProvider, + } +) + +// defaultOtelTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultOtelTracerProvider = "noop" + +// initOtel - initialize OpenTelemetry constructs +// tracing instrumentation code. +func otelInit(conf *configuration.Config) { + xpcTracer.OtelEnabled = conf.GetBoolean("webconfig.tracing.otel.enabled") + if !xpcTracer.OtelEnabled { + return + } + xpcTracer.otelProvider = strings.ToLower(conf.GetString("webconfig.tracing.otel.provider", defaultOtelTracerProvider)) + if xpcTracer.otelProvider == "" { + xpcTracer.otelProvider = defaultOtelTracerProvider + } + if xpcTracer.otelProvider == defaultOtelTracerProvider { + log.Debug("otel disabled, noop provider") + return + } + + log.Debug("otel enabled") + xpcTracer.otelEndpoint = conf.GetString("webconfig.tracing.otel.endpoint") + xpcTracer.otelOpName = conf.GetString("webconfig.tracing.otel.operation_name") + + if providerBuilder := providerBuilders[xpcTracer.otelProvider]; providerBuilder == nil { + log.Errorf("no builder func for otel provider %s", xpcTracer.otelProvider) + return + } else { + var err error + if xpcTracer.otelTracerProvider, err = providerBuilder(); err != nil { + log.Errorf("building otel provider for %s failed with %v", xpcTracer.otelProvider, err) + return + } + } + otel.SetTracerProvider(xpcTracer.otelTracerProvider) + + // Set up propagator. + xpcTracer.otelPropagator = propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) + otel.SetTextMapPropagator(xpcTracer.otelPropagator) + + xpcTracer.otelTracer = otel.Tracer(xpcTracer.appName) +} + +func otelNoopTraceProvider() (oteltrace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func otelStdoutTraceProvider() (oteltrace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), + ), + ), + ) + return tp, nil +} + +func otelHttpTraceProvider() (oteltrace.TracerProvider, error) { + // Send traces over HTTP + if xpcTracer.otelEndpoint == "" { + return nil, fmt.Errorf("building http otel provider failure, no endpoint specified") + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(xpcTracer.otelEndpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("building http otel provider failed with %v", err) + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), + ), + ), + ), nil +} + +func otelShutdown() { + sdkTraceProvider, ok := xpcTracer.otelTracerProvider.(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} + +// otelOpName should return "http.request" by default +func otelOpName() string { + opName := xpcTracer.otelOpName + if opName == "" { + opName = "http.request" + } + return opName +} + +func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { + ctx := r.Context() + var otelSpan oteltrace.Span + if !xpcTracer.OtelEnabled { + return ctx, otelSpan + } + + pathTemplate := "placeholder" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + var err error + pathTemplate, err = mux.CurrentRoute(r).GetPathTemplate() + if err != nil { + log.Debugf("unable to get path template: %v", err) + } + } + resourceName := r.Method + " " + pathTemplate + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) + + /* + required span attribute: HTTPMethodKey = attribute.Key("http.method") + required span attribute: HTTPRouteKey = attribute.Key("http.route") + required span attribute: HTTPStatusCodeKey = attribute.Key("http.status_code") + required span attribute: HTTPURLKey = attribute.Key("http.url") + custom Comcast attribute: X-Cl-Experiment: true/false + additional: env, operation.name, http.url_details.path + */ + ctx, otelSpan = xpcTracer.otelTracer.Start(ctx, otelOpName(), + oteltrace.WithSpanKind(oteltrace.SpanKindServer), + oteltrace.WithAttributes( + attribute.String("env", xpcTracer.appEnv), + attribute.String("http.method", r.Method), + attribute.String("http.route", pathTemplate), + attribute.String("http.url", r.URL.String()), + attribute.String("http.url_details.path", r.URL.Path), + attribute.String("operation.name", otelOpName()), + ), + ) + if xpcTracer.rgn != "" { + rgnAttr := attribute.String("region", xpcTracer.rgn) + otelSpan.SetAttributes(rgnAttr) + } + + log.Debugf("span started %s", resourceName) + log.Debugf("added span attribute key = env, value = %s", xpcTracer.appEnv) + log.Debugf("added span attribute key = http.method, value = %s", r.Method) + log.Debugf("added span attribute key = http.route, value = %s", pathTemplate) + log.Debugf("added span attribute key = http.url, value = %s", r.URL.String()) + log.Debugf("added span attribute key = http.url_details.path, value = %s", r.URL.Path) + log.Debugf("added span attribute key = operation.name, value = %s", otelOpName()) + + carrier := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, carrier) + for key, val := range carrier { + ctx = SetContext(ctx, "otel_"+key, val) + log.Debugf("OtelSpanCreation: otel %s = %s", key, val) + } + + ctx = SetContext(ctx, "otel_span", otelSpan) + return ctx, otelSpan +} + +func otelSetStatusCode(span oteltrace.Span, statusCode int) { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + log.Debugf("added span attribute key = http.status_code, value = %d", statusCode) + + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debugf("added span attribute key=http.response.error, value=%s", statusText) + } +} + +func otelEndSpan(span oteltrace.Span) { + if !xpcTracer.OtelEnabled { + return + } + span.End() +} + +func otelExtractParamsFromSpan(ctx context.Context, xpcTrace *XpcTrace) { + if !xpcTracer.OtelEnabled { + return + } + if tmp := GetContext(ctx, "otel_span"); tmp != nil { + if otelSpan, ok := tmp.(oteltrace.Span); ok { + xpcTrace.otelSpan = otelSpan + } + } + if xpcTrace.otelSpan == nil { + return + } + // if otel span is found, use the extracted traceparent and tracestate from the otel span + // We store the extracted values in ctx when we created the otel span + if tmp := GetContext(ctx, "otel_traceparent"); tmp != nil { + xpcTrace.otelTraceparent = tmp.(string) + log.Debugf("Tracing: otel traceparent = %s", xpcTrace.otelTraceparent) + xpcTrace.OutTraceparent = xpcTrace.otelTraceparent + } + if tmp := GetContext(ctx, "otel_tracestate"); tmp != nil { + xpcTrace.otelTracestate = tmp.(string) + log.Debugf("Tracing: otel tracestate = %s", xpcTrace.otelTracestate) + xpcTrace.OutTracestate = xpcTrace.otelTracestate + } +} diff --git a/tracing/span.go b/tracing/span.go new file mode 100644 index 0000000..137ac8b --- /dev/null +++ b/tracing/span.go @@ -0,0 +1,190 @@ +package tracing + +import ( + "net/http" + "strings" + + log "github.com/sirupsen/logrus" + + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/attribute" + + "github.com/rdkcentral/webconfig/common" +) + +// SpanMiddleware is a middleware that creates a new span for each incoming request. +func SpanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // For Otel, create the span explicitly + if xpcTracer.OtelEnabled { + ctx, otelSpan := otelNewSpan(r) + r = r.WithContext(ctx) + defer otelEndSpan(otelSpan) + } + next.ServeHTTP(w, r) + }) +} + +// XpcTrace is a carrier/baggage struct to extract data from spans, request headers for usage later +// Store the trace in ctx for easy retrieval. +// Ideal place to store it is ofc, xw +// But because of legacy reasons, xw is not always available in the API flow +type XpcTrace struct { + // This is a bit of overengineering, but multiple tags are possible + // e.g. X-Cl-Experiment-1, X-Cl-Experiment-xapproxy, X-Cl-Experiement-webconfig-25.1.1.1... + // For every key found in either req or resp, an explicit value of true/false will be set as an otel attribute + // or an otel span attribute + ReqMoracideTags map[string]string // These are request headers prefixed with MoracideTagPrefix + // The response moracide tags are stored in xw.audit + + // traceparent, tracestate can be set as req headers, may be extracted from otel spans + // These need to be propagated to any http calls we make + // Order of priority; use the value extracted from otel span; + // if no otel span as well, use the value in req headers (Note: this will create islands as both + // the app and its children will have the same tracestate + // If homegrown span modification is enabled in config, use it as a last resort. Otherwise, nothing + // will be passed to the child http calls, creating islands + // If any source is found, then it will be propagated to all child http calls + // TODO; also add this to Kafka headers, SNS message attributes + otelTraceparent string + otelTracestate string + ReqTraceparent string + ReqTracestate string + OutTraceparent string + OutTracestate string + + // At the end of API flow, add the status code to OtelSpan; add the Moracide tags to the spans + otelSpan oteltrace.Span + + // These are not useful as of now, just set them for the sake of completion and future + AuditID string + MoneyTrace string + ReqUserAgent string + OutUserAgent string + + TraceID string // use the value in outTraceparent, otherwise MoneyTrace +} + +// NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs +// It can also generate traceparent/state using homegrown algorithms, but avoid this as a rule +func NewXpcTrace(r *http.Request) * XpcTrace { + var xpcTrace XpcTrace + xpcTrace.ReqMoracideTags = make(map[string]string) + + extractParamsFromReq(r, &xpcTrace) + + if xpcTracer.OtelEnabled { + otelExtractParamsFromSpan(r.Context(), &xpcTrace) + } + + if xpcTrace.OutTraceparent == "" { + // No traceparent in otel spans i.e. no otel spans + // No traceparent in incoming req either + // Note: If otel root span, ts can be empty, but tp can be set, so cannot check for ts being empty + tryHomegrownTpTs(r, &xpcTrace) + } + + return &xpcTrace +} + +func SetSpanStatusCode(fields log.Fields) { + var xpcTrace *XpcTrace + if tmp, ok := fields["xpc_trace"]; ok { + xpcTrace = tmp.(*XpcTrace) + } + if xpcTrace == nil { + // Something went wrong, cannot instrument this span + log.Error("instrumentation error, no trace info") + } + if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { + if tmp, ok := fields["status"]; ok { + statusCode := tmp.(int) + otelSetStatusCode(xpcTrace.otelSpan, statusCode) + } + } +} + +func SetSpanMoracideTags(fields log.Fields) { + var xpcTrace *XpcTrace + if tmp, ok := fields["xpc_trace"]; ok { + xpcTrace = tmp.(*XpcTrace) + } + if xpcTrace == nil { + // Something went wrong, cannot instrument this span + log.Error("instrumentation error, cannot set moracide tags, no trace info") + } + + if tmp, ok := fields["xpc_trace"]; ok { + xpcTrace = tmp.(*XpcTrace) + } + moracideTags := make(map[string]string) + reqMoracideTagPrefix := strings.ToLower("req_"+xpcTracer.MoracideTagPrefix) + respMoracideTagPrefix := strings.ToLower("resp_"+xpcTracer.MoracideTagPrefix) + + for key, val := range fields { + if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { + log.Debugf("Adding moracide tag from req %s = %s to response", key, val) + moracideTags[key[4:]] = val.(string) + } + if strings.HasPrefix(strings.ToLower(key), respMoracideTagPrefix) { + log.Debugf("Adding moracide tag from resp %s = %s to response", key, val) + realKey := key[5:] + if existingVal, ok := moracideTags[realKey]; !ok || (ok && existingVal != "true") { + moracideTags[realKey] = val.(string) + } + } + } + if len(moracideTags) == 0 { + // No moracide tags in request or any response + // So set at least one span tag, x-cl-expt: false + moracideTags[xpcTracer.MoracideTagPrefix] = "false" + } + for key, val := range moracideTags { + if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { + xpcTrace.otelSpan.SetAttributes(attribute.String(key, val)) + log.Debugf("added otel span moracide tag key = %s, value = %s", key, val) + } + } +} + +func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace) { + xpcTrace.ReqTraceparent = r.Header.Get(common.HeaderTraceparent) + xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) + xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent + xpcTrace.OutTracestate = xpcTrace.ReqTracestate + log.Debugf("Tracing: input traceparent : %s, tracestate : %s", xpcTrace.ReqTraceparent, xpcTrace.ReqTracestate) + + // AuditID is used internally within xdp subsystem + // We will not move this to tracing module + // xpcTrace.AuditID = r.Header.Get(AuditIDHeader) + // if xpcTrace.AuditID == "" { + // xpcTrace.AuditID = util.GetAuditId() + // } + + // Moneytrace is Comcast's original solution for tracing + // Unused for now, decide whether we need to merge existing moneytrace code here + // Being replaced by traceparent/tracestate headers anyway + // xpcTrace.MoneyTrace = r.Header.Get(MoneyTraceHeader) + + xpcTrace.ReqUserAgent = r.Header.Get(UserAgentHeader) + + // In future, -H 'X-Cl-Experiment-1', -H 'X-Cl-Experiment-oswebconfig'... OR 'X-Cl-Experiment-xapproxy_25.1.1.1' are all possible + // So walk through all headers and collect any header that starts with this prefix + moracideTagPrefix := strings.ToLower(xpcTracer.MoracideTagPrefix) + for headerKey, headerVals := range r.Header { + if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { + if len(headerVals) > 1 { + log.Debugf("Tracing: moracide tag key = %s, has multiple values = %+v", headerKey, headerVals) + } + val := "false" + for _, v := range headerVals { + if v == "true" { + val = v + break + } + } + xpcTrace.ReqMoracideTags[headerKey] = val + log.Debugf("Tracing: found moracide tag key = %s, val = %s, all vals = %+v", headerKey, val, headerVals) + } + } +} diff --git a/tracing/tracer.go b/tracing/tracer.go new file mode 100644 index 0000000..80fdcc1 --- /dev/null +++ b/tracing/tracer.go @@ -0,0 +1,99 @@ +package tracing + +import ( + "strings" + "os" + + "github.com/go-akka/configuration" + + oteltrace "go.opentelemetry.io/otel/trace" + otelpropagation "go.opentelemetry.io/otel/propagation" +) + +const ( + AuditIDHeader = "X-Auditid" + UserAgentHeader = "User-Agent" + + defaultMoracideTagPrefix = "X-Cl-Experiment" +) + +// XpcTracer is a wrapper around tracer setup +type XpcTracer struct { + OtelEnabled bool + MoracideTagPrefix string // Special request header for moracide expts e.g. canary deployments + + // internal vars used by Otel + appEnv string // set this to dev for red, staging for yellow and prod for green + appName string + appVersion string + appSHA string // unused + rgn string // AWS Region e.g. us-west-2, unused, use it as a otel span attribute + siteColor string // red/yellow/green, unused, use it as a otel span attribute + + // internal otel vars + otelEndpoint string + otelOpName string + otelProvider string + otelTracerProvider oteltrace.TracerProvider + otelPropagator otelpropagation.TextMapPropagator + otelTracer oteltrace.Tracer + + // internal vars for homegrown trace generation/modification + appID string // used in the homegrown traceparent,tracestate header generation + homegrownTracePropagation bool // Use homegrown algorithm for traceparent, tracestate modification + homegrownTraceGeneration bool // Generate traceparent etc. using homegrown algorithm if not present in incoming req +} + +var xpcTracer XpcTracer // global tracer + +func NewXpcTracer(conf *configuration.Config) *XpcTracer { + initAppData(conf) + initHomegrownTracing(conf) + + otelInit(conf) + + xpcTracer.MoracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", defaultMoracideTagPrefix) + return &xpcTracer +} + +// defer this func in the main of the app +func StopXpcTracer() { + otelShutdown() +} + +// Global func to access the moracide tag +func GetMoracideTagPrefix() string { + return xpcTracer.MoracideTagPrefix +} + +func initAppData(conf *configuration.Config) { + codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") + xpcTracer.appName = codeGitCommit[0] + if len(codeGitCommit) > 1 { + xpcTracer.appVersion = codeGitCommit[1] + } + if len(codeGitCommit) > 2 { + xpcTracer.appSHA = codeGitCommit[2] + } + + // Env vars + xpcTracer.appEnv = "dev" + siteColor := os.Getenv("SITE_COLOR") + if strings.EqualFold(siteColor, "yellow") { + xpcTracer.appEnv = "staging" + } else if strings.EqualFold(siteColor, "green") { + xpcTracer.appEnv = "prod" + } + xpcTracer.rgn = os.Getenv("SITE_REGION") +} + +func initHomegrownTracing(conf *configuration.Config) { + // TODO: Marshal these tracing params from config, ok to temporarily read each separately + xpcTracer.appID = conf.GetString("webconfig.tracing.homegrown_algorithm.app_id", defaultAppID) + xpcTracer.homegrownTracePropagation = conf.GetBoolean("webconfig.tracing.homegrown_algorithm.xpc_trace_propagation") + xpcTracer.homegrownTraceGeneration = conf.GetBoolean("webconfig.tracing.homegrown_algorithm.xpc_trace_generation") +} + +func GetServiceName() string { + return xpcTracer.appName +} From 0e5f43af8d65f463ffad6209d8b4b9f9e98b7121 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Mon, 30 Dec 2024 12:23:46 -0800 Subject: [PATCH 141/215] Remove all homegrown algorithm code as it is never going to be used --- tracing/homegrown.go | 230 ------------------------------------- tracing/homegrown_test.go | 231 -------------------------------------- tracing/span.go | 11 +- tracing/tracer.go | 15 --- 4 files changed, 1 insertion(+), 486 deletions(-) delete mode 100644 tracing/homegrown.go delete mode 100644 tracing/homegrown_test.go diff --git a/tracing/homegrown.go b/tracing/homegrown.go deleted file mode 100644 index 40f4a86..0000000 --- a/tracing/homegrown.go +++ /dev/null @@ -1,230 +0,0 @@ -/** - * @license - * Copyright Comcast. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @author RV - -/* - W3C Doc: https://www.w3.org/TR/trace-context-1 - - Usage: - 1. AppID should be set in the config - Config Template (from HC): - ------------------------------------------- - "Tracing": { - // This identifies the microservice - "AppID": "0000000000000001" - }, - ------------------------------------------- - 2. Call GetTracingHeader with a http Request, and it will return a modified - traceparent/tracestate header - 3. Set it in req.Header - 4. If the generate flag is set in GetTracingHeader and if the input req doesn't - have a traceparent header, these two headers will be created - TODO: - Also pass a Kafka message as an input to do the above - - Spec: - traceparent header has 4 sections - "---" - version is 2 digits, traceID is 32 digits, callerIF is 16, and flags is 2 - if traceparent is present in headers, replace the callerID with app's ID - - Template - hc's ID defaults to "0000000000000001" and can be changed in config - - If tracestate header exists, add "=," to the beginning of the header - else set it to "=" - - As of now, we don't generate traceparent if the header is not passed in - Any trace flag setting that tells us to regenerate the traceID is ignored. - Version, TraceID, and trace flags are passed as-is. Only CallerID is modified - TraceState will be generated if it doesn't exist - - Epic: XPC-16688 - HC Ticket: ODP-25026 - Xapproxy Ticket: XPC-17946 - - Changes: - otel implements their own traceparent/tracestate propagation and this clashes with - our home-grown propagation. If otel is enabled, suppress the home-grown propagation. - TODO: Change homegrown propagation too to a per API basis instead of per app basis -*/ - -package tracing - -import ( - "fmt" - "math/rand" - "net/http" - "strings" - // "sync" - "time" - log "github.com/sirupsen/logrus" - "github.com/rdkcentral/webconfig/common" -) - -const ( - // These are two completely internal - traceVersion = "01" - traceFlags = "00" - - defaultAppID = "0000000000000001" - - traceAppIDNotSetErr = "Tracing: AppID not set" -) - -type TraceHeader struct { - Key, Value string -} - -func GetTraceHeaders(r *http.Request, generateTrace bool) ([]TraceHeader, error) { - // This is not concurrency safe (hat tip to Jay) - // Hence it is instantiated in the goroutine that handles the API req - // Google search says this is inexpensive and acceptable - // Prefer not to use a mutex as multiple Kafka reqs can wait on this lock - - randGen := rand.NewSource(time.Now().UnixNano()) - - headers := make([]TraceHeader, 0) - traceParent := r.Header.Get("traceparent") - traceComponents := strings.Split(traceParent, "-") - if len(traceComponents) != 4 { - // traceparent header is either incorrectly formatted or doesn't exist - if !generateTrace { - // Don't create traceparent header if flag is not set - // Don't bother about tracestate header - return headers, nil - } - - if xpcTracer.appID == "" { - return headers, fmt.Errorf(traceAppIDNotSetErr) - } - // Create a new traceparent header - traceParent = fmt.Sprintf("%s-%016x%016x-%s-%s", - traceVersion, - // genInt63(randGen), genInt63(randGen), - randGen.Int63(), randGen.Int63(), - xpcTracer.appID, - traceFlags) - } else { - // Replace the appID part - if xpcTracer.appID == "" { - return headers, fmt.Errorf(traceAppIDNotSetErr) - } - // Create a new traceparent header - traceParent = fmt.Sprintf("%s-%s-%s-%s", - traceComponents[0], - traceComponents[1], - xpcTracer.appID, - traceComponents[3]) - } - headers = append(headers, TraceHeader{ - Key: "traceparent", - Value: traceParent, - }) - - traceState := r.Header.Get("tracestate") - if traceState == "" { - // Create tracestate header if it doesn't exist - traceState = fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID) - } else { - // Add current app to traceState - traceState = fmt.Sprintf("%s=%s,%s", xpcTracer.appName, xpcTracer.appID, traceState) - } - headers = append(headers, TraceHeader{ - Key: "tracestate", - Value: traceState, - }) - return headers, nil -} - -func tryHomegrownTpTs(r *http.Request, xpcTrace *XpcTrace) { - // if homegrownTracePropagation is true, we modify existing traceparent/tracestate and forward them - // but we will not generate them if they don't exist - // if homegrownTraceGeneration is true, and there is no tp/ts, we generate it - - if !xpcTracer.homegrownTracePropagation { - return - } - // ctx := r.Context() - log.Debug("Tracing: using homegrown traceparent propagation/generation") - if traceHeaders, err := GetTraceHeaders(r, xpcTracer.homegrownTraceGeneration); err == nil { - // TODO: Change homegrown propagation too to a per API basis instead of per app basis - // Reverse engineer otel basically - for _, h := range traceHeaders { - log.Debugf("Tracing: %s set to %s", h.Key, h.Value) - if h.Key == common.HeaderTraceparent { - xpcTrace.OutTraceparent = h.Value - } - if h.Key == common.HeaderTracestate { - xpcTrace.OutTracestate = h.Value - } - } - } else { - log.Errorf("Error in homegrown traceparent handling %+v", err) - } -} - -/* - // Benchmarking expt to compare mutex based random vs one source per goroutine - var ( - // This is for a benchmarking expt - global bool - globalLock sync.Mutex - globalRandGen = rand.NewSource(time.Now().UnixNano()) - - // Sample Benchmarking Results - // BenchmarkSharedRandGen-16 352419 3052 ns/op - // BenchmarkIndependentRandGen-16 670383 2755 ns/op - ) - - func setGlobal(g bool) { - global = g - } - - func genInt63(randGen rand.Source) int64 { - if global { - globalLock.Lock() - i := globalRandGen.Int63() - globalLock.Unlock() - return i - } - return randGen.Int63() - } -*/ - -/* - traceparentParentID string - tracestateVendorID string - otelEnabled bool - traceparentParentID: traceparentParentID, - tracestateVendorID: tracestateVendorID, -func (s *WebconfigServer) TraceparentParentID() string { - return s.traceparentParentID -} - -func (s *WebconfigServer) SetTraceparentParentID(x string) { - s.traceparentParentID = x -} - -func (s *WebconfigServer) TracestateVendorID() string { - return s.tracestateVendorID -} - -func (s *WebconfigServer) SetTracestateVendorID(x string) { - s.tracestateVendorID = x -} -*/ diff --git a/tracing/homegrown_test.go b/tracing/homegrown_test.go deleted file mode 100644 index 6e36c36..0000000 --- a/tracing/homegrown_test.go +++ /dev/null @@ -1,231 +0,0 @@ -/** - * @license - * Copyright Comcast. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @author RV - -package tracing - -import ( - "bytes" - "fmt" - "net/http" - "strings" - // "sync" - "testing" - - "github.com/stretchr/testify/require" -) - -const ( - testTraceID = "0123456789abcdef0123456789abcdef" - testCallerID = "fedcba9876543210" - testTraceParent = traceVersion + "-" + testTraceID + "-" + testCallerID + "-" + traceFlags - - testCallerName = "test" - testTraceState = testCallerName + "=" + testCallerID - - testMyName = "whatever" - testMyID = "0123456789abcdef" -) - -func TestAppIDNotSet(t *testing.T) { - // Mimic appName/appID not set by explicitly setting them to empty strs - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = "" - xpcTracer.appName = "" - - req := makeReq(t) - req.Header.Add("traceparent", testTraceParent) - _, err := GetTraceHeaders(req, false) - require.Equal(t, traceAppIDNotSetErr, err.Error()) - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestNoGenerate(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - req := makeReq(t) - headers, _ := GetTraceHeaders(req, false) - require.Equal(t, 0, len(headers)) - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestOnlyParentHeader(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - req := makeReq(t) - req.Header.Add("traceparent", testTraceParent) - - headers, _ := GetTraceHeaders(req, false) - require.Equal(t, 2, len(headers)) - validateTraceParentHeader(t, headers) - for _, h := range headers { - if h.Key == "tracestate" { - require.Equal(t, fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID), h.Value) - } - } - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestOnlyStateHeader(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - req := makeReq(t) - req.Header.Add("tracestate", testTraceState) - headers, _ := GetTraceHeaders(req, false) - require.Equal(t, 0, len(headers)) - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestParentPlusStateHeaders(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - req := makeReq(t) - req.Header.Add("traceparent", testTraceParent) - req.Header.Add("tracestate", testTraceState) - headers, _ := GetTraceHeaders(req, false) - require.Equal(t, 2, len(headers)) - validateTraceParentHeader(t, headers) - for _, h := range headers { - if h.Key == "tracestate" { - require.Equal(t, fmt.Sprintf("%s=%s,%s", xpcTracer.appName, xpcTracer.appID,testTraceState), h.Value) - } - } - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestTraceHeaderGeneration(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - testTraceHeaderGeneration(t) - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func validateTraceParentHeader(t *testing.T, headers []TraceHeader) { - for _, h := range headers { - if h.Key == "traceparent" { - traceComponents := strings.Split(h.Value, "-") - require.Equal(t, 4, len(traceComponents), 4) - require.Equal(t, traceVersion, traceComponents[0]) - require.Equal(t, testTraceID, traceComponents[1]) - require.Equal(t, xpcTracer.appID, traceComponents[2]) - require.Equal(t, traceFlags, traceComponents[3]) - } - } -} - -func makeReq(t *testing.T) *http.Request { - url := "/testurl" - var body []byte - req, err := http.NewRequest("GET", url, bytes.NewReader(body)) - if err != nil { - t.Fatal(err) - } - return req -} - -func testTraceHeaderGeneration(t *testing.T) { - req := makeReq(t) - headers, _ := GetTraceHeaders(req, true) - require.Equal(t, 2, len(headers)) - for _, h := range headers { - if h.Key == "traceparent" { - traceComponents := strings.Split(h.Value, "-") - require.Equal(t, 4, len(traceComponents)) - require.Equal(t, traceVersion, traceComponents[0]) - // We don't know the ID here, we can only check it is 32 digits long - require.Equal(t, 32, len(traceComponents[1])) - require.Equal(t, xpcTracer.appID, traceComponents[2]) - require.Equal(t, traceFlags, traceComponents[3]) - } - if h.Key == "tracestate" { - require.Equal(t, fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID), h.Value) - } - } -} - -/* - // These tests/benchmarks are to test mutex based random generation - // We are not going to use mutex as of now, as using one source - // per goroutine approach is faster - func TestTraceHeaderGenerationGlobal(t *testing.T) { - setGlobal(true) - testTraceHeaderGeneration(t) - setGlobal(false) - } - - func BenchmarkSharedRandGen(b *testing.B) { - setGlobal(true) - url := "/testurl" - var body []byte - req, _ := http.NewRequest("GET", url, bytes.NewReader(body)) - var wg sync.WaitGroup - for n := 0; n < b.N; n++ { - wg.Add(1) - go func() { - defer wg.Done() - GetTraceHeaders(req, true) - }() - } - wg.Wait() - setGlobal(false) - } - - func BenchmarkIndependentRandGen(b *testing.B) { - url := "/testurl" - var body []byte - req, _ := http.NewRequest("GET", url, bytes.NewReader(body)) - var wg sync.WaitGroup - - for n := 0; n < b.N; n++ { - wg.Add(1) - go func() { - defer wg.Done() - GetTraceHeaders(req, true) - }() - } - wg.Wait() - } -*/ diff --git a/tracing/span.go b/tracing/span.go index 137ac8b..15002b6 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -42,8 +42,7 @@ type XpcTrace struct { // Order of priority; use the value extracted from otel span; // if no otel span as well, use the value in req headers (Note: this will create islands as both // the app and its children will have the same tracestate - // If homegrown span modification is enabled in config, use it as a last resort. Otherwise, nothing - // will be passed to the child http calls, creating islands + // Otherwise, nothing will be passed to the child http calls, creating islands // If any source is found, then it will be propagated to all child http calls // TODO; also add this to Kafka headers, SNS message attributes otelTraceparent string @@ -66,7 +65,6 @@ type XpcTrace struct { } // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs -// It can also generate traceparent/state using homegrown algorithms, but avoid this as a rule func NewXpcTrace(r *http.Request) * XpcTrace { var xpcTrace XpcTrace xpcTrace.ReqMoracideTags = make(map[string]string) @@ -77,13 +75,6 @@ func NewXpcTrace(r *http.Request) * XpcTrace { otelExtractParamsFromSpan(r.Context(), &xpcTrace) } - if xpcTrace.OutTraceparent == "" { - // No traceparent in otel spans i.e. no otel spans - // No traceparent in incoming req either - // Note: If otel root span, ts can be empty, but tp can be set, so cannot check for ts being empty - tryHomegrownTpTs(r, &xpcTrace) - } - return &xpcTrace } diff --git a/tracing/tracer.go b/tracing/tracer.go index 80fdcc1..d447ee2 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -37,21 +37,13 @@ type XpcTracer struct { otelTracerProvider oteltrace.TracerProvider otelPropagator otelpropagation.TextMapPropagator otelTracer oteltrace.Tracer - - // internal vars for homegrown trace generation/modification - appID string // used in the homegrown traceparent,tracestate header generation - homegrownTracePropagation bool // Use homegrown algorithm for traceparent, tracestate modification - homegrownTraceGeneration bool // Generate traceparent etc. using homegrown algorithm if not present in incoming req } var xpcTracer XpcTracer // global tracer func NewXpcTracer(conf *configuration.Config) *XpcTracer { initAppData(conf) - initHomegrownTracing(conf) - otelInit(conf) - xpcTracer.MoracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", defaultMoracideTagPrefix) return &xpcTracer } @@ -87,13 +79,6 @@ func initAppData(conf *configuration.Config) { xpcTracer.rgn = os.Getenv("SITE_REGION") } -func initHomegrownTracing(conf *configuration.Config) { - // TODO: Marshal these tracing params from config, ok to temporarily read each separately - xpcTracer.appID = conf.GetString("webconfig.tracing.homegrown_algorithm.app_id", defaultAppID) - xpcTracer.homegrownTracePropagation = conf.GetBoolean("webconfig.tracing.homegrown_algorithm.xpc_trace_propagation") - xpcTracer.homegrownTraceGeneration = conf.GetBoolean("webconfig.tracing.homegrown_algorithm.xpc_trace_generation") -} - func GetServiceName() string { return xpcTracer.appName } From 4905893a4b82022c6b8dc21837e57683225d9a44 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Thu, 2 Jan 2025 14:44:31 -0800 Subject: [PATCH 142/215] case diffs in env var e.g. use site_color instead of SITE_COLOR --- tracing/tracer.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tracing/tracer.go b/tracing/tracer.go index d447ee2..ed48374 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -5,6 +5,7 @@ import ( "os" "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" oteltrace "go.opentelemetry.io/otel/trace" otelpropagation "go.opentelemetry.io/otel/propagation" @@ -70,13 +71,17 @@ func initAppData(conf *configuration.Config) { // Env vars xpcTracer.appEnv = "dev" - siteColor := os.Getenv("SITE_COLOR") + siteColor := os.Getenv("site_color") if strings.EqualFold(siteColor, "yellow") { xpcTracer.appEnv = "staging" } else if strings.EqualFold(siteColor, "green") { xpcTracer.appEnv = "prod" } - xpcTracer.rgn = os.Getenv("SITE_REGION") + xpcTracer.rgn = os.Getenv("site_region") + if xpcTracer.rgn == "" { + xpcTracer.rgn = os.Getenv("site_region_name") + } + log.Debugf("site_color = %f, env = %s, rgn = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) } func GetServiceName() string { From a50cf7f1337f87bcc514a14dab851aa1e141881d Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Fri, 3 Jan 2025 12:44:51 -0800 Subject: [PATCH 143/215] Typo --- tracing/tracer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracing/tracer.go b/tracing/tracer.go index ed48374..8e5888a 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -81,7 +81,7 @@ func initAppData(conf *configuration.Config) { if xpcTracer.rgn == "" { xpcTracer.rgn = os.Getenv("site_region_name") } - log.Debugf("site_color = %f, env = %s, rgn = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) + log.Debugf("site_color = %s, env = %s, rgn = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) } func GetServiceName() string { From dbdee274264775ad8756dae2bf207a891717843a Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Mon, 6 Jan 2025 18:14:08 -0800 Subject: [PATCH 144/215] Standardize logs - use "message" instead of "msg" --- config/sample_webconfig.conf | 5 ----- main.go | 1 + tracing/tracer.go | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index e5e93f8..02def24 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -6,11 +6,6 @@ webconfig { panic_exit_enabled = false tracing { moracide_tag_prefix = "X-Cl-Experiment" - homegrown_algorithm { - app_id = "0000000000000001" - xpc_trace_propagation = false - xpc_trace_generation = false - } opentelemetry { enabled = false endpoint = "127.0.0.1:4318" diff --git a/main.go b/main.go index 284012c..a6e5f80 100644 --- a/main.go +++ b/main.go @@ -81,6 +81,7 @@ func main() { TimestampFormat: common.LoggingTimeFormat, FieldMap: log.FieldMap{ log.FieldKeyTime: "timestamp", + log.FieldKeyMsg: "message", }, }) diff --git a/tracing/tracer.go b/tracing/tracer.go index 8e5888a..2f78c3f 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -81,7 +81,7 @@ func initAppData(conf *configuration.Config) { if xpcTracer.rgn == "" { xpcTracer.rgn = os.Getenv("site_region_name") } - log.Debugf("site_color = %s, env = %s, rgn = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) + log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) } func GetServiceName() string { From b6bdcf2387e873d1ef26bacbc86fa4005c9e4de8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 8 Jan 2025 14:20:01 -0800 Subject: [PATCH 145/215] fix a bug that new spanid are not included in the traceparent of outgoing headers --- go.mod | 3 --- http/http_client.go | 4 ++-- http/response.go | 6 ++--- tracing/ctx.go | 17 ++++++++++++++ tracing/otel.go | 25 ++++++++++++++++---- tracing/span.go | 56 +++++++++++++++++++++++++++++---------------- tracing/tracer.go | 23 ++++++++++++++++--- 7 files changed, 99 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index e085f6d..95108f9 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/sirupsen/logrus v1.9.0 - github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 @@ -57,7 +56,6 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -75,5 +73,4 @@ require ( google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/http/http_client.go b/http/http_client.go index 5ec0e72..e9397b3 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -154,12 +154,12 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h header.Set("X-Webpa-Transaction-Id", transactionId) } + c.addMoracideTags(header, auditFields) req.Header = header.Clone() if len(c.userAgent) > 0 { req.Header.Set(common.HeaderUserAgent, c.userAgent) } - c.addMoracideTags(header, auditFields) logHeader := header.Clone() auth := logHeader.Get("Authorization") if len(auth) > 0 { @@ -428,7 +428,7 @@ func (c *HttpClient) StatusHandler(status int) StatusHandlerFunc { // addMoracideTags - if ctx has a moracide tag as a header, add it to the headers // Also add traceparent, tracestate headers func (c *HttpClient) addMoracideTags(header http.Header, fields log.Fields) { - moracideTagPrefix := strings.ToLower("req_"+tracing.GetMoracideTagPrefix()) + moracideTagPrefix := strings.ToLower("req_" + tracing.GetMoracideTagPrefix()) for key, val := range fields { if key == "out_traceparent" { header.Set("traceparent", val.(string)) diff --git a/http/response.go b/http/response.go index 5fbd1bc..6c229be 100644 --- a/http/response.go +++ b/http/response.go @@ -22,10 +22,10 @@ import ( "fmt" "net/http" "strings" - log "github.com/sirupsen/logrus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/tracing" + log "github.com/sirupsen/logrus" ) const ( @@ -176,8 +176,8 @@ func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { return } - reqMoracideTagPrefix := strings.ToLower("req_"+tracing.GetMoracideTagPrefix()) - respMoracideTagPrefix := strings.ToLower("resp_"+tracing.GetMoracideTagPrefix()) + reqMoracideTagPrefix := strings.ToLower("req_" + tracing.GetMoracideTagPrefix()) + respMoracideTagPrefix := strings.ToLower("resp_" + tracing.GetMoracideTagPrefix()) fields := xw.Audit() moracideTags := make(map[string]string) for key, val := range fields { diff --git a/tracing/ctx.go b/tracing/ctx.go index 204e084..4a490fb 100644 --- a/tracing/ctx.go +++ b/tracing/ctx.go @@ -1,3 +1,20 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package tracing import "context" diff --git a/tracing/otel.go b/tracing/otel.go index 22e4728..66dfb69 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -1,3 +1,20 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package tracing import ( @@ -7,18 +24,18 @@ import ( "strings" "time" - "github.com/gorilla/mux" "github.com/go-akka/configuration" + "github.com/gorilla/mux" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/propagation" oteltrace "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" @@ -29,9 +46,9 @@ type providerConstructor func() (oteltrace.TracerProvider, error) var ( providerBuilders = map[string]providerConstructor{ - "http": otelHttpTraceProvider, + "http": otelHttpTraceProvider, "stdout": otelStdoutTraceProvider, - "noop": otelNoopTraceProvider, + "noop": otelNoopTraceProvider, } ) diff --git a/tracing/span.go b/tracing/span.go index 15002b6..c79c0e3 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -1,15 +1,31 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package tracing import ( "net/http" "strings" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" - oteltrace "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/attribute" - - "github.com/rdkcentral/webconfig/common" + oteltrace "go.opentelemetry.io/otel/trace" ) // SpanMiddleware is a middleware that creates a new span for each incoming request. @@ -34,7 +50,7 @@ type XpcTrace struct { // e.g. X-Cl-Experiment-1, X-Cl-Experiment-xapproxy, X-Cl-Experiement-webconfig-25.1.1.1... // For every key found in either req or resp, an explicit value of true/false will be set as an otel attribute // or an otel span attribute - ReqMoracideTags map[string]string // These are request headers prefixed with MoracideTagPrefix + ReqMoracideTags map[string]string // These are request headers prefixed with MoracideTagPrefix // The response moracide tags are stored in xw.audit // traceparent, tracestate can be set as req headers, may be extracted from otel spans @@ -45,27 +61,27 @@ type XpcTrace struct { // Otherwise, nothing will be passed to the child http calls, creating islands // If any source is found, then it will be propagated to all child http calls // TODO; also add this to Kafka headers, SNS message attributes - otelTraceparent string - otelTracestate string - ReqTraceparent string - ReqTracestate string - OutTraceparent string - OutTracestate string + otelTraceparent string + otelTracestate string + ReqTraceparent string + ReqTracestate string + OutTraceparent string + OutTracestate string // At the end of API flow, add the status code to OtelSpan; add the Moracide tags to the spans - otelSpan oteltrace.Span + otelSpan oteltrace.Span // These are not useful as of now, just set them for the sake of completion and future - AuditID string - MoneyTrace string - ReqUserAgent string - OutUserAgent string + AuditID string + MoneyTrace string + ReqUserAgent string + OutUserAgent string - TraceID string // use the value in outTraceparent, otherwise MoneyTrace + TraceID string // use the value in outTraceparent, otherwise MoneyTrace } // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs -func NewXpcTrace(r *http.Request) * XpcTrace { +func NewXpcTrace(r *http.Request) *XpcTrace { var xpcTrace XpcTrace xpcTrace.ReqMoracideTags = make(map[string]string) @@ -109,8 +125,8 @@ func SetSpanMoracideTags(fields log.Fields) { xpcTrace = tmp.(*XpcTrace) } moracideTags := make(map[string]string) - reqMoracideTagPrefix := strings.ToLower("req_"+xpcTracer.MoracideTagPrefix) - respMoracideTagPrefix := strings.ToLower("resp_"+xpcTracer.MoracideTagPrefix) + reqMoracideTagPrefix := strings.ToLower("req_" + xpcTracer.MoracideTagPrefix) + respMoracideTagPrefix := strings.ToLower("resp_" + xpcTracer.MoracideTagPrefix) for key, val := range fields { if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { @@ -149,7 +165,7 @@ func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace) { // We will not move this to tracing module // xpcTrace.AuditID = r.Header.Get(AuditIDHeader) // if xpcTrace.AuditID == "" { - // xpcTrace.AuditID = util.GetAuditId() + // xpcTrace.AuditID = util.GetAuditId() // } // Moneytrace is Comcast's original solution for tracing diff --git a/tracing/tracer.go b/tracing/tracer.go index 2f78c3f..8f3be02 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -1,18 +1,35 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package tracing import ( - "strings" "os" + "strings" "github.com/go-akka/configuration" log "github.com/sirupsen/logrus" - oteltrace "go.opentelemetry.io/otel/trace" otelpropagation "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" ) const ( - AuditIDHeader = "X-Auditid" + AuditIDHeader = "X-Auditid" UserAgentHeader = "User-Agent" defaultMoracideTagPrefix = "X-Cl-Experiment" From e9e969a40a76c03e5267b27fd8921693edb10798 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 13 Jan 2025 11:40:33 -0800 Subject: [PATCH 146/215] Make external url templates configurable --- config/sample_webconfig.conf | 36 ++++++++++++++++------------- http/http_client.go | 2 +- http/mqtt_connector.go | 20 ++++++++++++---- http/upstream_connector.go | 19 +++++++-------- http/webpa_connector.go | 45 ++++++++++++++++++------------------ http/xconf_connector.go | 20 ++++++++++++---- 6 files changed, 82 insertions(+), 60 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 02def24..36c8843 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -5,17 +5,17 @@ webconfig { panic_exit_enabled = false tracing { - moracide_tag_prefix = "X-Cl-Experiment" - opentelemetry { - enabled = false - endpoint = "127.0.0.1:4318" - operation_name = "http.request" - // Allowed values: "noop", "stdout", "http" - // "noop" will generate no trace - // "stdout" will use stdoutTracer and output spans to stdout - // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector - provider = "noop" - } + moracide_tag_prefix = "X-Cl-Experiment" + otel { + enabled = false + endpoint = "127.0.0.1:4318" + operation_name = "http.request" + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop" + } } // build info @@ -49,10 +49,11 @@ webconfig { read_timeout_in_secs = 142 max_idle_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "https://api.webpa.comcast.net" + host = "http://localhost:12345" async_poke_enabled = false async_poke_concurrent_calls = 100 api_version = "v2" + url_template = "%s/%s/%s" } xconf { @@ -63,7 +64,8 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "http://qa2.xconfds.coast.xcal.tv:8080" + host = "http://localhost:12346" + url_template = "%s/%s" } mqtt { @@ -74,7 +76,8 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "https://hcbroker.staging.us-west-2.plume.comcast.net" + host = "http://localhost:12347" + url_template = "%s/%s" } upstream { @@ -86,8 +89,9 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "http://localhost:9009" - url_template = "/api/v1/device/%v/upstream" + host = "http://localhost:12348" + url_template = "%s/%s" + profile_url_template = "%s/%s/%s" } http_client { diff --git a/http/http_client.go b/http/http_client.go index e9397b3..db8ca45 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -110,7 +110,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } func (c *HttpClient) Do(ctx context.Context, method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { - fields := common.FilterLogFields(auditFields) + fields := common.FilterLogFields(auditFields, "status") var respMoracideTagsFound bool defer func(found *bool) { diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 064f2e5..034c3bb 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -31,25 +31,27 @@ import ( ) const ( - mqttHostDefault = "https://hcbroker.staging.us-west-2.plume.comcast.net" - mqttUrlTemplate = "%s/v2/mqtt/pub/x/to/%s/webconfig" + defaultMqttHost = "http://localhost:12347" + defaultMqttUrlTemplate = "%s/%s" ) type MqttConnector struct { *HttpClient host string serviceName string + urlTemplate string } func NewMqttConnector(conf *configuration.Config, tlsConfig *tls.Config) *MqttConnector { serviceName := "mqtt" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, mqttHostDefault) + host := conf.GetString("webconfig.mqtt.host", defaultMqttHost) + urlTemplate := conf.GetString("webconfig.mqtt.url_template", defaultMqttUrlTemplate) return &MqttConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, + urlTemplate: urlTemplate, } } @@ -61,12 +63,20 @@ func (c *MqttConnector) SetMqttHost(host string) { c.host = host } +func (c *MqttConnector) MqttUrlTemplate() string { + return c.urlTemplate +} + +func (c *MqttConnector) SetMqttUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *MqttConnector) ServiceName() string { return c.serviceName } func (c *MqttConnector) PostMqtt(ctx context.Context, cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { - url := fmt.Sprintf(mqttUrlTemplate, c.MqttHost(), cpeMac) + url := fmt.Sprintf(c.MqttUrlTemplate(), c.MqttHost(), cpeMac) var traceId, xmTraceId, outTraceparent, outTracestate string if itf, ok := fields["xmoney_trace_id"]; ok { diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 303f2b6..2a3520a 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -30,9 +30,9 @@ import ( ) const ( - upstreamHostDefault = "http://localhost:1234" - defaultUpstreamUrlTemplate = "/api/v1/device/%v/upstream" - defaultProfileUrlTemplate = "/api/v1/device/%v/profile?%v" + defaultUpstreamHost = "http://localhost:12348" + defaultUpstreamUrlTemplate = "%s/%s" + defaultProfileUrlTemplate = "%s/%s/%s" ) type UpstreamConnector struct { @@ -45,12 +45,9 @@ type UpstreamConnector struct { func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *UpstreamConnector { serviceName := "upstream" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, upstreamHostDefault) - confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) - upstreamUrlTemplate := conf.GetString(confKey, defaultUpstreamUrlTemplate) - confKey = fmt.Sprintf("webconfig.%v.profile_url_template", serviceName) - profileUrlTemplate := conf.GetString(confKey, defaultProfileUrlTemplate) + host := conf.GetString("webconfig.upstream.host", defaultUpstreamHost) + upstreamUrlTemplate := conf.GetString("webconfig.upstream.url_template", defaultUpstreamUrlTemplate) + profileUrlTemplate := conf.GetString("webconfig.upstream.profile_url_template", defaultProfileUrlTemplate) return &UpstreamConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), @@ -74,7 +71,7 @@ func (c *UpstreamConnector) ServiceName() string { } func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { - url := c.UpstreamHost() + fmt.Sprintf(c.upstreamUrlTemplate, mac) + url := fmt.Sprintf(c.upstreamUrlTemplate, c.UpstreamHost(), mac) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) @@ -98,7 +95,7 @@ func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header } func (c *UpstreamConnector) GetUpstreamProfiles(ctx context.Context, mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { - url := c.UpstreamHost() + fmt.Sprintf(c.profileUrlTemplate, mac, queryParams) + url := fmt.Sprintf(c.profileUrlTemplate, c.UpstreamHost(), mac, queryParams) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) diff --git a/http/webpa_connector.go b/http/webpa_connector.go index f184e1f..014f291 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -33,14 +33,14 @@ import ( ) const ( - defaultWebpaHost = "https://api.webpa.comcast.net:8090" + defaultWebpaHost = "http://localhost:12345" defaultApiVersion = "v2" webpaServiceName = "webpa" asyncWebpaServiceName = "asyncwebpa" - webpaUrlTemplate = "%s/api/%s/device/mac:%s/config" - webpaError404 = `{"code": 521, "message": "Device not found in webpa"}` - webpaError520 = `{"code": 520, "message": "Error unsupported namespace"}` + defaultWebpaUrlTemplate = "%s/%s/%s" + webpaError404 = `{"code": 521, "message": "Device not found in webpa"}` + webpaError520 = `{"code": 520, "message": "Error unsupported namespace"}` // a new error code to indicate it is webpa 520 // but it is caused by some temporary conditions, @@ -66,6 +66,7 @@ type WebpaConnector struct { syncClient *HttpClient asyncClient *HttpClient host string + urlTemplate string queue chan struct{} retries int retryInMsecs int @@ -112,33 +113,24 @@ func asyncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { } func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *WebpaConnector { - confKey := fmt.Sprintf("webconfig.%v.host", webpaServiceName) - host := conf.GetString(confKey, defaultWebpaHost) - - confKey = fmt.Sprintf("webconfig.%v.async_poke_enabled", webpaServiceName) - asyncPokeEnabled := conf.GetBoolean(confKey, false) - - confKey = fmt.Sprintf("webconfig.%v.async_poke_concurrent_calls", webpaServiceName) - concurrentCalls := int(conf.GetInt32(confKey, 0)) + host := conf.GetString("webconfig.webpa.host", defaultWebpaHost) + asyncPokeEnabled := conf.GetBoolean("webconfig.webpa.async_poke_enabled", false) + concurrentCalls := int(conf.GetInt32("webconfig.webpa.async_poke_concurrent_calls", 0)) var queue chan struct{} if concurrentCalls > 0 { queue = make(chan struct{}, concurrentCalls) } - confKey = fmt.Sprintf("webconfig.%v.retries", webpaServiceName) - retries := int(conf.GetInt32(confKey, defaultRetries)) - - confKey = fmt.Sprintf("webconfig.%v.retry_in_msecs", webpaServiceName) - retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) + retries := int(conf.GetInt32("webconfig.webpa.retries", defaultRetries)) + retryInMsecs := int(conf.GetInt32("webconfig.webpa.retry_in_msecs", defaultRetriesInMsecs)) + apiVersion := conf.GetString("webconfig.webpa.api_version", defaultApiVersion) + urlTemplate := conf.GetString("webconfig.webpa.url_template", defaultWebpaUrlTemplate) syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig) syncClient.SetStatusHandler(520, syncHandle520) asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig) asyncClient.SetStatusHandler(520, asyncHandle520) - confKey = fmt.Sprintf("webconfig.%v.api_version", webpaServiceName) - apiVersion := conf.GetString(confKey, defaultApiVersion) - connector := WebpaConnector{ syncClient: syncClient, asyncClient: asyncClient, @@ -148,6 +140,7 @@ func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *Webpa retryInMsecs: retryInMsecs, asyncPokeEnabled: asyncPokeEnabled, apiVersion: apiVersion, + urlTemplate: urlTemplate, } return &connector @@ -161,6 +154,14 @@ func (c *WebpaConnector) SetWebpaHost(host string) { c.host = host } +func (c *WebpaConnector) WebpaUrlTemplate() string { + return c.urlTemplate +} + +func (c *WebpaConnector) SetWebpaUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *WebpaConnector) ApiVersion() string { return c.apiVersion } @@ -187,7 +188,7 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { } func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { - url := fmt.Sprintf(webpaUrlTemplate, c.WebpaHost(), c.ApiVersion(), cpeMac) + url := fmt.Sprintf(c.WebpaUrlTemplate(), c.WebpaHost(), c.ApiVersion(), cpeMac) var traceId, xmTraceId string if itf, ok := fields["trace_id"]; ok { @@ -241,7 +242,7 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac } func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - tfields := common.FilterLogFields(fields) + tfields := common.FilterLogFields(fields, "status") tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) diff --git a/http/xconf_connector.go b/http/xconf_connector.go index 7a39974..024589d 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -29,25 +29,27 @@ import ( ) const ( - xconfHostDefault = "http://qa2.xconfds.coast.xcal.tv:8080" - xconfUrlTemplate = "%s/loguploader/getTelemetryProfiles?%s" + defaultXconfHost = "http://localhost:12346" + defaultXconfUrlTemplate = "%s/%s" ) type XconfConnector struct { *HttpClient host string serviceName string + urlTemplate string } func NewXconfConnector(conf *configuration.Config, tlsConfig *tls.Config) *XconfConnector { serviceName := "xconf" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, xconfHostDefault) + host := conf.GetString("webconfig.xconf.host", defaultXconfHost) + urlTemplate := conf.GetString("webconfig.xconf.url_template", defaultXconfUrlTemplate) return &XconfConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, + urlTemplate: urlTemplate, } } @@ -59,12 +61,20 @@ func (c *XconfConnector) SetXconfHost(host string) { c.host = host } +func (c *XconfConnector) XconfUrlTemplate() string { + return c.urlTemplate +} + +func (c *XconfConnector) SetXconfUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *XconfConnector) ServiceName() string { return c.serviceName } func (c *XconfConnector) GetProfiles(ctx context.Context, urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { - url := fmt.Sprintf(xconfUrlTemplate, c.XconfHost(), urlSuffix) + url := fmt.Sprintf(c.XconfUrlTemplate(), c.XconfHost(), urlSuffix) rbytes, resHeader, err := c.DoWithRetries(ctx, "GET", url, nil, nil, fields, c.ServiceName()) if err != nil { return rbytes, resHeader, owcommon.NewError(err) From 51f619e2fcfcc1ef5febfce9ae7203fab8fb12e0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 14 Jan 2025 21:29:22 -0800 Subject: [PATCH 147/215] tracing codes rearranged --- common/log_fields.go | 3 ++ http/http_client.go | 74 +++++++++++++++++++++++----------------- http/response.go | 29 +++++++--------- http/response_writer.go | 18 +++++++++- http/router.go | 5 ++- http/webconfig_server.go | 71 +++++++++++++++++++++++--------------- tracing/otel.go | 50 +++++++++++---------------- tracing/span.go | 50 +++++++-------------------- tracing/tracer.go | 60 +++++++++++++++++--------------- util/print.go | 10 +++++- 10 files changed, 192 insertions(+), 178 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index c14a280..10fd63a 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -27,6 +27,9 @@ var ( unloggedFields = []string{ "moneytrace", "token", + "xpc_trace", + "req_moracide_tags", + "resp_moracide_tags", } coreFields = []string{ "app_name", diff --git a/http/http_client.go b/http/http_client.go index db8ca45..4bbbb6f 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -62,6 +62,7 @@ type HttpClient struct { retryInMsecs int statusHandlerFuncMap map[int]StatusHandlerFunc userAgent string + moracideTagPrefix string } func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config) *HttpClient { @@ -84,6 +85,8 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) userAgent := conf.GetString("webconfig.http_client.user_agent") + moracideTagPrefix := strings.ToLower(conf.GetString("webconfig.tracing.moracide_tag_prefix", tracing.DefaultMoracideTagPrefix)) + var transport http.RoundTripper = &http.Transport{ DialContext: (&net.Dialer{ Timeout: time.Duration(connectTimeout) * time.Second, @@ -106,6 +109,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs: retryInMsecs, statusHandlerFuncMap: map[int]StatusHandlerFunc{}, userAgent: userAgent, + moracideTagPrefix: moracideTagPrefix, } } @@ -228,26 +232,8 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h // i.e. timeout, 503 etc. wouldn't have a valid resp, but possible that // the err returned by http.Do actually includes an err returned by the backend // In which case, resp would be non-nil - moracideTagPrefix := strings.ToLower(tracing.GetMoracideTagPrefix()) if res != nil { - for headerKey, headerVals := range res.Header { - if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { - respMoracideTagsFound = true - log.Debugf("http_client: moracide tag %s = %s found in response", headerKey, headerVals[0]) - if len(headerVals) > 1 { - log.Debugf("Tracing: moracide tag key = %s, has multiple values = %+v", headerKey, headerVals) - } - val := "false" - for _, v := range headerVals { - if v == "true" { - val = v - break - } - } - auditFields["resp_"+headerKey] = headerVals[0] - log.Debugf("Tracing: found moracide tag in response key = %s, val = %s", headerKey, val) - } - } + respMoracideTagsFound = c.addMoracideTagsFromResponse(res.Header, auditFields) } var endMessage string if retry > 0 { @@ -428,21 +414,45 @@ func (c *HttpClient) StatusHandler(status int) StatusHandlerFunc { // addMoracideTags - if ctx has a moracide tag as a header, add it to the headers // Also add traceparent, tracestate headers func (c *HttpClient) addMoracideTags(header http.Header, fields log.Fields) { - moracideTagPrefix := strings.ToLower("req_" + tracing.GetMoracideTagPrefix()) - for key, val := range fields { - if key == "out_traceparent" { - header.Set("traceparent", val.(string)) - } - if key == "out_tracestate" { - header.Set("tracestate", val.(string)) + if itf, ok := fields["out_traceparent"]; ok { + if ss, ok := itf.(string); ok { + if len(ss) > 0 { + header.Set(common.HeaderTraceparent, ss) + } } - if len(key) < 5 { - // Should be of the form "req_x-cl-experiment" - continue + } + if itf, ok := fields["out_tracestate"]; ok { + if ss, ok := itf.(string); ok { + if len(ss) > 0 { + header.Set(common.HeaderTracestate, ss) + } } - if strings.HasPrefix(strings.ToLower(key), moracideTagPrefix) { - log.Debugf("Adding moracide tag %s = %s to outgoing req", key, val) - header.Set(key[4:], val.(string)) + } + + itf, ok := fields["req_moracide_tags"] + if !ok { + return + } + moracideTagMap := itf.(map[string]string) + + for key, val := range moracideTagMap { + log.WithFields(fields).Debugf("Adding moracide tag %s = %s to outgoing req", key, val) + header.Set(key, val) + } +} + +func (c *HttpClient) addMoracideTagsFromResponse(header http.Header, fields log.Fields) bool { + var respMoracideTagsFound bool + m := make(map[string]string) + for k := range header { + if strings.HasPrefix(strings.ToLower(k), c.moracideTagPrefix) { + respMoracideTagsFound = true + val := header.Get(k) + m[k] = val } } + if len(m) > 0 { + fields["resp_moracide_tags"] = m + } + return respMoracideTagsFound } diff --git a/http/response.go b/http/response.go index 6c229be..5b748c3 100644 --- a/http/response.go +++ b/http/response.go @@ -21,11 +21,8 @@ import ( "encoding/json" "fmt" "net/http" - "strings" "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/tracing" - log "github.com/sirupsen/logrus" ) const ( @@ -176,23 +173,21 @@ func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { return } - reqMoracideTagPrefix := strings.ToLower("req_" + tracing.GetMoracideTagPrefix()) - respMoracideTagPrefix := strings.ToLower("resp_" + tracing.GetMoracideTagPrefix()) - fields := xw.Audit() + reqMoracideTags := xw.ReqMoracideTags() + respMoracideTags := xw.RespMoracideTags() + moracideTags := make(map[string]string) - for key, val := range fields { - if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { - log.Debugf("Adding moracide tag from req %s = %s to response", key, val) - moracideTags[key[4:]] = val.(string) - } - if strings.HasPrefix(strings.ToLower(key), respMoracideTagPrefix) { - log.Debugf("Adding moracide tag from resp %s = %s to response", key, val) - realKey := key[5:] - if existingVal, ok := moracideTags[realKey]; !ok || (ok && existingVal != "true") { - moracideTags[realKey] = val.(string) - } + for key, val := range reqMoracideTags { + xw.LogDebug(nil, "request", fmt.Sprintf("Adding moracide tag from req %s = %s to response", key, val)) + moracideTags[key] = val + } + for key, val := range respMoracideTags { + if existingVal, ok := moracideTags[key]; !ok || (ok && existingVal != "true") { + xw.LogDebug(nil, "request", fmt.Sprintf("Adding moracide tag from resp %s = %s to response", key, val)) + moracideTags[key] = val } } + xw.LogDebug(nil, "request", fmt.Sprintf("moracideTags = %+v", moracideTags)) for key, val := range moracideTags { w.Header().Set(key, val) } diff --git a/http/response_writer.go b/http/response_writer.go index 9f0372c..ecc7e0b 100644 --- a/http/response_writer.go +++ b/http/response_writer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -165,3 +165,19 @@ func (w *XResponseWriter) SetPartnerId(partnerId string) { w.partnerId = partnerId w.audit["partner"] = partnerId } + +func (w *XResponseWriter) ReqMoracideTags() map[string]string { + itf, ok := w.audit["req_moracide_tags"] + if !ok { + return nil + } + return itf.(map[string]string) +} + +func (w *XResponseWriter) RespMoracideTags() map[string]string { + itf, ok := w.audit["resp_moracide_tags"] + if !ok { + return nil + } + return itf.(map[string]string) +} diff --git a/http/router.go b/http/router.go index 4969326..103c77a 100644 --- a/http/router.go +++ b/http/router.go @@ -19,7 +19,6 @@ package http import ( "github.com/gorilla/mux" - "github.com/rdkcentral/webconfig/tracing" ) func (s *WebconfigServer) AddBaseRoutes(testOnly bool, router *mux.Router) { @@ -96,9 +95,9 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.TestingMiddleware) } else { if s.ServerApiTokenAuthEnabled() { - sub2.Use(tracing.SpanMiddleware, s.ApiMiddleware) + sub2.Use(s.SpanMiddleware, s.ApiMiddleware) } else { - sub2.Use(tracing.SpanMiddleware, s.NoAuthMiddleware) + sub2.Use(s.SpanMiddleware, s.NoAuthMiddleware) } } sub2.HandleFunc("", s.PokeHandler).Methods("POST") diff --git a/http/webconfig_server.go b/http/webconfig_server.go index fbc7d0c..67c24bc 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -43,6 +43,7 @@ import ( "github.com/rdkcentral/webconfig/tracing" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" + sdktrace "go.opentelemetry.io/otel/sdk/trace" ) // TODO enum, probably no need @@ -330,7 +331,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } func (s *WebconfigServer) Stop() { - tracing.StopXpcTracer() + s.StopXpcTracer() } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -730,7 +731,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques token = elements[1] } - var xmTraceId, traceId string + var xmTraceId string // extract moneytrace from the header tracePart := strings.Split(r.Header.Get("X-Moneytrace"), ";")[0] @@ -740,12 +741,6 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } } - // extract traceparent from the header - traceparent := r.Header.Get(common.HeaderTraceparent) - if len(traceparent) == 55 { - traceId = traceparent[3:35] - } - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { @@ -753,27 +748,29 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } // traceparent handling for E2E tracing - xpcTrace := tracing.NewXpcTrace(r) + xpcTrace := tracing.NewXpcTrace(r, s.XpcTracer) + traceId := xpcTrace.TraceID + if len(traceId) == 0 { + traceId = xmTraceId + } headerMap := util.HeaderToMap(header) fields := log.Fields{ - "path": r.URL.String(), - "method": r.Method, - "audit_id": auditId, - "remote_ip": remoteIp, - "host_name": host, - "header": headerMap, - "logger": "request", - "trace_id": traceId, - "app_name": s.AppName(), - "traceparent": xpcTrace.ReqTraceparent, - "tracestate": xpcTrace.ReqTracestate, - "out_traceparent": xpcTrace.OutTraceparent, - "out_tracestate": xpcTrace.OutTracestate, - "xpc_trace": xpcTrace, - } - for key, val := range xpcTrace.ReqMoracideTags { - fields["req_"+key] = val + "path": r.URL.String(), + "method": r.Method, + "audit_id": auditId, + "remote_ip": remoteIp, + "host_name": host, + "header": headerMap, + "logger": "request", + "trace_id": traceId, + "app_name": s.AppName(), + "traceparent": xpcTrace.ReqTraceparent, + "tracestate": xpcTrace.ReqTracestate, + "out_traceparent": xpcTrace.OutTraceparent, + "out_tracestate": xpcTrace.OutTracestate, + "req_moracide_tags": xpcTrace.ReqMoracideTags, + "xpc_trace": xpcTrace, } userAgent := r.UserAgent() @@ -902,8 +899,8 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["duration"] = duration fields["logger"] = "request" - tracing.SetSpanStatusCode(fields) - tracing.SetSpanMoracideTags(fields) + tracing.SetSpanStatusCode(s.XpcTracer, fields) + tracing.SetSpanMoracideTags(s.XpcTracer, fields) var userAgent string if itf, ok := fields["user_agent"]; ok { @@ -1118,3 +1115,21 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { } } } + +func (s *WebconfigServer) StopXpcTracer() { + sdkTraceProvider, ok := s.XpcTracer.OtelTracerProvider().(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} + +func (s *WebconfigServer) SpanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.XpcTracer.OtelEnabled { + ctx, otelSpan := tracing.NewOtelSpan(r, s.XpcTracer) + r = r.WithContext(ctx) + defer tracing.EndOtelSpan(s.XpcTracer, otelSpan) + } + next.ServeHTTP(w, r) + }) +} diff --git a/tracing/otel.go b/tracing/otel.go index 66dfb69..c9dd75a 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -42,7 +42,7 @@ import ( log "github.com/sirupsen/logrus" ) -type providerConstructor func() (oteltrace.TracerProvider, error) +type providerConstructor func(*XpcTracer) (oteltrace.TracerProvider, error) var ( providerBuilders = map[string]providerConstructor{ @@ -59,7 +59,7 @@ const defaultOtelTracerProvider = "noop" // initOtel - initialize OpenTelemetry constructs // tracing instrumentation code. -func otelInit(conf *configuration.Config) { +func otelInit(conf *configuration.Config, xpcTracer *XpcTracer) { xpcTracer.OtelEnabled = conf.GetBoolean("webconfig.tracing.otel.enabled") if !xpcTracer.OtelEnabled { return @@ -82,7 +82,7 @@ func otelInit(conf *configuration.Config) { return } else { var err error - if xpcTracer.otelTracerProvider, err = providerBuilder(); err != nil { + if xpcTracer.otelTracerProvider, err = providerBuilder(xpcTracer); err != nil { log.Errorf("building otel provider for %s failed with %v", xpcTracer.otelProvider, err) return } @@ -99,11 +99,11 @@ func otelInit(conf *configuration.Config) { xpcTracer.otelTracer = otel.Tracer(xpcTracer.appName) } -func otelNoopTraceProvider() (oteltrace.TracerProvider, error) { +func otelNoopTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { return noop.NewTracerProvider(), nil } -func otelStdoutTraceProvider() (oteltrace.TracerProvider, error) { +func otelStdoutTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { option := stdouttrace.WithPrettyPrint() exporter, err := stdouttrace.New(option) if err != nil { @@ -124,7 +124,7 @@ func otelStdoutTraceProvider() (oteltrace.TracerProvider, error) { return tp, nil } -func otelHttpTraceProvider() (oteltrace.TracerProvider, error) { +func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { // Send traces over HTTP if xpcTracer.otelEndpoint == "" { return nil, fmt.Errorf("building http otel provider failure, no endpoint specified") @@ -149,23 +149,7 @@ func otelHttpTraceProvider() (oteltrace.TracerProvider, error) { ), nil } -func otelShutdown() { - sdkTraceProvider, ok := xpcTracer.otelTracerProvider.(*sdktrace.TracerProvider) - if ok && sdkTraceProvider != nil { - sdkTraceProvider.Shutdown(context.TODO()) - } -} - -// otelOpName should return "http.request" by default -func otelOpName() string { - opName := xpcTracer.otelOpName - if opName == "" { - opName = "http.request" - } - return opName -} - -func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { +func NewOtelSpan(r *http.Request, xpcTracer *XpcTracer) (context.Context, oteltrace.Span) { ctx := r.Context() var otelSpan oteltrace.Span if !xpcTracer.OtelEnabled { @@ -191,7 +175,7 @@ func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { custom Comcast attribute: X-Cl-Experiment: true/false additional: env, operation.name, http.url_details.path */ - ctx, otelSpan = xpcTracer.otelTracer.Start(ctx, otelOpName(), + ctx, otelSpan = xpcTracer.otelTracer.Start(ctx, xpcTracer.OtelOpName(), oteltrace.WithSpanKind(oteltrace.SpanKindServer), oteltrace.WithAttributes( attribute.String("env", xpcTracer.appEnv), @@ -199,11 +183,11 @@ func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { attribute.String("http.route", pathTemplate), attribute.String("http.url", r.URL.String()), attribute.String("http.url_details.path", r.URL.Path), - attribute.String("operation.name", otelOpName()), + attribute.String("operation.name", xpcTracer.OtelOpName()), ), ) - if xpcTracer.rgn != "" { - rgnAttr := attribute.String("region", xpcTracer.rgn) + if xpcTracer.region != "" { + rgnAttr := attribute.String("region", xpcTracer.region) otelSpan.SetAttributes(rgnAttr) } @@ -213,7 +197,7 @@ func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { log.Debugf("added span attribute key = http.route, value = %s", pathTemplate) log.Debugf("added span attribute key = http.url, value = %s", r.URL.String()) log.Debugf("added span attribute key = http.url_details.path, value = %s", r.URL.Path) - log.Debugf("added span attribute key = operation.name, value = %s", otelOpName()) + log.Debugf("added span attribute key = operation.name, value = %s", xpcTracer.OtelOpName()) carrier := propagation.MapCarrier{} otel.GetTextMapPropagator().Inject(ctx, carrier) @@ -239,20 +223,26 @@ func otelSetStatusCode(span oteltrace.Span, statusCode int) { } } -func otelEndSpan(span oteltrace.Span) { +func EndOtelSpan(xpcTracer *XpcTracer, span oteltrace.Span) { if !xpcTracer.OtelEnabled { return } span.End() } -func otelExtractParamsFromSpan(ctx context.Context, xpcTrace *XpcTrace) { +func otelExtractParamsFromSpan(ctx context.Context, xpcTracer *XpcTracer, xpcTrace *XpcTrace) { if !xpcTracer.OtelEnabled { return } if tmp := GetContext(ctx, "otel_span"); tmp != nil { if otelSpan, ok := tmp.(oteltrace.Span); ok { xpcTrace.otelSpan = otelSpan + if xpcTrace.otelSpan == nil { + return + } + + spanCtx := otelSpan.SpanContext() + xpcTrace.TraceID = spanCtx.TraceID().String() } } if xpcTrace.otelSpan == nil { diff --git a/tracing/span.go b/tracing/span.go index c79c0e3..ba7edc1 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -28,19 +28,6 @@ import ( oteltrace "go.opentelemetry.io/otel/trace" ) -// SpanMiddleware is a middleware that creates a new span for each incoming request. -func SpanMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // For Otel, create the span explicitly - if xpcTracer.OtelEnabled { - ctx, otelSpan := otelNewSpan(r) - r = r.WithContext(ctx) - defer otelEndSpan(otelSpan) - } - next.ServeHTTP(w, r) - }) -} - // XpcTrace is a carrier/baggage struct to extract data from spans, request headers for usage later // Store the trace in ctx for easy retrieval. // Ideal place to store it is ofc, xw @@ -81,20 +68,20 @@ type XpcTrace struct { } // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs -func NewXpcTrace(r *http.Request) *XpcTrace { +func NewXpcTrace(r *http.Request, xpcTracer *XpcTracer) *XpcTrace { var xpcTrace XpcTrace xpcTrace.ReqMoracideTags = make(map[string]string) - extractParamsFromReq(r, &xpcTrace) + extractParamsFromReq(r, xpcTracer, &xpcTrace) if xpcTracer.OtelEnabled { - otelExtractParamsFromSpan(r.Context(), &xpcTrace) + otelExtractParamsFromSpan(r.Context(), xpcTracer, &xpcTrace) } return &xpcTrace } -func SetSpanStatusCode(fields log.Fields) { +func SetSpanStatusCode(xpcTracer *XpcTracer, fields log.Fields) { var xpcTrace *XpcTrace if tmp, ok := fields["xpc_trace"]; ok { xpcTrace = tmp.(*XpcTrace) @@ -102,6 +89,7 @@ func SetSpanStatusCode(fields log.Fields) { if xpcTrace == nil { // Something went wrong, cannot instrument this span log.Error("instrumentation error, no trace info") + return } if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { if tmp, ok := fields["status"]; ok { @@ -111,7 +99,7 @@ func SetSpanStatusCode(fields log.Fields) { } } -func SetSpanMoracideTags(fields log.Fields) { +func SetSpanMoracideTags(xpcTracer *XpcTracer, fields log.Fields) { var xpcTrace *XpcTrace if tmp, ok := fields["xpc_trace"]; ok { xpcTrace = tmp.(*XpcTrace) @@ -119,14 +107,12 @@ func SetSpanMoracideTags(fields log.Fields) { if xpcTrace == nil { // Something went wrong, cannot instrument this span log.Error("instrumentation error, cannot set moracide tags, no trace info") + return } - if tmp, ok := fields["xpc_trace"]; ok { - xpcTrace = tmp.(*XpcTrace) - } moracideTags := make(map[string]string) - reqMoracideTagPrefix := strings.ToLower("req_" + xpcTracer.MoracideTagPrefix) - respMoracideTagPrefix := strings.ToLower("resp_" + xpcTracer.MoracideTagPrefix) + reqMoracideTagPrefix := strings.ToLower("req_" + xpcTracer.MoracideTagPrefix()) + respMoracideTagPrefix := strings.ToLower("resp_" + xpcTracer.MoracideTagPrefix()) for key, val := range fields { if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { @@ -144,7 +130,7 @@ func SetSpanMoracideTags(fields log.Fields) { if len(moracideTags) == 0 { // No moracide tags in request or any response // So set at least one span tag, x-cl-expt: false - moracideTags[xpcTracer.MoracideTagPrefix] = "false" + moracideTags[xpcTracer.MoracideTagPrefix()] = "false" } for key, val := range moracideTags { if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { @@ -154,30 +140,18 @@ func SetSpanMoracideTags(fields log.Fields) { } } -func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace) { +func extractParamsFromReq(r *http.Request, xpcTracer *XpcTracer, xpcTrace *XpcTrace) { xpcTrace.ReqTraceparent = r.Header.Get(common.HeaderTraceparent) xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent xpcTrace.OutTracestate = xpcTrace.ReqTracestate log.Debugf("Tracing: input traceparent : %s, tracestate : %s", xpcTrace.ReqTraceparent, xpcTrace.ReqTracestate) - // AuditID is used internally within xdp subsystem - // We will not move this to tracing module - // xpcTrace.AuditID = r.Header.Get(AuditIDHeader) - // if xpcTrace.AuditID == "" { - // xpcTrace.AuditID = util.GetAuditId() - // } - - // Moneytrace is Comcast's original solution for tracing - // Unused for now, decide whether we need to merge existing moneytrace code here - // Being replaced by traceparent/tracestate headers anyway - // xpcTrace.MoneyTrace = r.Header.Get(MoneyTraceHeader) - xpcTrace.ReqUserAgent = r.Header.Get(UserAgentHeader) // In future, -H 'X-Cl-Experiment-1', -H 'X-Cl-Experiment-oswebconfig'... OR 'X-Cl-Experiment-xapproxy_25.1.1.1' are all possible // So walk through all headers and collect any header that starts with this prefix - moracideTagPrefix := strings.ToLower(xpcTracer.MoracideTagPrefix) + moracideTagPrefix := strings.ToLower(xpcTracer.MoracideTagPrefix()) for headerKey, headerVals := range r.Header { if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { if len(headerVals) > 1 { diff --git a/tracing/tracer.go b/tracing/tracer.go index 8f3be02..fffe1f2 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -29,24 +29,23 @@ import ( ) const ( - AuditIDHeader = "X-Auditid" - UserAgentHeader = "User-Agent" - - defaultMoracideTagPrefix = "X-Cl-Experiment" + AuditIDHeader = "X-Auditid" + UserAgentHeader = "User-Agent" + DefaultMoracideTagPrefix = "X-Cl-Experiment" ) // XpcTracer is a wrapper around tracer setup type XpcTracer struct { OtelEnabled bool - MoracideTagPrefix string // Special request header for moracide expts e.g. canary deployments + moracideTagPrefix string // Special request header for moracide expts e.g. canary deployments // internal vars used by Otel appEnv string // set this to dev for red, staging for yellow and prod for green appName string appVersion string appSHA string // unused - rgn string // AWS Region e.g. us-west-2, unused, use it as a otel span attribute - siteColor string // red/yellow/green, unused, use it as a otel span attribute + region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute + siteColor string // red/yellow/green, unused, use it as a span attribute // internal otel vars otelEndpoint string @@ -57,26 +56,35 @@ type XpcTracer struct { otelTracer oteltrace.Tracer } -var xpcTracer XpcTracer // global tracer - func NewXpcTracer(conf *configuration.Config) *XpcTracer { - initAppData(conf) - otelInit(conf) - xpcTracer.MoracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", defaultMoracideTagPrefix) - return &xpcTracer + xpcTracer := new(XpcTracer) + initAppData(conf, xpcTracer) + otelInit(conf, xpcTracer) + xpcTracer.moracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", DefaultMoracideTagPrefix) + return xpcTracer } -// defer this func in the main of the app -func StopXpcTracer() { - otelShutdown() +func (t *XpcTracer) MoracideTagPrefix() string { + return t.moracideTagPrefix +} + +// otelOpName should return "http.request" by default +func (t *XpcTracer) OtelOpName() string { + if len(t.otelOpName) == 0 { + return "http.request" + } + return t.otelOpName } -// Global func to access the moracide tag -func GetMoracideTagPrefix() string { - return xpcTracer.MoracideTagPrefix +func (t *XpcTracer) OtelTracerProvider() oteltrace.TracerProvider { + return t.otelTracerProvider } -func initAppData(conf *configuration.Config) { +func (t *XpcTracer) AppName() string { + return t.appName +} + +func initAppData(conf *configuration.Config, xpcTracer *XpcTracer) { codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") xpcTracer.appName = codeGitCommit[0] if len(codeGitCommit) > 1 { @@ -94,13 +102,9 @@ func initAppData(conf *configuration.Config) { } else if strings.EqualFold(siteColor, "green") { xpcTracer.appEnv = "prod" } - xpcTracer.rgn = os.Getenv("site_region") - if xpcTracer.rgn == "" { - xpcTracer.rgn = os.Getenv("site_region_name") + xpcTracer.region = os.Getenv("site_region") + if xpcTracer.region == "" { + xpcTracer.region = os.Getenv("site_region_name") } - log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) -} - -func GetServiceName() string { - return xpcTracer.appName + log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) } diff --git a/util/print.go b/util/print.go index e8d55bd..afe7de4 100644 --- a/util/print.go +++ b/util/print.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -40,6 +40,14 @@ func PrettyJson(input interface{}) string { if bbytes, err := json.MarshalIndent(input, "", " "); err == nil { pretty = string(bbytes) } + case http.Header: + m := make(map[string]string) + for k := range ty { + m[k] = ty.Get(k) + } + if bbytes, err := json.MarshalIndent(m, "", " "); err == nil { + pretty = string(bbytes) + } } return pretty From 13e8d5c3ea959ba384c08838cb08fb5c401d73bc Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 15 Jan 2025 13:29:52 -0800 Subject: [PATCH 148/215] remove unused func args --- http/http_client.go | 7 +++---- http/mqtt_connector.go | 2 +- http/upstream_connector.go | 4 ++-- http/webpa_connector.go | 8 ++++---- http/xconf_connector.go | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index 4bbbb6f..71ff1f9 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -19,7 +19,6 @@ package http import ( "bytes" - "context" "crypto/tls" "encoding/base64" "encoding/json" @@ -113,7 +112,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } } -func (c *HttpClient) Do(ctx context.Context, method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { +func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields, "status") var respMoracideTagsFound bool @@ -374,7 +373,7 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h return rbytes, res.Header, false, nil } -func (c *HttpClient) DoWithRetries(ctx context.Context, method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { +func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { var respBytes []byte var respHeader http.Header var err error @@ -388,7 +387,7 @@ func (c *HttpClient) DoWithRetries(ctx context.Context, method string, url strin if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, cont, err = c.Do(ctx, method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 034c3bb..d658cd2 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -105,7 +105,7 @@ func (c *MqttConnector) PostMqtt(ctx context.Context, cpeMac string, bbytes []by header.Set(common.HeaderTraceparent, outTraceparent) header.Set(common.HeaderTracestate, outTracestate) - rbytes, _, err := c.DoWithRetries(ctx, "POST", url, header, bbytes, fields, c.ServiceName()) + rbytes, _, err := c.DoWithRetries("POST", url, header, bbytes, fields, c.ServiceName()) if err != nil { return rbytes, common.NewError(err) } diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 2a3520a..f5037d2 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -87,7 +87,7 @@ func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header } } - rbytes, header, err := c.DoWithRetries(ctx, "POST", url, header, bbytes, fields, c.ServiceName()) + rbytes, header, err := c.DoWithRetries("POST", url, header, bbytes, fields, c.ServiceName()) if err != nil { return rbytes, header, owcommon.NewError(err) } @@ -111,7 +111,7 @@ func (c *UpstreamConnector) GetUpstreamProfiles(ctx context.Context, mac, queryP } } - rbytes, header, err := c.DoWithRetries(ctx, "GET", url, header, nil, fields, c.ServiceName()) + rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) if err != nil { return rbytes, header, owcommon.NewError(err) } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 014f291..d6cd7cc 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -212,7 +212,7 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac header.Set("X-Webpa-Transaction-Id", transactionId) method := "PATCH" - _, _, cont, err := c.syncClient.Do(ctx, method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -230,7 +230,7 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac } } if cont { - _, _, err := c.syncClient.DoWithRetries(ctx, "PATCH", url, header, bbytes, fields, webpaServiceName) + _, _, err := c.syncClient.DoWithRetries("PATCH", url, header, bbytes, fields, webpaServiceName) if err != nil { return transactionId, common.NewError(err) } @@ -250,7 +250,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, cont, _ := c.asyncClient.Do(ctx, method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { msg := fmt.Sprintf("finished success after 1 retry") if i > 1 { @@ -278,7 +278,7 @@ func (c *WebpaConnector) SyncDoWithRetries(ctx context.Context, method string, u if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, cont, err = c.syncClient.Do(ctx, method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError diff --git a/http/xconf_connector.go b/http/xconf_connector.go index 024589d..5057a5e 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -75,7 +75,7 @@ func (c *XconfConnector) ServiceName() string { func (c *XconfConnector) GetProfiles(ctx context.Context, urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(c.XconfUrlTemplate(), c.XconfHost(), urlSuffix) - rbytes, resHeader, err := c.DoWithRetries(ctx, "GET", url, nil, nil, fields, c.ServiceName()) + rbytes, resHeader, err := c.DoWithRetries("GET", url, nil, nil, fields, c.ServiceName()) if err != nil { return rbytes, resHeader, owcommon.NewError(err) } From 087035a716ce648b8dcc41b901933f17cbda9ca5 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 15 Jan 2025 14:02:24 -0800 Subject: [PATCH 149/215] clean up unused contexts --- http/mqtt_connector.go | 3 +-- http/multipart.go | 4 ++-- http/poke_handler.go | 7 ++----- http/supplementary_handler.go | 6 ++---- http/upstream_connector.go | 5 ++--- http/upstream_connector_test.go | 4 +--- http/webconfig_server.go | 4 ++-- http/webpa_connector.go | 11 +++++------ http/xconf_connector.go | 3 +-- kafka/consumer.go | 2 +- 10 files changed, 19 insertions(+), 30 deletions(-) diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index d658cd2..66ac2a6 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -18,7 +18,6 @@ package http import ( - "context" "crypto/tls" "fmt" "net/http" @@ -75,7 +74,7 @@ func (c *MqttConnector) ServiceName() string { return c.serviceName } -func (c *MqttConnector) PostMqtt(ctx context.Context, cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { +func (c *MqttConnector) PostMqtt(cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { url := fmt.Sprintf(c.MqttUrlTemplate(), c.MqttHost(), cpeMac) var traceId, xmTraceId, outTraceparent, outTracestate string diff --git a/http/multipart.go b/http/multipart.go index a431552..91f55fc 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -287,7 +287,7 @@ func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader htt upstreamHeader.Set(common.HeaderUpstreamOldSchemaVersion, oldRootDocument.SchemaVersion) } - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(ctx, mac, upstreamHeader, respBytes, fields) + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, respBytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -410,7 +410,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, ctx context.Context, rHeader } // call /upstream to handle factory reset - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(ctx, mac, upstreamHeader, oldDocBytes, fields) + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { diff --git a/http/poke_handler.go b/http/poke_handler.go index 902441b..29e19f3 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -32,9 +32,6 @@ import ( ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { - // tracing propagation - ctx := r.Context() - // handler params := mux.Vars(r) mac := params["mac"] @@ -107,7 +104,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - _, err = s.PostMqtt(ctx, deviceId, mbytes, fields) + _, err = s.PostMqtt(deviceId, mbytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -160,7 +157,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(ctx, r.Header, mac, token, pokeStr, fields) + transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 401c4b7..20dc8fd 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -34,8 +34,6 @@ const ( ) func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - // ==== data integrity check ==== params := mux.Vars(r) mac, ok := params["mac"] @@ -82,7 +80,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - baseProfileBytes, resHeader, err := s.GetProfiles(ctx, urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) xconfNotFound := false if err != nil { var rherr common.RemoteHttpError @@ -108,7 +106,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r var profileBytes []byte if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { // Get profiles from the second source - extraProfileBytes, _, err := s.GetUpstreamProfiles(ctx, mac, queryParams, r.Header, fields) + extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) if err != nil { exitNow := true var rherr common.RemoteHttpError diff --git a/http/upstream_connector.go b/http/upstream_connector.go index f5037d2..cae0e4a 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -18,7 +18,6 @@ package http import ( - "context" "crypto/tls" "fmt" "net/http" @@ -70,7 +69,7 @@ func (c *UpstreamConnector) ServiceName() string { return c.serviceName } -func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { +func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(c.upstreamUrlTemplate, c.UpstreamHost(), mac) if itf, ok := fields["audit_id"]; ok { @@ -94,7 +93,7 @@ func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header return rbytes, header, nil } -func (c *UpstreamConnector) GetUpstreamProfiles(ctx context.Context, mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { +func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(c.profileUrlTemplate, c.UpstreamHost(), mac, queryParams) if itf, ok := fields["audit_id"]; ok { diff --git a/http/upstream_connector_test.go b/http/upstream_connector_test.go index 13fddbb..d09e1a7 100644 --- a/http/upstream_connector_test.go +++ b/http/upstream_connector_test.go @@ -18,7 +18,6 @@ package http import ( - "context" "net/http" "net/http/httptest" "testing" @@ -52,7 +51,6 @@ func TestUpstreamConnector(t *testing.T) { bbytes := []byte("hello world") var err error fields := log.Fields{} - ctx := context.Background() - _, _, err = server.PostUpstream(ctx, mac, header, bbytes, fields) + _, _, err = server.PostUpstream(mac, header, bbytes, fields) assert.NilError(t, err) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 67c24bc..2941f31 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -701,9 +701,9 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(ctx context.Context, rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) - transactionId, err := c.Patch(ctx, rHeader, cpeMac, token, []byte(body), fields) + transactionId, err := c.Patch(rHeader, cpeMac, token, []byte(body), fields) if err != nil { return "", common.NewError(err) } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index d6cd7cc..95b18fa 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -18,7 +18,6 @@ package http import ( - "context" "crypto/tls" "encoding/json" "errors" @@ -187,7 +186,7 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { c.asyncPokeEnabled = enabled } -func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { +func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { url := fmt.Sprintf(c.WebpaUrlTemplate(), c.WebpaHost(), c.ApiVersion(), cpeMac) var traceId, xmTraceId string @@ -219,9 +218,9 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac if rherr.StatusCode == 524 { if c.asyncPokeEnabled { c.queue <- struct{}{} - go c.AsyncDoWithRetries(ctx, method, url, header, bbytes, fields, asyncWebpaServiceName) + go c.AsyncDoWithRetries(method, url, header, bbytes, fields, asyncWebpaServiceName) } else { - _, err := c.SyncDoWithRetries(ctx, method, url, header, bbytes, fields, webpaServiceName) + _, err := c.SyncDoWithRetries(method, url, header, bbytes, fields, webpaServiceName) if err != nil { return transactionId, common.NewError(err) } @@ -241,7 +240,7 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac return transactionId, nil } -func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { +func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { tfields := common.FilterLogFields(fields, "status") tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { @@ -267,7 +266,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, } // this has 1 less retries compared to the standard DoWithRetries() -func (c *WebpaConnector) SyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, error) { +func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, error) { var rbytes []byte var err error var cont bool diff --git a/http/xconf_connector.go b/http/xconf_connector.go index 5057a5e..0c46ec0 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -18,7 +18,6 @@ package http import ( - "context" "crypto/tls" "fmt" "net/http" @@ -73,7 +72,7 @@ func (c *XconfConnector) ServiceName() string { return c.serviceName } -func (c *XconfConnector) GetProfiles(ctx context.Context, urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { +func (c *XconfConnector) GetProfiles(urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(c.XconfUrlTemplate(), c.XconfHost(), urlSuffix) rbytes, resHeader, err := c.DoWithRetries("GET", url, nil, nil, fields, c.ServiceName()) if err != nil { diff --git a/kafka/consumer.go b/kafka/consumer.go index 51cc318..5f95dd5 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -162,7 +162,7 @@ func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common. } mqttBytes := common.BuildPayloadAsHttp(status, respHeader, respBytes) - _, err = c.PostMqtt(ctx, cpeMac, mqttBytes, fields) + _, err = c.PostMqtt(cpeMac, mqttBytes, fields) if err != nil { return &m, common.NewError(err) } From 719198dbfb6a12d21a3d0e19a98d1e2c810ecd3c Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 15 Jan 2025 14:34:34 -0800 Subject: [PATCH 150/215] clean up used contexts cont. --- http/multipart.go | 10 ++++------ http/multipart_test.go | 20 ++++++++------------ kafka/consumer.go | 5 +---- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 91f55fc..5e39531 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -18,7 +18,6 @@ package http import ( - "context" "errors" "fmt" "net/http" @@ -44,7 +43,6 @@ var ( ) func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() // check if this is a Supplementary service, if so, call a different handler if hd := r.Header.Get(common.HeaderSupplementaryService); len(hd) > 0 { s.MultipartSupplementaryHandler(w, r) @@ -96,7 +94,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. r.Header.Set(common.HeaderSchemaVersion, "none") } - status, respHeader, respBytes, err := BuildWebconfigResponse(s, ctx, r.Header, common.RouteHttp, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) switch status { case http.StatusNotFound: @@ -120,7 +118,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. _, _ = w.Write(respBytes) } -func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader http.Header, route string, fields log.Fields) (int, http.Header, []byte, error) { +func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route string, fields log.Fields) (int, http.Header, []byte, error) { fields["for_device"] = true fields["is_primary"] = true @@ -133,7 +131,7 @@ func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader htt // factory reset handling ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) if ifNoneMatch == "NONE" || ifNoneMatch == "NONE-REBOOT" { - status, respHeader, rbytes, err := BuildFactoryResetResponse(s, ctx, rHeader, fields) + status, respHeader, rbytes, err := BuildFactoryResetResponse(s, rHeader, fields) if err != nil { return status, respHeader, rbytes, common.NewError(err) } @@ -339,7 +337,7 @@ func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader htt return http.StatusOK, upstreamRespHeader, finalFilteredBytes, nil } -func BuildFactoryResetResponse(s *WebconfigServer, ctx context.Context, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { +func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { c := s.DatabaseClient uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) diff --git a/http/multipart_test.go b/http/multipart_test.go index b20bb45..594550d 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -19,7 +19,6 @@ package http import ( "bytes" - "context" "encoding/hex" "fmt" "io" @@ -299,7 +298,6 @@ func TestCpeMiddleware(t *testing.T) { func TestVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) - ctx := context.Background() cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== @@ -417,7 +415,7 @@ func TestVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") fields := make(log.Fields) - status, respHeader, respBytes, err := BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType := respHeader.Get(common.HeaderContentType) @@ -435,7 +433,7 @@ func TestVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDocName, "root,lan,wan") kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusNotModified) assert.Equal(t, len(respBytes), 0) @@ -619,8 +617,6 @@ func TestUpstreamVersionFiltering(t *testing.T) { func TestMqttUpstreamVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) - ctx := context.Background() - cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== subdocId := "lan" @@ -680,7 +676,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader := make(http.Header) kHeader.Set(common.HeaderDeviceId, cpeMac) fields := make(log.Fields) - status, respHeader, respBytes, err := BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType := respHeader.Get(common.HeaderContentType) @@ -710,7 +706,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.Assert(t, err != nil) assert.Equal(t, status, http.StatusServiceUnavailable) @@ -718,7 +714,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader = make(http.Header) kHeader.Set(common.HeaderDeviceId, cpeMac) fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.Assert(t, err != nil) assert.Equal(t, status, http.StatusServiceUnavailable) @@ -744,7 +740,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType = respHeader.Get(common.HeaderContentType) @@ -768,7 +764,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusNotModified) @@ -779,7 +775,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType = respHeader.Get(common.HeaderContentType) diff --git a/kafka/consumer.go b/kafka/consumer.go index 5f95dd5..40e8ec1 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -18,7 +18,6 @@ package kafka import ( - "context" "encoding/base64" "encoding/json" "fmt" @@ -106,8 +105,6 @@ func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common // NOTE we choose to return an EventMessage object just to pass along the metricsAgent func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common.EventMessage, error) { - ctx := context.TODO() - rHeader, _ := util.ParseHttp(inbytes) params := rHeader.Get(common.HeaderDocName) cpeMac := rHeader.Get(common.HeaderDeviceId) @@ -151,7 +148,7 @@ func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common. rHeader.Set(common.HeaderSchemaVersion, "none") } - status, respHeader, respBytes, err := wchttp.BuildWebconfigResponse(c.WebconfigServer, ctx, rHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := wchttp.BuildWebconfigResponse(c.WebconfigServer, rHeader, common.RouteMqtt, fields) if err != nil && respBytes == nil { respBytes = []byte(err.Error()) } From 3bf15ca4071ffee75672cd950db86a20f136afaf Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 16 Jan 2025 11:39:46 -0800 Subject: [PATCH 151/215] small changes in tracing context functions --- tracing/ctx.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tracing/ctx.go b/tracing/ctx.go index 4a490fb..8115737 100644 --- a/tracing/ctx.go +++ b/tracing/ctx.go @@ -19,18 +19,12 @@ package tracing import "context" -type contextKey string - -func (c contextKey) String() string { - return string(c) -} - // SetContext - setting context for logging func SetContext(ctx context.Context, ctxName string, ctxValue interface{}) context.Context { - return context.WithValue(ctx, contextKey(ctxName), ctxValue) + return context.WithValue(ctx, ctxName, ctxValue) } // GetContext - getting context value from context func GetContext(ctx context.Context, key string) interface{} { - return ctx.Value(contextKey(key)) + return ctx.Value(key) } From 85b9308c52376bf60359d17d3e73994d72c0fe80 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 20 Jan 2025 11:28:57 -0800 Subject: [PATCH 152/215] small changes based on review --- http/webconfig_server.go | 7 +++---- tracing/ctx.go | 6 ++++-- tracing/otel.go | 14 +++++--------- tracing/span.go | 26 +++++++++++++------------- tracing/tracer.go | 37 ++++++++++++++++++++++++++++++++++--- 5 files changed, 59 insertions(+), 31 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 2941f31..1f1b4ac 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -748,7 +748,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } // traceparent handling for E2E tracing - xpcTrace := tracing.NewXpcTrace(r, s.XpcTracer) + xpcTrace := tracing.NewXpcTrace(s.XpcTracer, r) traceId := xpcTrace.TraceID if len(traceId) == 0 { traceId = xmTraceId @@ -899,8 +899,7 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["duration"] = duration fields["logger"] = "request" - tracing.SetSpanStatusCode(s.XpcTracer, fields) - tracing.SetSpanMoracideTags(s.XpcTracer, fields) + s.XpcTracer.SetSpan(fields, s.XpcTracer.MoracideTagPrefix()) var userAgent string if itf, ok := fields["user_agent"]; ok { @@ -1126,7 +1125,7 @@ func (s *WebconfigServer) StopXpcTracer() { func (s *WebconfigServer) SpanMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if s.XpcTracer.OtelEnabled { - ctx, otelSpan := tracing.NewOtelSpan(r, s.XpcTracer) + ctx, otelSpan := tracing.NewOtelSpan(s.XpcTracer, r) r = r.WithContext(ctx) defer tracing.EndOtelSpan(s.XpcTracer, otelSpan) } diff --git a/tracing/ctx.go b/tracing/ctx.go index 8115737..b106712 100644 --- a/tracing/ctx.go +++ b/tracing/ctx.go @@ -19,12 +19,14 @@ package tracing import "context" +type contextKey string + // SetContext - setting context for logging func SetContext(ctx context.Context, ctxName string, ctxValue interface{}) context.Context { - return context.WithValue(ctx, ctxName, ctxValue) + return context.WithValue(ctx, contextKey(ctxName), ctxValue) } // GetContext - getting context value from context func GetContext(ctx context.Context, key string) interface{} { - return ctx.Value(key) + return ctx.Value(contextKey(key)) } diff --git a/tracing/otel.go b/tracing/otel.go index c9dd75a..6617224 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -59,7 +59,7 @@ const defaultOtelTracerProvider = "noop" // initOtel - initialize OpenTelemetry constructs // tracing instrumentation code. -func otelInit(conf *configuration.Config, xpcTracer *XpcTracer) { +func otelInit(xpcTracer *XpcTracer, conf *configuration.Config) { xpcTracer.OtelEnabled = conf.GetBoolean("webconfig.tracing.otel.enabled") if !xpcTracer.OtelEnabled { return @@ -149,7 +149,7 @@ func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, erro ), nil } -func NewOtelSpan(r *http.Request, xpcTracer *XpcTracer) (context.Context, oteltrace.Span) { +func NewOtelSpan(xpcTracer *XpcTracer, r *http.Request) (context.Context, oteltrace.Span) { ctx := r.Context() var otelSpan oteltrace.Span if !xpcTracer.OtelEnabled { @@ -230,17 +230,13 @@ func EndOtelSpan(xpcTracer *XpcTracer, span oteltrace.Span) { span.End() } -func otelExtractParamsFromSpan(ctx context.Context, xpcTracer *XpcTracer, xpcTrace *XpcTrace) { - if !xpcTracer.OtelEnabled { - return - } +func otelExtractParamsFromSpan(ctx context.Context, xpcTrace *XpcTrace) { if tmp := GetContext(ctx, "otel_span"); tmp != nil { if otelSpan, ok := tmp.(oteltrace.Span); ok { - xpcTrace.otelSpan = otelSpan - if xpcTrace.otelSpan == nil { + if otelSpan == nil { return } - + xpcTrace.otelSpan = otelSpan spanCtx := otelSpan.SpanContext() xpcTrace.TraceID = spanCtx.TraceID().String() } diff --git a/tracing/span.go b/tracing/span.go index ba7edc1..c04fa65 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -68,20 +68,20 @@ type XpcTrace struct { } // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs -func NewXpcTrace(r *http.Request, xpcTracer *XpcTracer) *XpcTrace { +func NewXpcTrace(xpcTracer *XpcTracer, r *http.Request) *XpcTrace { var xpcTrace XpcTrace xpcTrace.ReqMoracideTags = make(map[string]string) - extractParamsFromReq(r, xpcTracer, &xpcTrace) + extractParamsFromReq(r, &xpcTrace, xpcTracer.MoracideTagPrefix()) if xpcTracer.OtelEnabled { - otelExtractParamsFromSpan(r.Context(), xpcTracer, &xpcTrace) + otelExtractParamsFromSpan(r.Context(), &xpcTrace) } return &xpcTrace } -func SetSpanStatusCode(xpcTracer *XpcTracer, fields log.Fields) { +func SetSpanStatusCode(fields log.Fields) { var xpcTrace *XpcTrace if tmp, ok := fields["xpc_trace"]; ok { xpcTrace = tmp.(*XpcTrace) @@ -91,7 +91,7 @@ func SetSpanStatusCode(xpcTracer *XpcTracer, fields log.Fields) { log.Error("instrumentation error, no trace info") return } - if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { + if xpcTrace.otelSpan != nil { if tmp, ok := fields["status"]; ok { statusCode := tmp.(int) otelSetStatusCode(xpcTrace.otelSpan, statusCode) @@ -99,7 +99,7 @@ func SetSpanStatusCode(xpcTracer *XpcTracer, fields log.Fields) { } } -func SetSpanMoracideTags(xpcTracer *XpcTracer, fields log.Fields) { +func SetSpanMoracideTags(fields log.Fields, moracideTagPrefix string) { var xpcTrace *XpcTrace if tmp, ok := fields["xpc_trace"]; ok { xpcTrace = tmp.(*XpcTrace) @@ -111,8 +111,8 @@ func SetSpanMoracideTags(xpcTracer *XpcTracer, fields log.Fields) { } moracideTags := make(map[string]string) - reqMoracideTagPrefix := strings.ToLower("req_" + xpcTracer.MoracideTagPrefix()) - respMoracideTagPrefix := strings.ToLower("resp_" + xpcTracer.MoracideTagPrefix()) + reqMoracideTagPrefix := strings.ToLower("req_" + moracideTagPrefix) + respMoracideTagPrefix := strings.ToLower("resp_" + moracideTagPrefix) for key, val := range fields { if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { @@ -130,17 +130,17 @@ func SetSpanMoracideTags(xpcTracer *XpcTracer, fields log.Fields) { if len(moracideTags) == 0 { // No moracide tags in request or any response // So set at least one span tag, x-cl-expt: false - moracideTags[xpcTracer.MoracideTagPrefix()] = "false" + moracideTags[moracideTagPrefix] = "false" } - for key, val := range moracideTags { - if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { + if xpcTrace.otelSpan != nil { + for key, val := range moracideTags { xpcTrace.otelSpan.SetAttributes(attribute.String(key, val)) log.Debugf("added otel span moracide tag key = %s, value = %s", key, val) } } } -func extractParamsFromReq(r *http.Request, xpcTracer *XpcTracer, xpcTrace *XpcTrace) { +func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace, moracideTagPrefix string) { xpcTrace.ReqTraceparent = r.Header.Get(common.HeaderTraceparent) xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent @@ -151,7 +151,7 @@ func extractParamsFromReq(r *http.Request, xpcTracer *XpcTracer, xpcTrace *XpcTr // In future, -H 'X-Cl-Experiment-1', -H 'X-Cl-Experiment-oswebconfig'... OR 'X-Cl-Experiment-xapproxy_25.1.1.1' are all possible // So walk through all headers and collect any header that starts with this prefix - moracideTagPrefix := strings.ToLower(xpcTracer.MoracideTagPrefix()) + moracideTagPrefix = strings.ToLower(moracideTagPrefix) for headerKey, headerVals := range r.Header { if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { if len(headerVals) > 1 { diff --git a/tracing/tracer.go b/tracing/tracer.go index fffe1f2..a65455a 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -34,6 +34,8 @@ const ( DefaultMoracideTagPrefix = "X-Cl-Experiment" ) +type SpanSetterFunc func(log.Fields, string) + // XpcTracer is a wrapper around tracer setup type XpcTracer struct { OtelEnabled bool @@ -54,13 +56,22 @@ type XpcTracer struct { otelTracerProvider oteltrace.TracerProvider otelPropagator otelpropagation.TextMapPropagator otelTracer oteltrace.Tracer + + SetSpan SpanSetterFunc } func NewXpcTracer(conf *configuration.Config) *XpcTracer { xpcTracer := new(XpcTracer) - initAppData(conf, xpcTracer) - otelInit(conf, xpcTracer) + initAppData(xpcTracer, conf) + otelInit(xpcTracer, conf) xpcTracer.moracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", DefaultMoracideTagPrefix) + + if xpcTracer.OtelEnabled { + xpcTracer.SetSpan = OtelSetSpan + } else { + xpcTracer.SetSpan = NoopSetSpan + } + return xpcTracer } @@ -84,7 +95,19 @@ func (t *XpcTracer) AppName() string { return t.appName } -func initAppData(conf *configuration.Config, xpcTracer *XpcTracer) { +func (t *XpcTracer) AppVersion() string { + return t.appVersion +} + +func (t *XpcTracer) AppEnv() string { + return t.appEnv +} + +func (t *XpcTracer) Region() string { + return t.region +} + +func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") xpcTracer.appName = codeGitCommit[0] if len(codeGitCommit) > 1 { @@ -108,3 +131,11 @@ func initAppData(conf *configuration.Config, xpcTracer *XpcTracer) { } log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) } + +func OtelSetSpan(fields log.Fields, tag string) { + SetSpanStatusCode(fields) + SetSpanMoracideTags(fields, tag) +} + +func NoopSetSpan(fields log.Fields, tag string) { +} From d99ae6d8443c4b2259c7896ce0e0637a354c456b Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Thu, 23 Jan 2025 15:57:55 -0800 Subject: [PATCH 153/215] Refactoring missed the tag settig part --- http/response.go | 2 +- tracing/span.go | 28 ++++++++++++---------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/http/response.go b/http/response.go index 5b748c3..c2c8f48 100644 --- a/http/response.go +++ b/http/response.go @@ -182,7 +182,7 @@ func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { moracideTags[key] = val } for key, val := range respMoracideTags { - if existingVal, ok := moracideTags[key]; !ok || (ok && existingVal != "true") { + if val == "true" { xw.LogDebug(nil, "request", fmt.Sprintf("Adding moracide tag from resp %s = %s to response", key, val)) moracideTags[key] = val } diff --git a/tracing/span.go b/tracing/span.go index c04fa65..e75e91e 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -111,27 +111,23 @@ func SetSpanMoracideTags(fields log.Fields, moracideTagPrefix string) { } moracideTags := make(map[string]string) - reqMoracideTagPrefix := strings.ToLower("req_" + moracideTagPrefix) - respMoracideTagPrefix := strings.ToLower("resp_" + moracideTagPrefix) - for key, val := range fields { - if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { - log.Debugf("Adding moracide tag from req %s = %s to response", key, val) - moracideTags[key[4:]] = val.(string) + if itf, ok := fields["req_moracide_tags"]; ok { + reqMoracideTags := itf.(map[string]string) + for key, val := range reqMoracideTags { + moracideTags[key] = val } - if strings.HasPrefix(strings.ToLower(key), respMoracideTagPrefix) { - log.Debugf("Adding moracide tag from resp %s = %s to response", key, val) - realKey := key[5:] - if existingVal, ok := moracideTags[realKey]; !ok || (ok && existingVal != "true") { - moracideTags[realKey] = val.(string) + } + + if itf, ok := fields["resp_moracide_tags"]; ok { + respMoracideTags := itf.(map[string]string) + for key, val := range respMoracideTags { + if val == "true" { + moracideTags[key] = val } } } - if len(moracideTags) == 0 { - // No moracide tags in request or any response - // So set at least one span tag, x-cl-expt: false - moracideTags[moracideTagPrefix] = "false" - } + if xpcTrace.otelSpan != nil { for key, val := range moracideTags { xpcTrace.otelSpan.SetAttributes(attribute.String(key, val)) From 514667e68fb34001485eadcb8972d617f7761d67 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 28 Jan 2025 22:57:48 -0800 Subject: [PATCH 154/215] filter webconfig output by bitmap --- common/const_var.go | 2 + common/document.go | 18 + common/firmware_bitmap.go | 147 +++++++ common/firmware_bitmap_test.go | 589 ++++++++++++++++++++++++++ common/root_document.go | 19 + db/service.go | 9 +- http/multipart.go | 28 +- http/rootdocument_handler_test.go | 4 +- http/supported_groups_handler.go | 6 +- http/supported_groups_handler_test.go | 10 +- http/webconfig_server.go | 12 + http/webconfig_server_test.go | 7 + util/firmware_bitmap.go | 19 +- util/firmware_bitmap_test.go | 17 + 14 files changed, 867 insertions(+), 20 deletions(-) create mode 100644 common/firmware_bitmap.go create mode 100644 common/firmware_bitmap_test.go diff --git a/common/const_var.go b/common/const_var.go index b8b8ec3..fee526c 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -112,6 +112,8 @@ const ( HeaderContentLength = "Content-Length" HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" HeaderUpstreamResponse = "X-Upstream-Response" + HeaderMoneytrace = "X-Moneytrace" + HeaderWebconfigTransactionId = "X-Webconfig-Transaction-Id" ) const ( diff --git a/common/document.go b/common/document.go index 7254dc0..e216c9a 100644 --- a/common/document.go +++ b/common/document.go @@ -219,3 +219,21 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { return BuildPayloadAsHttp(http.StatusOK, header, bbytes), nil } + +func (d *Document) FilterByBitmap() *Document { + rootdoc := d.GetRootDocument() + if rootdoc == nil { + return d + } + + newdoc := NewDocument(rootdoc) + supportedMap := GetSupportedMap(rootdoc.Bitmap) + + for subdocId, subDocument := range d.docmap { + if supportedMap[subdocId] { + newdoc.SetSubDocument(subdocId, &subDocument) + } + } + + return newdoc +} diff --git a/common/firmware_bitmap.go b/common/firmware_bitmap.go new file mode 100644 index 0000000..4ddef83 --- /dev/null +++ b/common/firmware_bitmap.go @@ -0,0 +1,147 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "strconv" + "strings" +) + +// bitmap is used to name int variables +// bitarray is used to name string variables + +func SetBitmapByGroup(cpeBitmap *int, groupId int, groupBitmap int) error { + tuples, ok := SupportedDocsBitMaskMap[groupId] + if !ok { + return nil + } + + for _, tuple := range tuples { + mask := 1 << (tuple.GroupBit - 1) + masked := groupBitmap & mask + if masked > 0 { + (*cpeBitmap) |= 1 << (tuple.CpeBit - 1) + } + } + return nil +} + +func ParseFirmwareGroupBitarray(s string) (int, int, error) { + i, err := strconv.Atoi(s) + if err != nil { + return 0, 0, NewError(err) + } + groupId := i >> 24 + groupBitMask := 1<<24 - 1 + groupBitmap := i & groupBitMask + return groupId, groupBitmap, nil +} + +func GetCpeBitmap(rdkSupportedDocsHeaderStr string) (int, error) { + cpeBitmap := 0 + + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + for _, sid := range sids { + groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) + if err != nil { + return 0, NewError(err) + } + + err = SetBitmapByGroup(&cpeBitmap, groupId, groupBitmap) + if err != nil { + return 0, NewError(err) + } + } + + return cpeBitmap, nil +} + +func PrettyBitarray(i int) string { + x := fmt.Sprintf("%032b", i) + return fmt.Sprintf("%s %s %s %s %s %s %s %s", + x[0:4], + x[4:8], + x[8:12], + x[12:16], + x[16:20], + x[20:24], + x[24:28], + x[28:32]) +} + +func PrettyGroupBitarray(i int) string { + x := fmt.Sprintf("%032b", i) + return fmt.Sprintf("%s %s %s %s %s %s %s", + x[0:8], + x[8:12], + x[12:16], + x[16:20], + x[20:24], + x[24:28], + x[28:32]) +} + +func IsSubdocSupported(cpeBitmap int, subdocId string) bool { + index, ok := SubdocBitIndexMap[subdocId] + if !ok { + return false + } + + shift := index - 1 + bitmask := 1 << shift + masked := cpeBitmap & bitmask + return masked != 0 +} + +func GetSupportedMap(cpeBitmap int) map[string]bool { + supportedMap := map[string]bool{} + + for k, index := range SubdocBitIndexMap { + shift := index - 1 + bitmask := 1 << shift + supportedMap[k] = false + if masked := cpeBitmap & bitmask; masked > 0 { + supportedMap[k] = true + } + } + return supportedMap +} + +func BitarrayToBitmap(src string) (int, error) { + s := strings.ReplaceAll(src, " ", "") + v, err := strconv.ParseInt(s, 2, 64) + if err != nil { + return 0, NewError(err) + } + i := int(v) + return i, nil +} + +func GetBitmapFromSupportedMap(srcMap map[string]bool) int { + var bitmap int + + for k, index := range SubdocBitIndexMap { + shift := index - 1 + bitmask := 1 << shift + if srcMap[k] { + bitmap += bitmask + } + } + return bitmap +} diff --git a/common/firmware_bitmap_test.go b/common/firmware_bitmap_test.go new file mode 100644 index 0000000..85c4e0b --- /dev/null +++ b/common/firmware_bitmap_test.go @@ -0,0 +1,589 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "math/rand" + "strings" + "testing" + + "gotest.tools/assert" +) + +// bitmap is used to name int variables +// bitarray is used to name string variables + +func TestPrettyBitarray(t *testing.T) { + i := 8 + bs := PrettyBitarray(i) + expected := "0000 0000 0000 0000 0000 0000 0000 1000" + assert.Equal(t, bs, expected) + + j := 16777231 + bs = PrettyGroupBitarray(j) + expected = "00000001 0000 0000 0000 0000 0000 1111" + assert.Equal(t, bs, expected) +} + +func TestParseRdkGroupBitarray(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + expectedMap := map[int]int{ + 1: 15, + 2: 3, + 3: 1, + 4: 1, + 5: 1, + 6: 1, + 7: 1, + 8: 1, + } + + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + for _, sid := range sids { + groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) + assert.NilError(t, err) + // assert.Equal(t, groupId, 1) + + expectedBitmap, ok := expectedMap[groupId] + assert.Assert(t, ok) + assert.Equal(t, groupBitmap, expectedBitmap) + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + + // bs := PrettyGroupBitarray(cpeBitmap) + + // build expected + expectedEnabled := map[string]bool{ + "portforwarding": true, + "lan": true, + "wan": true, + "macbinding": true, + "hotspot": false, + "bridge": false, + "privatessid": true, + "homessid": true, + "radio": false, + "moca": true, + "xdns": true, + "advsecurity": true, + "mesh": true, + "aker": true, + "telemetry": true, + "statusreport": false, + "trafficreport": false, + "interfacereport": false, + "radioreport": false, + } + + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } +} + +func TestParseCustomizedGroupBitarray(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + // build expected + expectedEnabled := map[string]bool{ + "portforwarding": true, + "lan": true, + "wan": true, + "macbinding": true, + "hotspot": false, + "bridge": false, + "privatessid": true, + "homessid": true, + "radio": false, + "moca": true, + "xdns": true, + "advsecurity": true, + "mesh": true, + "aker": true, + "telemetry": true, + "statusreport": false, + "trafficreport": false, + "interfacereport": false, + "radioreport": false, + "telcovoip": false, + "telcovoice": false, + "wanmanager": false, + "voiceservice": false, + } + + newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" + group1Bitmap, err := BitarrayToBitmap(newGroup1Bitarray) + assert.NilError(t, err) + sids[0] = fmt.Sprintf("%v", group1Bitmap) + expectedEnabled["wan"] = false + expectedEnabled["macbinding"] = false + expectedEnabled["hotspot"] = true + expectedEnabled["bridge"] = true + + newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" + group2Bitmap, err := BitarrayToBitmap(newGroup2Bitarray) + assert.NilError(t, err) + sids[1] = fmt.Sprintf("%v", group2Bitmap) + expectedEnabled["privatessid"] = false + expectedEnabled["radio"] = true + + rdkSupportedDocsHeaderStr = strings.Join(sids, ",") + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocIds := []string{} + for k, v := range expectedEnabled { + if v { + supportedSubdocIds = append(supportedSubdocIds, k) + } + } + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseTelcovoipAndWanmanager(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549377,201326593" + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "privatessid", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "telcovoip", + "wanmanager", + } + + rdkSupportedDocsHeaderStr = strings.Join(sids, ",") + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for _, subdocId := range supportedSubdocIds { + assert.Assert(t, IsSubdocSupported(cpeBitmap, subdocId)) + } + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestBitmapParsing(t *testing.T) { + // clearn the wan bit + newBitmap := 16777231 & ^(1 << 2) + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,33554435,50331649,67108865,83886081,100663297,117440513,134217729", newBitmap) + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "macbinding", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseVoiceService(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809" + + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestManualBitmap(t *testing.T) { + for i := 0; i < 10; i++ { + bitmap := rand.Intn(40000) + parsedSupportedMap := GetSupportedMap(bitmap) + revBitmap := GetBitmapFromSupportedMap(parsedSupportedMap) + assert.Equal(t, bitmap, revBitmap) + } +} + +func TestParseSupportedDocsWithNewGroups(t *testing.T) { + cellularBitGroupId := 14 + xBitValue := (cellularBitGroupId << 24) + 1 + ss := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809,234881025" + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", + "cellularconfig", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,67108865,100663297,117440513,134217729,201326595,234881025" + + // build expected + supportedSubdocIds := []string{ + "aker", + "cellularconfig", + "homessid", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549378,201326595" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telcovoice", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,251658241,268435457" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderRfc(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "defaultrfc", + "rfc", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderHcm(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWebui(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} diff --git a/common/root_document.go b/common/root_document.go index c662f09..54cf801 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -162,6 +162,25 @@ func (d *RootDocument) Update(r *RootDocument) { } } +func (d *RootDocument) UpdateMetadata(r *RootDocument) { + // Version and QueryParams are cloud data, so not changed + if r.Bitmap > 0 { + d.Bitmap = r.Bitmap + } + if len(r.FirmwareVersion) > 0 { + d.FirmwareVersion = r.FirmwareVersion + } + if len(r.ModelName) > 0 { + d.ModelName = r.ModelName + } + if len(r.PartnerId) > 0 { + d.PartnerId = r.PartnerId + } + if len(r.SchemaVersion) > 0 { + d.SchemaVersion = r.SchemaVersion + } +} + func (d *RootDocument) String() string { m := d.ColumnMap() bbytes, err := json.MarshalIndent(m, "", " ") diff --git a/db/service.go b/db/service.go index 169f2a0..66be7e9 100644 --- a/db/service.go +++ b/db/service.go @@ -64,7 +64,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } if len(supportedDocs) > 0 { - bitmap, err = util.GetCpeBitmap(supportedDocs) + bitmap, err = common.GetCpeBitmap(supportedDocs) if err != nil { log.WithFields(fields).Warn(common.NewError(err)) } @@ -146,8 +146,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } + // the returned err is dbNotFound - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + // WARNING, this should be removed + // return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + cloudRootDocument = clonedRootDoc.Clone() } // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== @@ -620,7 +623,7 @@ func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerI var err error supportedDocs := rHeader.Get(common.HeaderSupportedDocs) if len(supportedDocs) > 0 { - bitmap, err = util.GetCpeBitmap(supportedDocs) + bitmap, err = common.GetCpeBitmap(supportedDocs) if err != nil { log.WithFields(fields).Warn(common.NewError(err)) } diff --git a/http/multipart.go b/http/multipart.go index 5e39531..4f27811 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -123,7 +123,6 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin fields["is_primary"] = true c := s.DatabaseClient - uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) respHeader := make(http.Header) userAgent := rHeader.Get("User-Agent") @@ -148,7 +147,12 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusConflict, respHeader, nil, common.NewError(err) } - if uconn == nil { + if !s.UpstreamEnabled() { + if postUpstream { + rdoc := oldRootDocument.Clone() + rdoc.UpdateMetadata(newRootDocument) + document.SetRootDocument(rdoc) + } if err != nil { if !s.IsDbNotFound(err) { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -170,6 +174,11 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } + + if s.FilterOutputByBitmapEnabled() { + document = document.FilterByBitmap() + } + respBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -208,13 +217,17 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin var respBytes []byte respStatus := http.StatusNotModified if document.Length() > 0 { - if !postUpstream { document, err = db.LoadRefSubDocuments(c, document, fields) if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } } + // 3333 + if s.FilterOutputByBitmapEnabled() { + document = document.FilterByBitmap() + } + // 44444 respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -302,7 +315,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) // filter by versionMap and filter by blockedIds - finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") + finalRootDocument := common.NewRootDocument(newRootDocument.Bitmap, "", "", "", "", upstreamRespEtag, "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) @@ -320,6 +333,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalFilteredDocument.DeleteSubDocument(subdocId) } + if s.FilterOutputByBitmapEnabled() { + finalFilteredDocument = finalFilteredDocument.FilterByBitmap() + } + // 304 if finalFilteredDocument.Length() == 0 { return http.StatusNotModified, upstreamRespHeader, nil, nil @@ -339,7 +356,6 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { c := s.DatabaseClient - uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) respHeader := make(http.Header) @@ -374,7 +390,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - if uconn == nil { + if !s.UpstreamEnabled() { err := c.DeleteDocument(mac) if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 8c4e0f6..6bc640c 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -79,7 +79,7 @@ func TestRootDocumentHandler(t *testing.T) { rootdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + expectedBitmap1, err := common.GetCpeBitmap(supportedDocs1) assert.NilError(t, err) expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") assert.DeepEqual(t, rootdoc, expectedRootdoc) @@ -278,7 +278,7 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { rootdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + expectedBitmap1, err := common.GetCpeBitmap(supportedDocs1) assert.NilError(t, err) expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") assert.DeepEqual(t, rootdoc, expectedRootdoc) diff --git a/http/supported_groups_handler.go b/http/supported_groups_handler.go index 6eac50b..90a1a27 100644 --- a/http/supported_groups_handler.go +++ b/http/supported_groups_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,9 +23,9 @@ import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) // The supported doc header in GET /config is parsed and stored as a bitmap @@ -54,7 +54,7 @@ func (s *WebconfigServer) GetSupportedGroupsHandler(w http.ResponseWriter, r *ht outdata := common.SupportedGroupsData{ Bitmap: rdoc.Bitmap, - Groups: util.GetSupportedMap(rdoc.Bitmap), + Groups: common.GetSupportedMap(rdoc.Bitmap), } WriteOkResponse(w, outdata) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index b0c7f52..9256696 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -62,7 +62,7 @@ func TestSupportedGroupsHandler(t *testing.T) { rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) @@ -156,7 +156,7 @@ func TestSupportedGroupsHandler(t *testing.T) { sids := strings.Split(rdkSupportedDocsHeaderStr, ",") newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" - group1Bitmap, err := util.BitarrayToBitmap(newGroup1Bitarray) + group1Bitmap, err := common.BitarrayToBitmap(newGroup1Bitarray) assert.NilError(t, err) sids[0] = fmt.Sprintf("%v", group1Bitmap) supportedSubdocMap["wan"] = false @@ -165,7 +165,7 @@ func TestSupportedGroupsHandler(t *testing.T) { supportedSubdocMap["bridge"] = true newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" - group2Bitmap, err := util.BitarrayToBitmap(newGroup2Bitarray) + group2Bitmap, err := common.BitarrayToBitmap(newGroup2Bitarray) assert.NilError(t, err) sids[1] = fmt.Sprintf("%v", group2Bitmap) supportedSubdocMap["privatessid"] = false @@ -231,7 +231,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) @@ -297,7 +297,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1f1b4ac..6d51df6 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -118,6 +118,7 @@ type WebconfigServer struct { queryParamsValidationEnabled bool minTrust int validSubdocIdMap map[string]int + filterOutputByBitmapEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -289,6 +290,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validSubdocIdMap[x] = 1 } + filterOutputByBitmapEnabled := conf.GetBoolean("webconfig.filter_output_by_bitmap_enabled") + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -325,6 +328,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, XpcTracer: xpcTracer, + filterOutputByBitmapEnabled: filterOutputByBitmapEnabled, } return ws @@ -686,6 +690,14 @@ func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { s.validSubdocIdMap = x } +func (s *WebconfigServer) FilterOutputByBitmapEnabled() bool { + return s.filterOutputByBitmapEnabled +} + +func (s *WebconfigServer) SetFilterOutputByBitmapEnabled(enabled bool) { + s.filterOutputByBitmapEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 0c028aa..f7d484e 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -122,6 +122,13 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetMinTrust(trust) assert.Equal(t, server.MinTrust(), trust) + x := true + server.SetFilterOutputByBitmapEnabled(x) + assert.Assert(t, server.FilterOutputByBitmapEnabled()) + x = false + server.SetFilterOutputByBitmapEnabled(x) + assert.Assert(t, !server.FilterOutputByBitmapEnabled()) + validSubdocIdMap := map[string]int{ "red": 1, "orange": 2, diff --git a/util/firmware_bitmap.go b/util/firmware_bitmap.go index ab308fc..d9fa17d 100644 --- a/util/firmware_bitmap.go +++ b/util/firmware_bitmap.go @@ -14,7 +14,24 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package util import ( diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index cf3d5a5..abd622e 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -13,6 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. * +* SPDX-License-Identifier: Apache-2.0 + */ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* * SPDX-License-Identifier: Apache-2.0 */ package util From fb559f477a99cabbea959333698944323e8a4b55 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 29 Jan 2025 21:49:25 -0800 Subject: [PATCH 155/215] add a log field and some cleanup --- config/sample_webconfig.conf | 2 + http/poke_handler.go | 1 + util/firmware_bitmap.go | 166 ---------- util/firmware_bitmap_test.go | 606 ----------------------------------- util/string.go | 14 + 5 files changed, 17 insertions(+), 772 deletions(-) delete mode 100644 util/firmware_bitmap.go delete mode 100644 util/firmware_bitmap_test.go diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 36c8843..779c0c7 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -273,4 +273,6 @@ webconfig { // subdoc ids accepted by the validator even if they are without bitmap definition valid_subdoc_ids = [] + + filter_output_by_bitmap_enabled = false } diff --git a/http/poke_handler.go b/http/poke_handler.go index 29e19f3..4c5093c 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -63,6 +63,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } fields := xw.Audit() + fields["API"] = util.GetAPIName(r.URL.RawQuery) // extract "metrics_agent" metricsAgent := "default" diff --git a/util/firmware_bitmap.go b/util/firmware_bitmap.go deleted file mode 100644 index d9fa17d..0000000 --- a/util/firmware_bitmap.go +++ /dev/null @@ -1,166 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 - */ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 - */ -package util - -import ( - "fmt" - "strconv" - "strings" - - "github.com/rdkcentral/webconfig/common" -) - -// bitmap is used to name int variables -// bitarray is used to name string variables - -func SetBitmapByGroup(cpeBitmap *int, groupId int, groupBitmap int) error { - tuples, ok := common.SupportedDocsBitMaskMap[groupId] - if !ok { - return nil - } - - for _, tuple := range tuples { - mask := 1 << (tuple.GroupBit - 1) - masked := groupBitmap & mask - if masked > 0 { - (*cpeBitmap) |= 1 << (tuple.CpeBit - 1) - } - } - return nil -} - -func ParseFirmwareGroupBitarray(s string) (int, int, error) { - i, err := strconv.Atoi(s) - if err != nil { - return 0, 0, common.NewError(err) - } - groupId := i >> 24 - groupBitMask := 1<<24 - 1 - groupBitmap := i & groupBitMask - return groupId, groupBitmap, nil -} - -func GetCpeBitmap(rdkSupportedDocsHeaderStr string) (int, error) { - cpeBitmap := 0 - - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - for _, sid := range sids { - groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) - if err != nil { - return 0, common.NewError(err) - } - - err = SetBitmapByGroup(&cpeBitmap, groupId, groupBitmap) - if err != nil { - return 0, common.NewError(err) - } - } - - return cpeBitmap, nil -} - -func PrettyBitarray(i int) string { - x := fmt.Sprintf("%032b", i) - return fmt.Sprintf("%s %s %s %s %s %s %s %s", - x[0:4], - x[4:8], - x[8:12], - x[12:16], - x[16:20], - x[20:24], - x[24:28], - x[28:32]) -} - -func PrettyGroupBitarray(i int) string { - x := fmt.Sprintf("%032b", i) - return fmt.Sprintf("%s %s %s %s %s %s %s", - x[0:8], - x[8:12], - x[12:16], - x[16:20], - x[20:24], - x[24:28], - x[28:32]) -} - -func IsSubdocSupported(cpeBitmap int, subdocId string) bool { - index, ok := common.SubdocBitIndexMap[subdocId] - if !ok { - return false - } - - shift := index - 1 - bitmask := 1 << shift - masked := cpeBitmap & bitmask - return masked != 0 -} - -func GetSupportedMap(cpeBitmap int) map[string]bool { - supportedMap := map[string]bool{} - - for k, index := range common.SubdocBitIndexMap { - shift := index - 1 - bitmask := 1 << shift - supportedMap[k] = false - if masked := cpeBitmap & bitmask; masked > 0 { - supportedMap[k] = true - } - } - return supportedMap -} - -func BitarrayToBitmap(src string) (int, error) { - s := strings.ReplaceAll(src, " ", "") - v, err := strconv.ParseInt(s, 2, 64) - if err != nil { - return 0, common.NewError(err) - } - i := int(v) - return i, nil -} - -func GetBitmapFromSupportedMap(srcMap map[string]bool) int { - var bitmap int - - for k, index := range common.SubdocBitIndexMap { - shift := index - 1 - bitmask := 1 << shift - if srcMap[k] { - bitmap += bitmask - } - } - return bitmap -} diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go deleted file mode 100644 index abd622e..0000000 --- a/util/firmware_bitmap_test.go +++ /dev/null @@ -1,606 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 - */ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 - */ -package util - -import ( - "fmt" - "strings" - "testing" - - "github.com/rdkcentral/webconfig/common" - "gotest.tools/assert" -) - -// bitmap is used to name int variables -// bitarray is used to name string variables - -func TestPrettyBitarray(t *testing.T) { - i := 8 - bs := PrettyBitarray(i) - expected := "0000 0000 0000 0000 0000 0000 0000 1000" - assert.Equal(t, bs, expected) - - j := 16777231 - bs = PrettyGroupBitarray(j) - expected = "00000001 0000 0000 0000 0000 0000 1111" - assert.Equal(t, bs, expected) -} - -func TestParseRdkGroupBitarray(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" - expectedMap := map[int]int{ - 1: 15, - 2: 3, - 3: 1, - 4: 1, - 5: 1, - 6: 1, - 7: 1, - 8: 1, - } - - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - for _, sid := range sids { - groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) - assert.NilError(t, err) - // assert.Equal(t, groupId, 1) - - expectedBitmap, ok := expectedMap[groupId] - assert.Assert(t, ok) - assert.Equal(t, groupBitmap, expectedBitmap) - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - - // bs := PrettyGroupBitarray(cpeBitmap) - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - } - - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } -} - -func TestParseCustomizedGroupBitarray(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - } - - newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" - group1Bitmap, err := BitarrayToBitmap(newGroup1Bitarray) - assert.NilError(t, err) - sids[0] = fmt.Sprintf("%v", group1Bitmap) - expectedEnabled["wan"] = false - expectedEnabled["macbinding"] = false - expectedEnabled["hotspot"] = true - expectedEnabled["bridge"] = true - - newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" - group2Bitmap, err := BitarrayToBitmap(newGroup2Bitarray) - assert.NilError(t, err) - sids[1] = fmt.Sprintf("%v", group2Bitmap) - expectedEnabled["privatessid"] = false - expectedEnabled["radio"] = true - - rdkSupportedDocsHeaderStr = strings.Join(sids, ",") - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocIds := []string{} - for k, v := range expectedEnabled { - if v { - supportedSubdocIds = append(supportedSubdocIds, k) - } - } - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseTelcovoipAndWanmanager(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549377,201326593" - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - // build expected - supportedSubdocIds := []string{ - "portforwarding", - "lan", - "wan", - "macbinding", - "privatessid", - "xdns", - "advsecurity", - "mesh", - "aker", - "telemetry", - "telcovoip", - "wanmanager", - } - - rdkSupportedDocsHeaderStr = strings.Join(sids, ",") - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for _, subdocId := range supportedSubdocIds { - assert.Assert(t, IsSubdocSupported(cpeBitmap, subdocId)) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestBitmapParsing(t *testing.T) { - // clearn the wan bit - newBitmap := 16777231 & ^(1 << 2) - rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,33554435,50331649,67108865,83886081,100663297,117440513,134217729", newBitmap) - - // build expected - supportedSubdocIds := []string{ - "portforwarding", - "lan", - "macbinding", - "privatessid", - "homessid", - "moca", - "xdns", - "advsecurity", - "mesh", - "aker", - "telemetry", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseVoiceService(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809" - - supportedSubdocIds := []string{ - "portforwarding", - "lan", - "wan", - "macbinding", - "hotspot", - "privatessid", - "homessid", - "moca", - "xdns", - "advsecurity", - "mesh", - "aker", - "telemetry", - "voiceservice", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestManualBitmap(t *testing.T) { - for i := 0; i < 10; i++ { - bitmap := RandomInt(40000) - parsedSupportedMap := GetSupportedMap(bitmap) - revBitmap := GetBitmapFromSupportedMap(parsedSupportedMap) - assert.Equal(t, bitmap, revBitmap) - } -} - -func TestParseSupportedDocsWithNewGroups(t *testing.T) { - cellularBitGroupId := 14 - xBitValue := (cellularBitGroupId << 24) + 1 - ss := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809,234881025" - rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) - - // build expected - supportedSubdocIds := []string{ - "portforwarding", - "lan", - "wan", - "macbinding", - "hotspot", - "privatessid", - "homessid", - "moca", - "xdns", - "advsecurity", - "mesh", - "aker", - "telemetry", - "voiceservice", - "cellularconfig", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,67108865,100663297,117440513,134217729,201326595,234881025" - - // build expected - supportedSubdocIds := []string{ - "aker", - "cellularconfig", - "homessid", - "lan", - "macbinding", - "mesh", - "portforwarding", - "privatessid", - "telemetry", - "wan", - "wanfailover", - "wanmanager", - "xdns", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549378,201326595" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "lan", - "macbinding", - "mesh", - "portforwarding", - "privatessid", - "telcovoice", - "telemetry", - "wan", - "wanfailover", - "wanmanager", - "xdns", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,251658241,268435457" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - "clienttosteeringprofile", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderRfc(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - "clienttosteeringprofile", - "defaultrfc", - "rfc", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderHcm(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - "clienttosteeringprofile", - "meshsteeringprofiles", - "wifistatsconfig", - "mwoconfigs", - "interference", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWebui(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - "clienttosteeringprofile", - "meshsteeringprofiles", - "wifistatsconfig", - "mwoconfigs", - "interference", - "xmspeedboost", - "webui", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} diff --git a/util/string.go b/util/string.go index 064cc8e..6c6cc0c 100644 --- a/util/string.go +++ b/util/string.go @@ -162,3 +162,17 @@ func CompactJson(sbytes []byte) ([]byte, error) { } return jsbytes, nil } + +func GetAPIName(queryStr string) string { + n := "poke" + if strings.Contains(queryStr, "slow") { + n = "slowpoke" + } else if strings.Contains(queryStr, "root") && strings.Contains(queryStr, "telemetry") { + n = "poke_both" + } else if strings.Contains(queryStr, "telemetry") { + n = "poke_telemetry" + } else if strings.Contains(queryStr, "mqtt") { + n = "poke_mqtt" + } + return n +} From c77a999f3018c839f3b75e37e6d9ab22bc6baef3 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 15 Feb 2025 17:00:43 -0800 Subject: [PATCH 156/215] propagate request response headers --- common/const_var.go | 1 + common/log_fields.go | 2 -- http/http_client.go | 26 +++++------------- http/response.go | 26 +++++++----------- http/response_writer.go | 16 ----------- http/webconfig_server.go | 30 ++++++++++---------- tracing/span.go | 59 ++++++---------------------------------- util/dict.go | 13 ++++++++- 8 files changed, 53 insertions(+), 120 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index fee526c..9b51d8b 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -114,6 +114,7 @@ const ( HeaderUpstreamResponse = "X-Upstream-Response" HeaderMoneytrace = "X-Moneytrace" HeaderWebconfigTransactionId = "X-Webconfig-Transaction-Id" + HeaderMoracide = "X-Cl-Experiment" ) const ( diff --git a/common/log_fields.go b/common/log_fields.go index 10fd63a..e5472df 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -28,8 +28,6 @@ var ( "moneytrace", "token", "xpc_trace", - "req_moracide_tags", - "resp_moracide_tags", } coreFields = []string{ "app_name", diff --git a/http/http_client.go b/http/http_client.go index 71ff1f9..2761f9c 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -428,30 +428,18 @@ func (c *HttpClient) addMoracideTags(header http.Header, fields log.Fields) { } } - itf, ok := fields["req_moracide_tags"] - if !ok { - return - } - moracideTagMap := itf.(map[string]string) - - for key, val := range moracideTagMap { - log.WithFields(fields).Debugf("Adding moracide tag %s = %s to outgoing req", key, val) - header.Set(key, val) + moracide := util.FieldsGetString(fields, "req_moracide_tag") + if len(moracide) > 0 { + header.Set(common.HeaderMoracide, moracide) } } func (c *HttpClient) addMoracideTagsFromResponse(header http.Header, fields log.Fields) bool { var respMoracideTagsFound bool - m := make(map[string]string) - for k := range header { - if strings.HasPrefix(strings.ToLower(k), c.moracideTagPrefix) { - respMoracideTagsFound = true - val := header.Get(k) - m[k] = val - } - } - if len(m) > 0 { - fields["resp_moracide_tags"] = m + moracide := header.Get(common.HeaderMoracide) + if len(moracide) > 0 { + fields["resp_moracide_tag"] = moracide + respMoracideTagsFound = true } return respMoracideTagsFound } diff --git a/http/response.go b/http/response.go index c2c8f48..baebe01 100644 --- a/http/response.go +++ b/http/response.go @@ -23,6 +23,7 @@ import ( "net/http" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -172,23 +173,16 @@ func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { if !ok { return } - - reqMoracideTags := xw.ReqMoracideTags() - respMoracideTags := xw.RespMoracideTags() - - moracideTags := make(map[string]string) - for key, val := range reqMoracideTags { - xw.LogDebug(nil, "request", fmt.Sprintf("Adding moracide tag from req %s = %s to response", key, val)) - moracideTags[key] = val + fields := xw.Audit() + if fields == nil { + return } - for key, val := range respMoracideTags { - if val == "true" { - xw.LogDebug(nil, "request", fmt.Sprintf("Adding moracide tag from resp %s = %s to response", key, val)) - moracideTags[key] = val - } + + moracide := util.FieldsGetString(fields, "resp_moracide_tag") + if len(moracide) == 0 { + moracide = util.FieldsGetString(fields, "req_moracide_tag") } - xw.LogDebug(nil, "request", fmt.Sprintf("moracideTags = %+v", moracideTags)) - for key, val := range moracideTags { - w.Header().Set(key, val) + if len(moracide) > 0 { + w.Header().Set(common.HeaderMoracide, moracide) } } diff --git a/http/response_writer.go b/http/response_writer.go index ecc7e0b..47c382e 100644 --- a/http/response_writer.go +++ b/http/response_writer.go @@ -165,19 +165,3 @@ func (w *XResponseWriter) SetPartnerId(partnerId string) { w.partnerId = partnerId w.audit["partner"] = partnerId } - -func (w *XResponseWriter) ReqMoracideTags() map[string]string { - itf, ok := w.audit["req_moracide_tags"] - if !ok { - return nil - } - return itf.(map[string]string) -} - -func (w *XResponseWriter) RespMoracideTags() map[string]string { - itf, ok := w.audit["resp_moracide_tags"] - if !ok { - return nil - } - return itf.(map[string]string) -} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 6d51df6..9073242 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -768,21 +768,21 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques headerMap := util.HeaderToMap(header) fields := log.Fields{ - "path": r.URL.String(), - "method": r.Method, - "audit_id": auditId, - "remote_ip": remoteIp, - "host_name": host, - "header": headerMap, - "logger": "request", - "trace_id": traceId, - "app_name": s.AppName(), - "traceparent": xpcTrace.ReqTraceparent, - "tracestate": xpcTrace.ReqTracestate, - "out_traceparent": xpcTrace.OutTraceparent, - "out_tracestate": xpcTrace.OutTracestate, - "req_moracide_tags": xpcTrace.ReqMoracideTags, - "xpc_trace": xpcTrace, + "path": r.URL.String(), + "method": r.Method, + "audit_id": auditId, + "remote_ip": remoteIp, + "host_name": host, + "header": headerMap, + "logger": "request", + "trace_id": traceId, + "app_name": s.AppName(), + "traceparent": xpcTrace.ReqTraceparent, + "tracestate": xpcTrace.ReqTracestate, + "out_traceparent": xpcTrace.OutTraceparent, + "out_tracestate": xpcTrace.OutTracestate, + "req_moracide_tag": xpcTrace.ReqMoracideTag, + "xpc_trace": xpcTrace, } userAgent := r.UserAgent() diff --git a/tracing/span.go b/tracing/span.go index e75e91e..efe1e97 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -19,9 +19,9 @@ package tracing import ( "net/http" - "strings" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" @@ -33,11 +33,7 @@ import ( // Ideal place to store it is ofc, xw // But because of legacy reasons, xw is not always available in the API flow type XpcTrace struct { - // This is a bit of overengineering, but multiple tags are possible - // e.g. X-Cl-Experiment-1, X-Cl-Experiment-xapproxy, X-Cl-Experiement-webconfig-25.1.1.1... - // For every key found in either req or resp, an explicit value of true/false will be set as an otel attribute - // or an otel span attribute - ReqMoracideTags map[string]string // These are request headers prefixed with MoracideTagPrefix + ReqMoracideTag string // The response moracide tags are stored in xw.audit // traceparent, tracestate can be set as req headers, may be extracted from otel spans @@ -70,8 +66,6 @@ type XpcTrace struct { // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs func NewXpcTrace(xpcTracer *XpcTracer, r *http.Request) *XpcTrace { var xpcTrace XpcTrace - xpcTrace.ReqMoracideTags = make(map[string]string) - extractParamsFromReq(r, &xpcTrace, xpcTracer.MoracideTagPrefix()) if xpcTracer.OtelEnabled { @@ -110,29 +104,13 @@ func SetSpanMoracideTags(fields log.Fields, moracideTagPrefix string) { return } - moracideTags := make(map[string]string) - - if itf, ok := fields["req_moracide_tags"]; ok { - reqMoracideTags := itf.(map[string]string) - for key, val := range reqMoracideTags { - moracideTags[key] = val - } - } - - if itf, ok := fields["resp_moracide_tags"]; ok { - respMoracideTags := itf.(map[string]string) - for key, val := range respMoracideTags { - if val == "true" { - moracideTags[key] = val - } - } + moracide := util.FieldsGetString(fields, "resp_moracide_tag") + if len(moracide) == 0 { + moracide = util.FieldsGetString(fields, "req_moracide_tag") } - if xpcTrace.otelSpan != nil { - for key, val := range moracideTags { - xpcTrace.otelSpan.SetAttributes(attribute.String(key, val)) - log.Debugf("added otel span moracide tag key = %s, value = %s", key, val) - } + if xpcTrace.otelSpan != nil && len(moracide) > 0 { + xpcTrace.otelSpan.SetAttributes(attribute.String(common.HeaderMoracide, moracide)) } } @@ -141,27 +119,6 @@ func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace, moracideTagPrefix xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent xpcTrace.OutTracestate = xpcTrace.ReqTracestate - log.Debugf("Tracing: input traceparent : %s, tracestate : %s", xpcTrace.ReqTraceparent, xpcTrace.ReqTracestate) - xpcTrace.ReqUserAgent = r.Header.Get(UserAgentHeader) - - // In future, -H 'X-Cl-Experiment-1', -H 'X-Cl-Experiment-oswebconfig'... OR 'X-Cl-Experiment-xapproxy_25.1.1.1' are all possible - // So walk through all headers and collect any header that starts with this prefix - moracideTagPrefix = strings.ToLower(moracideTagPrefix) - for headerKey, headerVals := range r.Header { - if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { - if len(headerVals) > 1 { - log.Debugf("Tracing: moracide tag key = %s, has multiple values = %+v", headerKey, headerVals) - } - val := "false" - for _, v := range headerVals { - if v == "true" { - val = v - break - } - } - xpcTrace.ReqMoracideTags[headerKey] = val - log.Debugf("Tracing: found moracide tag key = %s, val = %s, all vals = %+v", headerKey, val, headerVals) - } - } + xpcTrace.ReqMoracideTag = r.Header.Get(common.HeaderMoracide) } diff --git a/util/dict.go b/util/dict.go index d5242eb..fb8d570 100644 --- a/util/dict.go +++ b/util/dict.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -216,5 +216,16 @@ func HeaderToMap(header http.Header) map[string]string { m[k] = v[0] } return m +} + +func FieldsGetString(fields log.Fields, keyName string, args ...string) string { + var ret string + if len(args) == 1 { + ret = args[0] + } + if itf, ok := fields[keyName]; ok { + ret = itf.(string) + } + return ret } From abc0717ce1184a217ab2fd5b76f9517415c1dce3 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 18 Feb 2025 21:58:25 -0800 Subject: [PATCH 157/215] handle mqtt poke 403 errors --- http/poke_handler.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 4c5093c..266023d 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -109,10 +109,8 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - if rherr.StatusCode == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) - return - } + Error(w, rherr.StatusCode, common.NewError(err)) + return } Error(w, http.StatusInternalServerError, common.NewError(err)) return From ad8ff9a382fd37b03b4b1da7b35e8deebff30973 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 19 Feb 2025 15:57:50 -0800 Subject: [PATCH 158/215] Handle header x-cl-canary for moracide --- common/const_var.go | 1 + tracing/span.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 9b51d8b..2590d3b 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -115,6 +115,7 @@ const ( HeaderMoneytrace = "X-Moneytrace" HeaderWebconfigTransactionId = "X-Webconfig-Transaction-Id" HeaderMoracide = "X-Cl-Experiment" + HeaderCanary = "X-Cl-Canary" ) const ( diff --git a/tracing/span.go b/tracing/span.go index efe1e97..123925f 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -66,7 +66,7 @@ type XpcTrace struct { // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs func NewXpcTrace(xpcTracer *XpcTracer, r *http.Request) *XpcTrace { var xpcTrace XpcTrace - extractParamsFromReq(r, &xpcTrace, xpcTracer.MoracideTagPrefix()) + extractParamsFromReq(r, &xpcTrace, xpcTracer.AppName()) if xpcTracer.OtelEnabled { otelExtractParamsFromSpan(r.Context(), &xpcTrace) @@ -114,11 +114,18 @@ func SetSpanMoracideTags(fields log.Fields, moracideTagPrefix string) { } } -func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace, moracideTagPrefix string) { +func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace, serviceName string) { xpcTrace.ReqTraceparent = r.Header.Get(common.HeaderTraceparent) xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent xpcTrace.OutTracestate = xpcTrace.ReqTracestate xpcTrace.ReqUserAgent = r.Header.Get(UserAgentHeader) xpcTrace.ReqMoracideTag = r.Header.Get(common.HeaderMoracide) + if ss := r.Header.Get(common.HeaderCanary); ss == "true" { + if len(xpcTrace.ReqMoracideTag) > 0 { + xpcTrace.ReqMoracideTag += "," + serviceName + } else { + xpcTrace.ReqMoracideTag = serviceName + } + } } From a5e599d2063bdf41ba107b664b1f43bf50e86ba2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 28 Feb 2025 16:19:06 -0800 Subject: [PATCH 159/215] Fix a bug that 200 empty profiles were return when both xconf and upstream have no profiles --- common/log_fields.go | 17 +++ http/supplementary_handler.go | 16 ++- http/supplementary_handler_test.go | 192 +++++++++++++++++++++++++++++ http/webconfig_server.go | 32 ++++- 4 files changed, 250 insertions(+), 7 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index e5472df..4bbed90 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -19,6 +19,7 @@ package common import ( "maps" + "slices" log "github.com/sirupsen/logrus" ) @@ -35,6 +36,13 @@ var ( "body", "cpe_mac", } + nonEmptyFields = []string{ + "traceparent", + "tracestate", + "out_traceparent", + "out_tracestate", + "req_moracide_tag", + } ) func FilterLogFields(src log.Fields, excludes ...string) log.Fields { @@ -45,6 +53,14 @@ func FilterLogFields(src log.Fields, excludes ...string) log.Fields { fields[k] = maps.Clone(ty) case map[string]interface{}: fields[k] = maps.Clone(ty) + case string: + if slices.Contains(nonEmptyFields, k) { + if len(ty) > 0 { + fields[k] = ty + } + } else { + fields[k] = ty + } default: fields[k] = ty } @@ -59,6 +75,7 @@ func FilterLogFields(src log.Fields, excludes ...string) log.Fields { delete(fields, x) } } + return fields } diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 20dc8fd..8dace6d 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -88,6 +88,8 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r if rherr.StatusCode == http.StatusNotFound { if s.UpstreamProfilesEnabled() { xconfNotFound = true + } else if s.DefaultEmptyProfileEnabled() { + xconfNotFound = true } else { Error(w, rherr.StatusCode, rherr) return @@ -103,10 +105,10 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } } - var profileBytes []byte + var profileBytes, extraProfileBytes []byte if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { // Get profiles from the second source - extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) + extraProfileBytes, _, err = s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) if err != nil { exitNow := true var rherr common.RemoteHttpError @@ -139,6 +141,15 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r profileBytes = baseProfileBytes } + if xconfNotFound && extraProfileBytes == nil && !s.DefaultEmptyProfileEnabled() { + Error(w, http.StatusNotFound, nil) + return + } + + if len(profileBytes) == 0 && s.DefaultEmptyProfileEnabled() { + profileBytes = []byte(notFoundProfileText) + } + mpart, err := util.TelemetryBytesToMultipart(profileBytes) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) @@ -147,6 +158,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r mparts := []common.Multipart{ mpart, } + fields["telemetry_version"] = mpart.Version respBytes, err := common.WriteMultipartBytes(mparts) if err != nil { diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 1328b9c..f99bd95 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -1129,3 +1129,195 @@ func TestSupplementaryUpstreamProfiles(t *testing.T) { assert.Assert(t, ok) assert.Equal(t, profile3["name"].(string), "subname2") } + +func TestSupplementaryUpstreamProfilesNotFoundNotDefaultEmptyProfile(t *testing.T) { + log.SetOutput(io.Discard) + + conf := sc.Config + ss := "webconfig.upstream_profiles_enabled = true" + conf = conf.AddConfig(ss, nil) + tsc := *sc + tsc.Config = conf + + server := NewWebconfigServer(&tsc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + // set up root_document table + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true") + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + getRdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rdoc.Compare(getRdoc) == 0) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} + +func TestSupplementaryUpstreamProfilesNotFoundDefaultEmptyProfile(t *testing.T) { + log.SetOutput(io.Discard) + + conf := sc.Config + ss := "webconfig.upstream_profiles_enabled=true\nwebconfig.default_empty_profile_enabled=true" + conf = conf.AddConfig(ss, nil) + tsc := *sc + tsc.Config = conf + + server := NewWebconfigServer(&tsc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + // set up root_document table + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true") + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + getRdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rdoc.Compare(getRdoc) == 0) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusOK) +} + +func TestSupplementaryDefaultEmptyProfile(t *testing.T) { + log.SetOutput(io.Discard) + + conf := sc.Config + ss := "webconfig.default_empty_profile_enabled=true" + conf = conf.AddConfig(ss, nil) + tsc := *sc + tsc.Config = conf + + server := NewWebconfigServer(&tsc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + // set up root_document table + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true") + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + getRdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rdoc.Compare(getRdoc) == 0) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusOK) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 9073242..088b69a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -119,6 +119,7 @@ type WebconfigServer struct { minTrust int validSubdocIdMap map[string]int filterOutputByBitmapEnabled bool + defaultEmptyProfileEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -291,6 +292,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } filterOutputByBitmapEnabled := conf.GetBoolean("webconfig.filter_output_by_bitmap_enabled") + defaultEmptyProfileEnabled := conf.GetBoolean("webconfig.default_empty_profile_enabled") ws := &WebconfigServer{ Server: &http.Server{ @@ -329,6 +331,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validSubdocIdMap: validSubdocIdMap, XpcTracer: xpcTracer, filterOutputByBitmapEnabled: filterOutputByBitmapEnabled, + defaultEmptyProfileEnabled: defaultEmptyProfileEnabled, } return ws @@ -698,6 +701,14 @@ func (s *WebconfigServer) SetFilterOutputByBitmapEnabled(enabled bool) { s.filterOutputByBitmapEnabled = enabled } +func (s *WebconfigServer) DefaultEmptyProfileEnabled() bool { + return s.defaultEmptyProfileEnabled +} + +func (s *WebconfigServer) SetDefaultEmptyProfileEnabled(enabled bool) { + s.defaultEmptyProfileEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { @@ -809,7 +820,6 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques case "cpe": mac := params["gid"] mac = strings.ToUpper(mac) - fields["cpemac"] = mac fields["cpe_mac"] = mac case "configset": csid := params["gid"] @@ -818,7 +828,6 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } if mac, ok := params["mac"]; ok { mac = strings.ToUpper(mac) - fields["cpemac"] = mac fields["cpe_mac"] = mac } @@ -864,9 +873,22 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["response"] = mpdict } } else { - res_itf, res_text := GetResponseLogObjs(rbytes) - fields["response"] = res_itf - fields["response_text"] = res_text + var logged bool + if itf, ok := fields["telemetry_version"]; ok { + if version, ok := itf.(string); ok { + if len(version) > 0 { + logged = true + fields["response"] = map[string]string{ + "telemetry": version, + } + } + } + } + if !logged { + res_itf, res_text := GetResponseLogObjs(rbytes) + fields["response"] = res_itf + fields["response_text"] = res_text + } } var doc_map util.Dict From 762bae97e8989d48f3972117df13b319010a92dc Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Wed, 5 Mar 2025 11:18:29 -0800 Subject: [PATCH 160/215] Add version info to otel instrumentation --- tracing/otel.go | 2 ++ tracing/tracer.go | 24 +++++++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tracing/otel.go b/tracing/otel.go index 6617224..8e47be3 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -117,6 +117,7 @@ func otelStdoutTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, er resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceVersionKey.String(xpcTracer.versionForTracing), semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), ), ), @@ -143,6 +144,7 @@ func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, erro resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceVersionKey.String(xpcTracer.versionForTracing), semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), ), ), diff --git a/tracing/tracer.go b/tracing/tracer.go index a65455a..049a4fe 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -42,12 +42,13 @@ type XpcTracer struct { moracideTagPrefix string // Special request header for moracide expts e.g. canary deployments // internal vars used by Otel - appEnv string // set this to dev for red, staging for yellow and prod for green - appName string - appVersion string - appSHA string // unused - region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute - siteColor string // red/yellow/green, unused, use it as a span attribute + appEnv string // set this to dev for red, staging for yellow and prod for green + appName string + appVersion string + appSHA string // unused + region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute + siteColor string // red/yellow/green, unused, use it as a span attribute + versionForTracing string // internal otel vars otelEndpoint string @@ -129,7 +130,16 @@ func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { if xpcTracer.region == "" { xpcTracer.region = os.Getenv("site_region_name") } - log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) + + xpcTracer.versionForTracing = os.Getenv("OTEL_VERSION") + if xpcTracer.versionForTracing == "" { + xpcTracer.versionForTracing = xpcTracer.appVersion + } + if xpcTracer.versionForTracing == "" { + xpcTracer.versionForTracing = "v1" + } + + log.Debugf("site_color = %s, env = %s, region = %s version = %s", siteColor, xpcTracer.appEnv, xpcTracer.region, xpcTracer.versionForTracing) } func OtelSetSpan(fields log.Fields, tag string) { From ea14fd0ffc125e232ce3ec230a6da49411fd73a0 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Wed, 5 Mar 2025 21:49:33 -0800 Subject: [PATCH 161/215] Add SHA to version info --- tracing/tracer.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tracing/tracer.go b/tracing/tracer.go index 049a4fe..c3bceca 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -45,7 +45,6 @@ type XpcTracer struct { appEnv string // set this to dev for red, staging for yellow and prod for green appName string appVersion string - appSHA string // unused region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute siteColor string // red/yellow/green, unused, use it as a span attribute versionForTracing string @@ -109,14 +108,11 @@ func (t *XpcTracer) Region() string { } func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { - codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") + codeGitCommit := strings.SplitN(conf.GetString("webconfig.code_git_commit"), "-", 2) xpcTracer.appName = codeGitCommit[0] if len(codeGitCommit) > 1 { xpcTracer.appVersion = codeGitCommit[1] } - if len(codeGitCommit) > 2 { - xpcTracer.appSHA = codeGitCommit[2] - } // Env vars xpcTracer.appEnv = "dev" From 4e59b166e22a948abe88e9a4aa9c911db5c2197b Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Sun, 9 Mar 2025 21:21:53 -0700 Subject: [PATCH 162/215] version info to include app name --- config/sample_webconfig.conf | 4 ++-- tracing/tracer.go | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 779c0c7..dae7dd0 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -19,8 +19,8 @@ webconfig { } // build info - code_git_commit = "2ac7ff4" - build_time = "Thu Feb 14 01:57:26 2019 UTC" + code_git_commit = "oswebconfig-25.2.2.3-249.cea2264" + build_time = "2025-02-24_18:50:55_UTC" token_api_enabled = true server { diff --git a/tracing/tracer.go b/tracing/tracer.go index c3bceca..185acdb 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -108,10 +108,11 @@ func (t *XpcTracer) Region() string { } func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { - codeGitCommit := strings.SplitN(conf.GetString("webconfig.code_git_commit"), "-", 2) - xpcTracer.appName = codeGitCommit[0] - if len(codeGitCommit) > 1 { - xpcTracer.appVersion = codeGitCommit[1] + codeGitCommit := conf.GetString("webconfig.code_git_commit") + cgcData := strings.SplitN(codeGitCommit, "-", 2) + xpcTracer.appName = cgcData[0] + if len(cgcData) > 1 { + xpcTracer.appVersion = cgcData[1] } // Env vars @@ -129,7 +130,7 @@ func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { xpcTracer.versionForTracing = os.Getenv("OTEL_VERSION") if xpcTracer.versionForTracing == "" { - xpcTracer.versionForTracing = xpcTracer.appVersion + xpcTracer.versionForTracing = codeGitCommit } if xpcTracer.versionForTracing == "" { xpcTracer.versionForTracing = "v1" From d09d5b6e0cacd40eb05152dd43f4a4e733c4dfc5 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Tue, 11 Mar 2025 13:59:10 -0700 Subject: [PATCH 163/215] Cherrypick of adding appname to version --- tracing/otel.go | 2 ++ tracing/tracer.go | 35 +++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tracing/otel.go b/tracing/otel.go index 6617224..8e47be3 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -117,6 +117,7 @@ func otelStdoutTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, er resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceVersionKey.String(xpcTracer.versionForTracing), semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), ), ), @@ -143,6 +144,7 @@ func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, erro resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceVersionKey.String(xpcTracer.versionForTracing), semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), ), ), diff --git a/tracing/tracer.go b/tracing/tracer.go index a65455a..185acdb 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -42,12 +42,12 @@ type XpcTracer struct { moracideTagPrefix string // Special request header for moracide expts e.g. canary deployments // internal vars used by Otel - appEnv string // set this to dev for red, staging for yellow and prod for green - appName string - appVersion string - appSHA string // unused - region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute - siteColor string // red/yellow/green, unused, use it as a span attribute + appEnv string // set this to dev for red, staging for yellow and prod for green + appName string + appVersion string + region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute + siteColor string // red/yellow/green, unused, use it as a span attribute + versionForTracing string // internal otel vars otelEndpoint string @@ -108,13 +108,11 @@ func (t *XpcTracer) Region() string { } func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { - codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") - xpcTracer.appName = codeGitCommit[0] - if len(codeGitCommit) > 1 { - xpcTracer.appVersion = codeGitCommit[1] - } - if len(codeGitCommit) > 2 { - xpcTracer.appSHA = codeGitCommit[2] + codeGitCommit := conf.GetString("webconfig.code_git_commit") + cgcData := strings.SplitN(codeGitCommit, "-", 2) + xpcTracer.appName = cgcData[0] + if len(cgcData) > 1 { + xpcTracer.appVersion = cgcData[1] } // Env vars @@ -129,7 +127,16 @@ func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { if xpcTracer.region == "" { xpcTracer.region = os.Getenv("site_region_name") } - log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) + + xpcTracer.versionForTracing = os.Getenv("OTEL_VERSION") + if xpcTracer.versionForTracing == "" { + xpcTracer.versionForTracing = codeGitCommit + } + if xpcTracer.versionForTracing == "" { + xpcTracer.versionForTracing = "v1" + } + + log.Debugf("site_color = %s, env = %s, region = %s version = %s", siteColor, xpcTracer.appEnv, xpcTracer.region, xpcTracer.versionForTracing) } func OtelSetSpan(fields log.Fields, tag string) { From 9a2e876ddb569f374016df96327b9647bbadf153 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 18 Mar 2025 23:38:12 -0700 Subject: [PATCH 164/215] support subdocs without bitmaps --- common/document.go | 5 +- common/document_test.go | 57 ++++++++++- common/server_config.go | 29 +++++- http/bitmap_filter_test.go | 195 +++++++++++++++++++++++++++++++++++++ http/multipart.go | 12 ++- http/webconfig_server.go | 11 +++ tracing/otel.go | 2 - tracing/tracer.go | 35 +++---- 8 files changed, 316 insertions(+), 30 deletions(-) create mode 100644 http/bitmap_filter_test.go diff --git a/common/document.go b/common/document.go index e216c9a..506dc93 100644 --- a/common/document.go +++ b/common/document.go @@ -220,7 +220,7 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { return BuildPayloadAsHttp(http.StatusOK, header, bbytes), nil } -func (d *Document) FilterByBitmap() *Document { +func (d *Document) FilterByBitmap(alwaysTrueSubdocIds ...string) *Document { rootdoc := d.GetRootDocument() if rootdoc == nil { return d @@ -228,6 +228,9 @@ func (d *Document) FilterByBitmap() *Document { newdoc := NewDocument(rootdoc) supportedMap := GetSupportedMap(rootdoc.Bitmap) + for _, sid := range alwaysTrueSubdocIds { + supportedMap[sid] = true + } for subdocId, subDocument := range d.docmap { if supportedMap[subdocId] { diff --git a/common/document_test.go b/common/document_test.go index bb222d9..9a63919 100644 --- a/common/document_test.go +++ b/common/document_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -69,3 +69,58 @@ func TestDocument(t *testing.T) { assert.Assert(t, filteredDocument != nil) assert.Equal(t, len(filteredDocument.Items()), 4) } + +// write a test for FilterByBitmap +func TestFilterByBitmap(t *testing.T) { + document := NewDocument(nil) + assert.Equal(t, len(document.RootVersion()), 0) + + bitmap := 32479 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + document = NewDocument(rootdoc) + + subdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "remotedebugger", + } + mparts := []Multipart{} + versionMap := make(map[string]string) + for _, subdocId := range subdocIds { + blen := rand.Intn(10) + 10 + bbytes := make([]byte, blen) + crand.Read(bbytes) + version := strconv.Itoa(int(time.Now().Unix())) + mpart := Multipart{ + Bytes: bbytes, + Version: version, + Name: subdocId, + State: Deployed, + } + mparts = append(mparts, mpart) + versionMap[subdocId] = version + } + + document.SetSubDocuments(mparts) + assert.Equal(t, len(document.Items()), len(subdocIds)) + + filteredDocument := document.FilterByBitmap() + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) + + exemptedSubdocIds := []string{} + filteredDocument = document.FilterByBitmap(exemptedSubdocIds...) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) + + filteredDocument = document.FilterByBitmap("remotedebugger") + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 5) +} diff --git a/common/server_config.go b/common/server_config.go index 0873776..2a0296a 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "fmt" "os" + "strings" "github.com/go-akka/configuration" ) @@ -65,6 +66,32 @@ func (c *ServerConfig) KafkaClusterNames() []string { return clustersNode.GetKeys() } +// NOTE that "bad" entries (keys without values, ill-formatted) can still be added +// hence no parsing error +func (c *ServerConfig) AddConfig(args ...string) { + lines := []string{ + string(c.configBytes), + } + lines = append(lines, args...) + ss := strings.Join(lines, "\n") + c.Config = configuration.ParseString(ss) + c.configBytes = []byte(ss) +} + +// copy the config and add extra items +func (c *ServerConfig) Copy(args ...string) *ServerConfig { + lines := []string{ + string(c.configBytes), + } + lines = append(lines, args...) + ss := strings.Join(lines, "\n") + conf := configuration.ParseString(ss) + return &ServerConfig{ + Config: conf, + configBytes: []byte(ss), + } +} + func GetTestConfigFile() (string, error) { testConfigFile := os.Getenv("TEST_CONFIG_FILE") if len(testConfigFile) > 0 { diff --git a/http/bitmap_filter_test.go b/http/bitmap_filter_test.go new file mode 100644 index 0000000..88cb46a --- /dev/null +++ b/http/bitmap_filter_test.go @@ -0,0 +1,195 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "strconv" + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "gotest.tools/assert" +) + +func TestFilterOutputByBitmap(t *testing.T) { + tsc1 := sc.Copy("webconfig.filter_output_by_bitmap_enabled=true") + server := NewWebconfigServer(tsc1, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + now := time.Now() + + // ==== step 1 use epochNow as version and set a future expiry ==== + // post + subdocId := "remotedebugger" + remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + remotedebuggerBytes := util.RandomBytes(100, 150) + req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + + // prepare the version header + reqHeaderVersion := strconv.Itoa(int(now.Unix())) + req.Header.Set(common.HeaderSubdocumentVersion, reqHeaderVersion) + + // prepare a future expiry header + futureT := now.AddDate(0, 0, 2) + reqHeaderExpiry := strconv.Itoa(int(futureT.UnixNano() / 1000000)) + req.Header.Set(common.HeaderSubdocumentExpiry, reqHeaderExpiry) + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", remotedebuggerUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + + resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) + assert.Equal(t, reqHeaderVersion, resHeaderVersion) + resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) + assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) + + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, remotedebuggerBytes) + + // check the root doc version + rdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, len(rdoc.Version) > 0) + + // ==== step 2 get document ==== + supportedDocs1 := "16777217" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} + +func TestBitmapFilterExemptSubdocIds(t *testing.T) { + tsc1 := sc.Copy( + "webconfig.filter_output_by_bitmap_enabled=true", + `webconfig.bitmap_filter_exempt_subdoc_ids=["remotedebugger"]`, + ) + server := NewWebconfigServer(tsc1, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + now := time.Now() + + // ==== step 1 use epochNow as version and set a future expiry ==== + // post + subdocId := "remotedebugger" + remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + remotedebuggerBytes := util.RandomBytes(100, 150) + req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + + // prepare the version header + reqHeaderVersion := strconv.Itoa(int(now.Unix())) + req.Header.Set(common.HeaderSubdocumentVersion, reqHeaderVersion) + + // prepare a future expiry header + futureT := now.AddDate(0, 0, 2) + reqHeaderExpiry := strconv.Itoa(int(futureT.UnixNano() / 1000000)) + req.Header.Set(common.HeaderSubdocumentExpiry, reqHeaderExpiry) + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", remotedebuggerUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + + resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) + assert.Equal(t, reqHeaderVersion, resHeaderVersion) + resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) + assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) + + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, remotedebuggerBytes) + + // check the root doc version + rdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, len(rdoc.Version) > 0) + + // ==== step 2 get document ==== + supportedDocs1 := "16777217" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + + mpart, ok := mpartMap["remotedebugger"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, remotedebuggerBytes) +} diff --git a/http/multipart.go b/http/multipart.go index 4f27811..450509e 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -176,7 +176,11 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } if s.FilterOutputByBitmapEnabled() { - document = document.FilterByBitmap() + document = document.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) + } + + if document.Length() == 0 { + return http.StatusNotFound, respHeader, nil, nil } respBytes, err := document.Bytes() @@ -223,11 +227,11 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } } - // 3333 + if s.FilterOutputByBitmapEnabled() { - document = document.FilterByBitmap() + document = document.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) } - // 44444 + respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 088b69a..4775d1b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -120,6 +120,7 @@ type WebconfigServer struct { validSubdocIdMap map[string]int filterOutputByBitmapEnabled bool defaultEmptyProfileEnabled bool + bitmapFilterExemptSubdocIds []string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -293,6 +294,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer filterOutputByBitmapEnabled := conf.GetBoolean("webconfig.filter_output_by_bitmap_enabled") defaultEmptyProfileEnabled := conf.GetBoolean("webconfig.default_empty_profile_enabled") + bitmapFilterExemptSubdocIds := conf.GetStringList("webconfig.bitmap_filter_exempt_subdoc_ids") ws := &WebconfigServer{ Server: &http.Server{ @@ -332,6 +334,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer XpcTracer: xpcTracer, filterOutputByBitmapEnabled: filterOutputByBitmapEnabled, defaultEmptyProfileEnabled: defaultEmptyProfileEnabled, + bitmapFilterExemptSubdocIds: bitmapFilterExemptSubdocIds, } return ws @@ -709,6 +712,14 @@ func (s *WebconfigServer) SetDefaultEmptyProfileEnabled(enabled bool) { s.defaultEmptyProfileEnabled = enabled } +func (s *WebconfigServer) BitmapFilterExemptSubdocIds() []string { + return s.bitmapFilterExemptSubdocIds +} + +func (s *WebconfigServer) SetBitmapFilterExemptSubdocIds(x []string) { + s.bitmapFilterExemptSubdocIds = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/tracing/otel.go b/tracing/otel.go index 8e47be3..6617224 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -117,7 +117,6 @@ func otelStdoutTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, er resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(xpcTracer.appName), - semconv.ServiceVersionKey.String(xpcTracer.versionForTracing), semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), ), ), @@ -144,7 +143,6 @@ func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, erro resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(xpcTracer.appName), - semconv.ServiceVersionKey.String(xpcTracer.versionForTracing), semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), ), ), diff --git a/tracing/tracer.go b/tracing/tracer.go index 185acdb..a65455a 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -42,12 +42,12 @@ type XpcTracer struct { moracideTagPrefix string // Special request header for moracide expts e.g. canary deployments // internal vars used by Otel - appEnv string // set this to dev for red, staging for yellow and prod for green - appName string - appVersion string - region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute - siteColor string // red/yellow/green, unused, use it as a span attribute - versionForTracing string + appEnv string // set this to dev for red, staging for yellow and prod for green + appName string + appVersion string + appSHA string // unused + region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute + siteColor string // red/yellow/green, unused, use it as a span attribute // internal otel vars otelEndpoint string @@ -108,11 +108,13 @@ func (t *XpcTracer) Region() string { } func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { - codeGitCommit := conf.GetString("webconfig.code_git_commit") - cgcData := strings.SplitN(codeGitCommit, "-", 2) - xpcTracer.appName = cgcData[0] - if len(cgcData) > 1 { - xpcTracer.appVersion = cgcData[1] + codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") + xpcTracer.appName = codeGitCommit[0] + if len(codeGitCommit) > 1 { + xpcTracer.appVersion = codeGitCommit[1] + } + if len(codeGitCommit) > 2 { + xpcTracer.appSHA = codeGitCommit[2] } // Env vars @@ -127,16 +129,7 @@ func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { if xpcTracer.region == "" { xpcTracer.region = os.Getenv("site_region_name") } - - xpcTracer.versionForTracing = os.Getenv("OTEL_VERSION") - if xpcTracer.versionForTracing == "" { - xpcTracer.versionForTracing = codeGitCommit - } - if xpcTracer.versionForTracing == "" { - xpcTracer.versionForTracing = "v1" - } - - log.Debugf("site_color = %s, env = %s, region = %s version = %s", siteColor, xpcTracer.appEnv, xpcTracer.region, xpcTracer.versionForTracing) + log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) } func OtelSetSpan(fields log.Fields, tag string) { From de90cdbba8387873b754082b667c39f8492ed20c Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 19 Mar 2025 11:54:12 -0700 Subject: [PATCH 165/215] skip logging pending notifications --- common/error.go | 1 + db/service.go | 2 +- kafka/consumer.go | 9 +++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/common/error.go b/common/error.go index 9a1f4d1..880ea08 100644 --- a/common/error.go +++ b/common/error.go @@ -35,6 +35,7 @@ var ( ErrRootDocumentLocked = fmt.Errorf("root document is locked") ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") ErrLowTrust = fmt.Errorf("token trust is lower than threshold") + ErrPending = fmt.Errorf("application_status pending") ) type Http400Error struct { diff --git a/db/service.go b/db/service.go index 66be7e9..c519b41 100644 --- a/db/service.go +++ b/db/service.go @@ -441,7 +441,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return updatedSubdocIds, nil + return updatedSubdocIds, common.NewError(common.ErrPending) } targetGroupId := *m.Namespace diff --git a/kafka/consumer.go b/kafka/consumer.go index 40e8ec1..18105c2 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -20,6 +20,7 @@ package kafka import ( "encoding/base64" "encoding/json" + "errors" "fmt" "net/http" "strings" @@ -227,15 +228,19 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram fields["event_name"] = eventName fields["rpt"] = rptHeaderValue + forwardMessage := false if err != nil { if c.IsDbNotFound(err) { - log.WithFields(fields).Trace("db not found") + log.WithFields(fields).Info("db not found") + } else if errors.Is(err, common.ErrPending) { + log.WithFields(fields).Trace("pending") } else { fields["error"] = err.Error() fields["kafka_message"] = base64.StdEncoding.EncodeToString(message.Value) log.WithFields(fields).Error("errors") } } else { + forwardMessage = true log.WithFields(fields).Info(logMessage) } @@ -256,7 +261,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram metrics.CountKafkaEvents(eventName, status, message.Partition) } - if c.KafkaProducerEnabled() && m != nil { + if c.KafkaProducerEnabled() && m != nil && forwardMessage { c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && len(updatedSubdocIds) > 0 { From a1a2e58d0821373119971035881f8275a81abc07 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 20 Mar 2025 23:33:49 -0700 Subject: [PATCH 166/215] Parse the bitmap of the channelplan subdoc --- common/bitmap.go | 2 ++ common/firmware_bitmap_test.go | 43 +++++++++++++++++++++++++++++++++- config/sample_webconfig.conf | 2 ++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/common/bitmap.go b/common/bitmap.go index 89492cc..e55b782 100644 --- a/common/bitmap.go +++ b/common/bitmap.go @@ -62,6 +62,7 @@ var ( {5, 38}, // mwoconfigs {6, 39}, // interference {7, 34}, // wifimotionsettings + {8, 41}, // channelplan }, 7: { {1, 14}, @@ -148,6 +149,7 @@ var ( "mwoconfigs": 38, "interference": 39, "webui": 40, + "channelplan": 41, } ) diff --git a/common/firmware_bitmap_test.go b/common/firmware_bitmap_test.go index 85c4e0b..326237a 100644 --- a/common/firmware_bitmap_test.go +++ b/common/firmware_bitmap_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -587,3 +587,44 @@ func TestParseSupportedDocsHeaderWebui(t *testing.T) { supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } + +func TestParseSupportedDocsHeaderChannelplan(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663487,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + "channelplan", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index dae7dd0..33f3bf0 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -275,4 +275,6 @@ webconfig { valid_subdoc_ids = [] filter_output_by_bitmap_enabled = false + + bitmap_filter_exempt_subdoc_ids = [] } From 4f573d548289591a65ed4ce3043e34cbad98e9ad Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 19 Mar 2025 11:54:12 -0700 Subject: [PATCH 167/215] skip logging pending notifications --- common/error.go | 1 + db/service.go | 2 +- kafka/consumer.go | 9 +++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/common/error.go b/common/error.go index 9a1f4d1..880ea08 100644 --- a/common/error.go +++ b/common/error.go @@ -35,6 +35,7 @@ var ( ErrRootDocumentLocked = fmt.Errorf("root document is locked") ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") ErrLowTrust = fmt.Errorf("token trust is lower than threshold") + ErrPending = fmt.Errorf("application_status pending") ) type Http400Error struct { diff --git a/db/service.go b/db/service.go index 66be7e9..c519b41 100644 --- a/db/service.go +++ b/db/service.go @@ -441,7 +441,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return updatedSubdocIds, nil + return updatedSubdocIds, common.NewError(common.ErrPending) } targetGroupId := *m.Namespace diff --git a/kafka/consumer.go b/kafka/consumer.go index 40e8ec1..18105c2 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -20,6 +20,7 @@ package kafka import ( "encoding/base64" "encoding/json" + "errors" "fmt" "net/http" "strings" @@ -227,15 +228,19 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram fields["event_name"] = eventName fields["rpt"] = rptHeaderValue + forwardMessage := false if err != nil { if c.IsDbNotFound(err) { - log.WithFields(fields).Trace("db not found") + log.WithFields(fields).Info("db not found") + } else if errors.Is(err, common.ErrPending) { + log.WithFields(fields).Trace("pending") } else { fields["error"] = err.Error() fields["kafka_message"] = base64.StdEncoding.EncodeToString(message.Value) log.WithFields(fields).Error("errors") } } else { + forwardMessage = true log.WithFields(fields).Info(logMessage) } @@ -256,7 +261,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram metrics.CountKafkaEvents(eventName, status, message.Partition) } - if c.KafkaProducerEnabled() && m != nil { + if c.KafkaProducerEnabled() && m != nil && forwardMessage { c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && len(updatedSubdocIds) > 0 { From d6808b927158eb3aa26fb0ec724f54e21578158d Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 20 Mar 2025 23:33:49 -0700 Subject: [PATCH 168/215] Parse the bitmap of the channelplan subdoc --- common/bitmap.go | 2 ++ common/firmware_bitmap_test.go | 43 +++++++++++++++++++++++++++++++++- config/sample_webconfig.conf | 2 ++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/common/bitmap.go b/common/bitmap.go index 89492cc..e55b782 100644 --- a/common/bitmap.go +++ b/common/bitmap.go @@ -62,6 +62,7 @@ var ( {5, 38}, // mwoconfigs {6, 39}, // interference {7, 34}, // wifimotionsettings + {8, 41}, // channelplan }, 7: { {1, 14}, @@ -148,6 +149,7 @@ var ( "mwoconfigs": 38, "interference": 39, "webui": 40, + "channelplan": 41, } ) diff --git a/common/firmware_bitmap_test.go b/common/firmware_bitmap_test.go index 85c4e0b..326237a 100644 --- a/common/firmware_bitmap_test.go +++ b/common/firmware_bitmap_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -587,3 +587,44 @@ func TestParseSupportedDocsHeaderWebui(t *testing.T) { supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } + +func TestParseSupportedDocsHeaderChannelplan(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663487,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + "channelplan", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 779c0c7..99ae18e 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -275,4 +275,6 @@ webconfig { valid_subdoc_ids = [] filter_output_by_bitmap_enabled = false + + bitmap_filter_exempt_subdoc_ids = [] } From db04afea5afd042c230975982faaceebb1186163 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 25 Mar 2025 16:18:39 -0700 Subject: [PATCH 169/215] fix a bug that 200 was returned when no subdoc was sent --- http/multipart.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/http/multipart.go b/http/multipart.go index 450509e..0953e14 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -254,6 +254,11 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } + // 304 + if document.Length() == 0 { + respStatus = http.StatusNotModified + } + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return respStatus, respHeader, respBytes, nil @@ -338,7 +343,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } if s.FilterOutputByBitmapEnabled() { - finalFilteredDocument = finalFilteredDocument.FilterByBitmap() + finalFilteredDocument = finalFilteredDocument.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) } // 304 From 39cbd577c1697ede18840519f9b5b2cc2c0e3985 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 18 Mar 2025 23:38:12 -0700 Subject: [PATCH 170/215] cherrypick changes to support no-bitmap subdocs --- common/document.go | 5 +- common/document_test.go | 57 ++++++++++- common/server_config.go | 29 +++++- http/bitmap_filter_test.go | 195 +++++++++++++++++++++++++++++++++++++ http/multipart.go | 12 ++- http/webconfig_server.go | 12 +++ tracing/otel.go | 2 - tracing/tracer.go | 35 +++---- 8 files changed, 317 insertions(+), 30 deletions(-) create mode 100644 http/bitmap_filter_test.go diff --git a/common/document.go b/common/document.go index e216c9a..506dc93 100644 --- a/common/document.go +++ b/common/document.go @@ -220,7 +220,7 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { return BuildPayloadAsHttp(http.StatusOK, header, bbytes), nil } -func (d *Document) FilterByBitmap() *Document { +func (d *Document) FilterByBitmap(alwaysTrueSubdocIds ...string) *Document { rootdoc := d.GetRootDocument() if rootdoc == nil { return d @@ -228,6 +228,9 @@ func (d *Document) FilterByBitmap() *Document { newdoc := NewDocument(rootdoc) supportedMap := GetSupportedMap(rootdoc.Bitmap) + for _, sid := range alwaysTrueSubdocIds { + supportedMap[sid] = true + } for subdocId, subDocument := range d.docmap { if supportedMap[subdocId] { diff --git a/common/document_test.go b/common/document_test.go index bb222d9..9a63919 100644 --- a/common/document_test.go +++ b/common/document_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -69,3 +69,58 @@ func TestDocument(t *testing.T) { assert.Assert(t, filteredDocument != nil) assert.Equal(t, len(filteredDocument.Items()), 4) } + +// write a test for FilterByBitmap +func TestFilterByBitmap(t *testing.T) { + document := NewDocument(nil) + assert.Equal(t, len(document.RootVersion()), 0) + + bitmap := 32479 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + document = NewDocument(rootdoc) + + subdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "remotedebugger", + } + mparts := []Multipart{} + versionMap := make(map[string]string) + for _, subdocId := range subdocIds { + blen := rand.Intn(10) + 10 + bbytes := make([]byte, blen) + crand.Read(bbytes) + version := strconv.Itoa(int(time.Now().Unix())) + mpart := Multipart{ + Bytes: bbytes, + Version: version, + Name: subdocId, + State: Deployed, + } + mparts = append(mparts, mpart) + versionMap[subdocId] = version + } + + document.SetSubDocuments(mparts) + assert.Equal(t, len(document.Items()), len(subdocIds)) + + filteredDocument := document.FilterByBitmap() + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) + + exemptedSubdocIds := []string{} + filteredDocument = document.FilterByBitmap(exemptedSubdocIds...) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) + + filteredDocument = document.FilterByBitmap("remotedebugger") + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 5) +} diff --git a/common/server_config.go b/common/server_config.go index 0873776..2a0296a 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "fmt" "os" + "strings" "github.com/go-akka/configuration" ) @@ -65,6 +66,32 @@ func (c *ServerConfig) KafkaClusterNames() []string { return clustersNode.GetKeys() } +// NOTE that "bad" entries (keys without values, ill-formatted) can still be added +// hence no parsing error +func (c *ServerConfig) AddConfig(args ...string) { + lines := []string{ + string(c.configBytes), + } + lines = append(lines, args...) + ss := strings.Join(lines, "\n") + c.Config = configuration.ParseString(ss) + c.configBytes = []byte(ss) +} + +// copy the config and add extra items +func (c *ServerConfig) Copy(args ...string) *ServerConfig { + lines := []string{ + string(c.configBytes), + } + lines = append(lines, args...) + ss := strings.Join(lines, "\n") + conf := configuration.ParseString(ss) + return &ServerConfig{ + Config: conf, + configBytes: []byte(ss), + } +} + func GetTestConfigFile() (string, error) { testConfigFile := os.Getenv("TEST_CONFIG_FILE") if len(testConfigFile) > 0 { diff --git a/http/bitmap_filter_test.go b/http/bitmap_filter_test.go new file mode 100644 index 0000000..88cb46a --- /dev/null +++ b/http/bitmap_filter_test.go @@ -0,0 +1,195 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "strconv" + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "gotest.tools/assert" +) + +func TestFilterOutputByBitmap(t *testing.T) { + tsc1 := sc.Copy("webconfig.filter_output_by_bitmap_enabled=true") + server := NewWebconfigServer(tsc1, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + now := time.Now() + + // ==== step 1 use epochNow as version and set a future expiry ==== + // post + subdocId := "remotedebugger" + remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + remotedebuggerBytes := util.RandomBytes(100, 150) + req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + + // prepare the version header + reqHeaderVersion := strconv.Itoa(int(now.Unix())) + req.Header.Set(common.HeaderSubdocumentVersion, reqHeaderVersion) + + // prepare a future expiry header + futureT := now.AddDate(0, 0, 2) + reqHeaderExpiry := strconv.Itoa(int(futureT.UnixNano() / 1000000)) + req.Header.Set(common.HeaderSubdocumentExpiry, reqHeaderExpiry) + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", remotedebuggerUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + + resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) + assert.Equal(t, reqHeaderVersion, resHeaderVersion) + resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) + assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) + + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, remotedebuggerBytes) + + // check the root doc version + rdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, len(rdoc.Version) > 0) + + // ==== step 2 get document ==== + supportedDocs1 := "16777217" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} + +func TestBitmapFilterExemptSubdocIds(t *testing.T) { + tsc1 := sc.Copy( + "webconfig.filter_output_by_bitmap_enabled=true", + `webconfig.bitmap_filter_exempt_subdoc_ids=["remotedebugger"]`, + ) + server := NewWebconfigServer(tsc1, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + now := time.Now() + + // ==== step 1 use epochNow as version and set a future expiry ==== + // post + subdocId := "remotedebugger" + remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + remotedebuggerBytes := util.RandomBytes(100, 150) + req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + + // prepare the version header + reqHeaderVersion := strconv.Itoa(int(now.Unix())) + req.Header.Set(common.HeaderSubdocumentVersion, reqHeaderVersion) + + // prepare a future expiry header + futureT := now.AddDate(0, 0, 2) + reqHeaderExpiry := strconv.Itoa(int(futureT.UnixNano() / 1000000)) + req.Header.Set(common.HeaderSubdocumentExpiry, reqHeaderExpiry) + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", remotedebuggerUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + + resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) + assert.Equal(t, reqHeaderVersion, resHeaderVersion) + resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) + assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) + + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, remotedebuggerBytes) + + // check the root doc version + rdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, len(rdoc.Version) > 0) + + // ==== step 2 get document ==== + supportedDocs1 := "16777217" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + + mpart, ok := mpartMap["remotedebugger"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, remotedebuggerBytes) +} diff --git a/http/multipart.go b/http/multipart.go index 4f27811..450509e 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -176,7 +176,11 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } if s.FilterOutputByBitmapEnabled() { - document = document.FilterByBitmap() + document = document.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) + } + + if document.Length() == 0 { + return http.StatusNotFound, respHeader, nil, nil } respBytes, err := document.Bytes() @@ -223,11 +227,11 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } } - // 3333 + if s.FilterOutputByBitmapEnabled() { - document = document.FilterByBitmap() + document = document.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) } - // 44444 + respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 9073242..170d947 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -119,6 +119,8 @@ type WebconfigServer struct { minTrust int validSubdocIdMap map[string]int filterOutputByBitmapEnabled bool + defaultEmptyProfileEnabled bool + bitmapFilterExemptSubdocIds []string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -291,6 +293,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } filterOutputByBitmapEnabled := conf.GetBoolean("webconfig.filter_output_by_bitmap_enabled") + bitmapFilterExemptSubdocIds := conf.GetStringList("webconfig.bitmap_filter_exempt_subdoc_ids") ws := &WebconfigServer{ Server: &http.Server{ @@ -329,6 +332,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validSubdocIdMap: validSubdocIdMap, XpcTracer: xpcTracer, filterOutputByBitmapEnabled: filterOutputByBitmapEnabled, + bitmapFilterExemptSubdocIds: bitmapFilterExemptSubdocIds, } return ws @@ -698,6 +702,14 @@ func (s *WebconfigServer) SetFilterOutputByBitmapEnabled(enabled bool) { s.filterOutputByBitmapEnabled = enabled } +func (s *WebconfigServer) BitmapFilterExemptSubdocIds() []string { + return s.bitmapFilterExemptSubdocIds +} + +func (s *WebconfigServer) SetBitmapFilterExemptSubdocIds(x []string) { + s.bitmapFilterExemptSubdocIds = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/tracing/otel.go b/tracing/otel.go index 8e47be3..6617224 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -117,7 +117,6 @@ func otelStdoutTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, er resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(xpcTracer.appName), - semconv.ServiceVersionKey.String(xpcTracer.versionForTracing), semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), ), ), @@ -144,7 +143,6 @@ func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, erro resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(xpcTracer.appName), - semconv.ServiceVersionKey.String(xpcTracer.versionForTracing), semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), ), ), diff --git a/tracing/tracer.go b/tracing/tracer.go index 185acdb..a65455a 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -42,12 +42,12 @@ type XpcTracer struct { moracideTagPrefix string // Special request header for moracide expts e.g. canary deployments // internal vars used by Otel - appEnv string // set this to dev for red, staging for yellow and prod for green - appName string - appVersion string - region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute - siteColor string // red/yellow/green, unused, use it as a span attribute - versionForTracing string + appEnv string // set this to dev for red, staging for yellow and prod for green + appName string + appVersion string + appSHA string // unused + region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute + siteColor string // red/yellow/green, unused, use it as a span attribute // internal otel vars otelEndpoint string @@ -108,11 +108,13 @@ func (t *XpcTracer) Region() string { } func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { - codeGitCommit := conf.GetString("webconfig.code_git_commit") - cgcData := strings.SplitN(codeGitCommit, "-", 2) - xpcTracer.appName = cgcData[0] - if len(cgcData) > 1 { - xpcTracer.appVersion = cgcData[1] + codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") + xpcTracer.appName = codeGitCommit[0] + if len(codeGitCommit) > 1 { + xpcTracer.appVersion = codeGitCommit[1] + } + if len(codeGitCommit) > 2 { + xpcTracer.appSHA = codeGitCommit[2] } // Env vars @@ -127,16 +129,7 @@ func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { if xpcTracer.region == "" { xpcTracer.region = os.Getenv("site_region_name") } - - xpcTracer.versionForTracing = os.Getenv("OTEL_VERSION") - if xpcTracer.versionForTracing == "" { - xpcTracer.versionForTracing = codeGitCommit - } - if xpcTracer.versionForTracing == "" { - xpcTracer.versionForTracing = "v1" - } - - log.Debugf("site_color = %s, env = %s, region = %s version = %s", siteColor, xpcTracer.appEnv, xpcTracer.region, xpcTracer.versionForTracing) + log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) } func OtelSetSpan(fields log.Fields, tag string) { From 1bc05f85b5dc4820a9d89f01a5d76360fd03a4d8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 24 Apr 2025 11:28:05 -0700 Subject: [PATCH 171/215] fix a bug that 304 was returned when GET was without x-supported-docs header --- http/multipart.go | 8 ++- http/upstream_test.go | 137 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/http/multipart.go b/http/multipart.go index 0953e14..584eb2c 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -323,8 +323,14 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) + var bitmap int + if newRootDocument.Bitmap != 0 { + bitmap = newRootDocument.Bitmap + } else if oldRootDocument.Bitmap != 0 { + bitmap = oldRootDocument.Bitmap + } // filter by versionMap and filter by blockedIds - finalRootDocument := common.NewRootDocument(newRootDocument.Bitmap, "", "", "", "", upstreamRespEtag, "") + finalRootDocument := common.NewRootDocument(bitmap, "", "", "", "", upstreamRespEtag, "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) diff --git a/http/upstream_test.go b/http/upstream_test.go index f8c1bf3..0cb0b4a 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -1661,3 +1661,140 @@ func TestUpstreamBackfill(t *testing.T) { assert.Assert(t, len(sd.Payload()) > 0) } } + +func TestUpstreamNoBitmapHeader(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 GET /config to create root document meta ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + + supportedDocs := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion := "CGM4331COM_4.11p7s1_PROD_sey" + modelName := "CGM4331COM" + partner := "comcast" + schemaVersion := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partner) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 2 POST group lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 3 POST group wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 4 GET /config ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partner) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== step 5 GET /config but with header changes with mock ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.WriteHeader(http.StatusOK) + if rbytes, err := io.ReadAll(r.Body); err == nil { + _, err := w.Write(rbytes) + assert.NilError(t, err) + } + })) + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + server.SetUpstreamEnabled(true) + server.SetFilterOutputByBitmapEnabled(true) + + // ==== step 6 GET /config with no supported-docs/bitmap headers ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "0") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) +} From 8789682de9d54934e17ba6b4bcd4b6ac03d5a4a2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 24 Apr 2025 11:28:05 -0700 Subject: [PATCH 172/215] fix a bug that 304 was returned when GET was without x-supported-docs header --- http/multipart.go | 8 ++- http/upstream_test.go | 137 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/http/multipart.go b/http/multipart.go index 450509e..32b818d 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -318,8 +318,14 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) + var bitmap int + if newRootDocument.Bitmap != 0 { + bitmap = newRootDocument.Bitmap + } else if oldRootDocument.Bitmap != 0 { + bitmap = oldRootDocument.Bitmap + } // filter by versionMap and filter by blockedIds - finalRootDocument := common.NewRootDocument(newRootDocument.Bitmap, "", "", "", "", upstreamRespEtag, "") + finalRootDocument := common.NewRootDocument(bitmap, "", "", "", "", upstreamRespEtag, "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) diff --git a/http/upstream_test.go b/http/upstream_test.go index f8c1bf3..0cb0b4a 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -1661,3 +1661,140 @@ func TestUpstreamBackfill(t *testing.T) { assert.Assert(t, len(sd.Payload()) > 0) } } + +func TestUpstreamNoBitmapHeader(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 GET /config to create root document meta ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + + supportedDocs := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion := "CGM4331COM_4.11p7s1_PROD_sey" + modelName := "CGM4331COM" + partner := "comcast" + schemaVersion := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partner) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 2 POST group lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 3 POST group wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 4 GET /config ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partner) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== step 5 GET /config but with header changes with mock ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.WriteHeader(http.StatusOK) + if rbytes, err := io.ReadAll(r.Body); err == nil { + _, err := w.Write(rbytes) + assert.NilError(t, err) + } + })) + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + server.SetUpstreamEnabled(true) + server.SetFilterOutputByBitmapEnabled(true) + + // ==== step 6 GET /config with no supported-docs/bitmap headers ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "0") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) +} From b824a6a50930f41d7d6986ebfa4e8f11abefe11b Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 1 May 2025 20:06:09 -0700 Subject: [PATCH 173/215] update the dbinit.cql script for new columns --- dbinit.cql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbinit.cql b/dbinit.cql index be8d88b..62ec793 100644 --- a/dbinit.cql +++ b/dbinit.cql @@ -20,7 +20,7 @@ CREATE KEYSPACE IF NOT EXISTS webconfig WITH replication = {'class': 'SimpleStra USE webconfig; -CREATE TABLE root_document ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, model_name text, partner_id text, query_params text, schema_version text, version text); +CREATE TABLE root_document (cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, locked_till timestamp, model_name text, partner_id text, route text, query_params text, schema_version text, version text); -CREATE TABLE xpc_group_config ( cpe_mac text, group_id text, error_code int, error_details text, expiry timestamp, params text, payload blob, state int, updated_time timestamp, version text, PRIMARY KEY (cpe_mac, group_id)) WITH CLUSTERING ORDER BY (group_id ASC); +CREATE TABLE xpc_group_config (cpe_mac text, group_id text, error_code int, error_details text, expiry timestamp, params text, payload blob, state int, updated_time timestamp, version text, PRIMARY KEY (cpe_mac, group_id)) WITH CLUSTERING ORDER BY (group_id ASC); From dc6909fcd95ccac5a40666ecabbe7eff3383fb10 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 13 Jul 2025 11:18:41 -0700 Subject: [PATCH 174/215] change the log level for db not found --- kafka/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 18105c2..92619d6 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -231,7 +231,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram forwardMessage := false if err != nil { if c.IsDbNotFound(err) { - log.WithFields(fields).Info("db not found") + log.WithFields(fields).Trace("db not found") } else if errors.Is(err, common.ErrPending) { log.WithFields(fields).Trace("pending") } else { From 494e6a607c186f782e24296d72eda5711fe191fb Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Fri, 25 Jul 2025 09:55:48 -0700 Subject: [PATCH 175/215] Just a more detailed error message --- http/webconfig_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 4775d1b..54ccab8 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -732,7 +732,7 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return nil } } - return fmt.Errorf("invalid partner") + return fmt.Errorf("invalid partner %s", partner) } func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { From 00f1e91cd71fd8e860e54eff5b04e1527cbeb020 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Tue, 26 Aug 2025 16:28:27 -0700 Subject: [PATCH 176/215] Standarize "Request started" and "Request finished" messages --- http/webconfig_server.go | 6 +++--- kafka/consumer.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 54ccab8..44e8bda 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -849,7 +849,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques bbytes, err := io.ReadAll(r.Body) if err != nil { fields["error"] = err - log.WithFields(fields).Error("request starts") + log.WithFields(fields).Error("Request started") return xwriter } xwriter.SetBodyBytes(bbytes) @@ -858,7 +858,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if userAgent != "mget" { tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("request starts") + log.WithFields(tfields).Info("Request started") } xwriter.LogDebug(r, "tracing", fmt.Sprintf("Trace final out_traceparent %s out_traceState %s", xpcTrace.OutTraceparent, xpcTrace.OutTracestate)) @@ -952,7 +952,7 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { } if userAgent != "mget" { tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("request ends") + log.WithFields(tfields).Info("Request Finished") } } diff --git a/kafka/consumer.go b/kafka/consumer.go index 92619d6..5420464 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -211,7 +211,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram switch eventName { case "mqtt-get": m, err = c.handleGetMessage(message.Value, fields) - logMessage = "request ends" + logMessage = "Request Finished" case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") From 1e317bc72ab30cba991201057770cf2684350608 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Tue, 26 Aug 2025 16:37:44 -0700 Subject: [PATCH 177/215] Handle invalid cpemac value of 000000000000 --- http/multipart.go | 15 +++------------ http/webconfig_server.go | 25 +++++++++++++++++-------- util/validator.go | 7 +++++++ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 584eb2c..357d3b9 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -51,19 +51,10 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. // ==== data integrity check ==== params := mux.Vars(r) - mac, ok := params["mac"] - if !ok { - Error(w, http.StatusNotFound, nil) - return - } + mac := params["mac"] mac = strings.ToUpper(mac) - if s.ValidateMacEnabled() { - if !util.ValidateMac(mac) { - err := *common.NewHttp400Error("invalid mac") - Error(w, http.StatusBadRequest, common.NewError(err)) - return - } - } + // Moved validateMac to CpeMiddleware + r.Header.Set(common.HeaderDeviceId, mac) // ==== processing ==== diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 44e8bda..93edc7d 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -393,16 +393,25 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() fields := xw.Audit() + + params := mux.Vars(r) + mac, ok := params["mac"] + if !ok { + Error(xw, http.StatusForbidden, nil) + return + } + mac = strings.ToUpper(mac) + if s.ValidateMacEnabled() { + if !util.ValidateMac(mac) { + err := *common.NewHttp400Error("invalid mac") + Error(w, http.StatusBadRequest, common.NewError(err)) + return + } + } + authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { - params := mux.Vars(r) - mac, ok := params["mac"] - if !ok || len(mac) != 12 { - Error(xw, http.StatusForbidden, nil) - return - } - if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true fields["src_partner"] = partnerId @@ -952,7 +961,7 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { } if userAgent != "mget" { tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("Request Finished") + log.WithFields(tfields).Info("Request finished") } } diff --git a/util/validator.go b/util/validator.go index bbd269e..f8a89f2 100644 --- a/util/validator.go +++ b/util/validator.go @@ -32,10 +32,17 @@ func ValidateMac(mac string) bool { if len(mac) != 12 { return false } + allZeroes := true for _, r := range mac { if r < 48 || r > 70 || (r > 57 && r < 65) { return false } + if r != 48 { + allZeroes = false + } + } + if allZeroes { + return false } return true } From 4a8161b653440e1f0f1628cad3f5bf987b624726 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 11 Nov 2025 16:21:19 -0800 Subject: [PATCH 178/215] Fix a bug that payload was not updated after upstream if the new version matchs the device version --- db/service.go | 11 ++++++++--- http/multipart.go | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/db/service.go b/db/service.go index c519b41..dab1732 100644 --- a/db/service.go +++ b/db/service.go @@ -492,7 +492,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage return updatedSubdocIds, nil } -func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { +func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, deviceVersionMap map[string]string, fields log.Fields) error { var oldState int if oldSubdoc != nil && oldSubdoc.State() != nil { oldState = *oldSubdoc.State() @@ -504,9 +504,14 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old } labels["client"] = "default" - if oldVersion, ok := versionMap[subdocId]; ok { + var oldSubdocVersion string + if oldSubdoc != nil && oldSubdoc.Version() != nil { + oldSubdocVersion = *oldSubdoc.Version() + } + + if deviceVersion, ok := deviceVersionMap[subdocId]; ok { if newSubdoc.Version() != nil { - if oldVersion == *newSubdoc.Version() && oldSubdoc != nil { + if deviceVersion == *newSubdoc.Version() && oldSubdoc != nil && deviceVersion == oldSubdocVersion { return nil } } diff --git a/http/multipart.go b/http/multipart.go index 584eb2c..0f9d103 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -245,7 +245,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if userAgent == "mget" { respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) - return http.StatusOK, respHeader, respBytes, nil + return respStatus, respHeader, respBytes, nil } if !postUpstream { From eca2d6213446c3e7cc7d474dddb4185ad0618d23 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 11 Nov 2025 16:21:19 -0800 Subject: [PATCH 179/215] Fix a bug that payload was not updated after upstream if the new version matchs the device version --- db/service.go | 11 ++++++++--- http/multipart.go | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/db/service.go b/db/service.go index c519b41..dab1732 100644 --- a/db/service.go +++ b/db/service.go @@ -492,7 +492,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage return updatedSubdocIds, nil } -func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { +func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, deviceVersionMap map[string]string, fields log.Fields) error { var oldState int if oldSubdoc != nil && oldSubdoc.State() != nil { oldState = *oldSubdoc.State() @@ -504,9 +504,14 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old } labels["client"] = "default" - if oldVersion, ok := versionMap[subdocId]; ok { + var oldSubdocVersion string + if oldSubdoc != nil && oldSubdoc.Version() != nil { + oldSubdocVersion = *oldSubdoc.Version() + } + + if deviceVersion, ok := deviceVersionMap[subdocId]; ok { if newSubdoc.Version() != nil { - if oldVersion == *newSubdoc.Version() && oldSubdoc != nil { + if deviceVersion == *newSubdoc.Version() && oldSubdoc != nil && deviceVersion == oldSubdocVersion { return nil } } diff --git a/http/multipart.go b/http/multipart.go index 32b818d..75ead5a 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -245,7 +245,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if userAgent == "mget" { respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) - return http.StatusOK, respHeader, respBytes, nil + return respStatus, respHeader, respBytes, nil } if !postUpstream { From 5592a8f2b45237b091888911f363a944a7c24d0d Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Fri, 25 Jul 2025 09:55:48 -0700 Subject: [PATCH 180/215] Just a more detailed error message --- http/webconfig_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 170d947..81a6861 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -722,7 +722,7 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return nil } } - return fmt.Errorf("invalid partner") + return fmt.Errorf("invalid partner %s", partner) } func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { From ef1c1028a38e4dd731aef2049af63966f42142e4 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Tue, 26 Aug 2025 16:28:27 -0700 Subject: [PATCH 181/215] Standarize "Request started" and "Request finished" messages --- http/webconfig_server.go | 6 +++--- kafka/consumer.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 81a6861..76cf309 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -841,7 +841,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques bbytes, err := io.ReadAll(r.Body) if err != nil { fields["error"] = err - log.WithFields(fields).Error("request starts") + log.WithFields(fields).Error("Request started") return xwriter } xwriter.SetBodyBytes(bbytes) @@ -850,7 +850,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if userAgent != "mget" { tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("request starts") + log.WithFields(tfields).Info("Request started") } xwriter.LogDebug(r, "tracing", fmt.Sprintf("Trace final out_traceparent %s out_traceState %s", xpcTrace.OutTraceparent, xpcTrace.OutTracestate)) @@ -931,7 +931,7 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { } if userAgent != "mget" { tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("request ends") + log.WithFields(tfields).Info("Request Finished") } } diff --git a/kafka/consumer.go b/kafka/consumer.go index 18105c2..32b0345 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -211,7 +211,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram switch eventName { case "mqtt-get": m, err = c.handleGetMessage(message.Value, fields) - logMessage = "request ends" + logMessage = "Request Finished" case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") From 0acce1eccd545dea5d459ff505568ead167f0205 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Tue, 26 Aug 2025 16:37:44 -0700 Subject: [PATCH 182/215] Handle invalid cpemac value of 000000000000 --- http/multipart.go | 15 +++------------ http/webconfig_server.go | 25 +++++++++++++++++-------- util/validator.go | 7 +++++++ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 75ead5a..efcb7b7 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -51,19 +51,10 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. // ==== data integrity check ==== params := mux.Vars(r) - mac, ok := params["mac"] - if !ok { - Error(w, http.StatusNotFound, nil) - return - } + mac := params["mac"] mac = strings.ToUpper(mac) - if s.ValidateMacEnabled() { - if !util.ValidateMac(mac) { - err := *common.NewHttp400Error("invalid mac") - Error(w, http.StatusBadRequest, common.NewError(err)) - return - } - } + // Moved validateMac to CpeMiddleware + r.Header.Set(common.HeaderDeviceId, mac) // ==== processing ==== diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 76cf309..dd24fe5 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -391,16 +391,25 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() fields := xw.Audit() + + params := mux.Vars(r) + mac, ok := params["mac"] + if !ok { + Error(xw, http.StatusForbidden, nil) + return + } + mac = strings.ToUpper(mac) + if s.ValidateMacEnabled() { + if !util.ValidateMac(mac) { + err := *common.NewHttp400Error("invalid mac") + Error(w, http.StatusBadRequest, common.NewError(err)) + return + } + } + authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { - params := mux.Vars(r) - mac, ok := params["mac"] - if !ok || len(mac) != 12 { - Error(xw, http.StatusForbidden, nil) - return - } - if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true fields["src_partner"] = partnerId @@ -931,7 +940,7 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { } if userAgent != "mget" { tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("Request Finished") + log.WithFields(tfields).Info("Request finished") } } diff --git a/util/validator.go b/util/validator.go index bbd269e..f8a89f2 100644 --- a/util/validator.go +++ b/util/validator.go @@ -32,10 +32,17 @@ func ValidateMac(mac string) bool { if len(mac) != 12 { return false } + allZeroes := true for _, r := range mac { if r < 48 || r > 70 || (r > 57 && r < 65) { return false } + if r != 48 { + allZeroes = false + } + } + if allZeroes { + return false } return true } From 0a310074ff828f8e37a725b75c1dff38e7f1c3b9 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 2 Dec 2025 19:36:45 -0800 Subject: [PATCH 183/215] fix a bug that bitmap filtering was not applied for NONE/NONE-REBOOT call route --- http/factory_reset_upstream_test.go | 183 ++++++++++++++++++++++++++++ http/multipart.go | 10 +- http/webconfig_server.go | 2 +- util/list.go | 5 +- 4 files changed, 195 insertions(+), 5 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 5fc3c3a..1271836 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -388,3 +388,186 @@ func TestFactoryResetUpstreamAddData(t *testing.T) { assert.NilError(t, err) assert.Equal(t, rootDocument.Bitmap, 32479) } + +func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== group 3 webui ==== + subdocId = "webui" + webuiBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(webuiBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, webuiBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderProductClass, "XB8") + req.Header.Set(common.HeaderFirmwareVersion, "CGM4981COM_8.0p4s1_PROD_sey") + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderModelName, "CGM4981COM") + req.Header.Set(common.HeaderSupportedDocs, "16777439,33554435,50331649,67108865,83886081,100663423,117440513,134217735,201326594,218103809,251658241,268435457,285212673") + req.Header.Set(common.HeaderSchemaVersion, "16777232-1.3,33554433-1.3,33554434-1.3") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + // lanVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + mpart, ok = mparts["webui"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, webuiBytes) + + // ==== setup mock upstream server ==== + fields := make(log.Fields) + mockDoc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + + mockRootDoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + mockDoc.SetRootDocument(mockRootDoc) + + // mockDoc.DeleteSubDocument("wan") + // assert.NilError(t, err) + + mockBytes, err := mockDoc.Bytes() + assert.NilError(t, err) + + db.RefreshRootDocumentVersion(mockDoc) + refreshedRootVersion := mockDoc.RootVersion() + + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.Header().Set(common.HeaderContentLength, strconv.Itoa(len(mockBytes))) + ifNoneMatch := refreshedRootVersion + w.Header().Set(common.HeaderEtag, ifNoneMatch) + w.WriteHeader(http.StatusOK) + w.Write(mockBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(true) + server.SetFilterOutputByBitmapEnabled(true) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderProductClass, "XB8") + req.Header.Set(common.HeaderFirmwareVersion, "CGM4981COM_8.0p4s1_PROD_sey") + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderModelName, "CGM4981COM") + req.Header.Set(common.HeaderSupportedDocs, "16777439,33554435,50331649,67108865,83886081,100663423,117440513,134217735,201326594,218103809,251658241,268435457,285212673") + req.Header.Set(common.HeaderSchemaVersion, "16777232-1.3,33554433-1.3,33554434-1.3") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + + // verify Document is now empty + doc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + assert.Equal(t, doc.Length(), 3) +} diff --git a/http/multipart.go b/http/multipart.go index 4e84a5f..e579760 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -445,9 +445,9 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusInternalServerError, respHeader, oldDocBytes, common.NewError(err) } upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) + finalRootDocument := rootDocument.Clone() + finalRootDocument.Version = upstreamRespEtag - // filter by versionMap and filter by blockedIds - finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) for _, subdocId := range c.BlockedSubdocIds() { @@ -471,6 +471,12 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l if err != nil { return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) } + + // filter by bitmaps and blockedIds + if s.FilterOutputByBitmapEnabled() { + finalDocument = finalDocument.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) + } + finalBytes, err := finalDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 93edc7d..420dcd9 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -405,7 +405,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if !util.ValidateMac(mac) { err := *common.NewHttp400Error("invalid mac") Error(w, http.StatusBadRequest, common.NewError(err)) - return + return } } diff --git a/util/list.go b/util/list.go index f58b2b6..41e7641 100644 --- a/util/list.go +++ b/util/list.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -52,7 +52,8 @@ func Contains(collection interface{}, element interface{}) bool { } // TODO keep it for backward compatibility in "webconfig" for now -// plan to remove it later +// +// plan to remove it later func ContainsInt(data []int, x int) bool { for _, d := range data { if d == x { From 103f782c04a8e5bff55ef19561da67f03cb93d51 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 2 Dec 2025 22:26:22 -0800 Subject: [PATCH 184/215] modify the use patterns of rand package --- common/document_test.go | 10 ++---- {util => common}/random.go | 11 +++---- {util => common}/random_test.go | 2 +- db/cassandra/document_test.go | 17 ++++------ db/cassandra/refsubdocument_test.go | 9 ++---- db/cassandra/state_update_test.go | 4 +-- db/service_test.go | 6 ++-- db/sqlite/refsubdocument_test.go | 9 ++---- http/bitmap_filter_test.go | 6 ++-- http/document_handler_test.go | 22 ++++++------- http/factory_reset_upstream_test.go | 16 +++++----- http/multipart_test.go | 34 ++++++++++---------- http/poke_handler_test.go | 6 ++-- http/refsubdocument_handler_test.go | 6 ++-- http/rootdocument_handler_test.go | 10 +++--- http/upstream_connector_test.go | 3 +- http/upstream_test.go | 48 ++++++++++++++--------------- http/validator_test.go | 4 +-- util/mock.go | 9 ++---- util/murmur3_test.go | 5 +-- util/string_test.go | 2 +- 21 files changed, 109 insertions(+), 130 deletions(-) rename {util => common}/random.go (91%) rename {util => common}/random_test.go (98%) diff --git a/common/document_test.go b/common/document_test.go index 9a63919..c074aba 100644 --- a/common/document_test.go +++ b/common/document_test.go @@ -18,8 +18,6 @@ package common import ( - crand "crypto/rand" - "math/rand" "strconv" "testing" "time" @@ -44,9 +42,7 @@ func TestDocument(t *testing.T) { mparts := []Multipart{} versionMap := make(map[string]string) for _, subdocId := range subdocIds { - blen := rand.Intn(10) + 10 - bbytes := make([]byte, blen) - crand.Read(bbytes) + bbytes := RandomBytes(10, 20) version := strconv.Itoa(int(time.Now().Unix())) mpart := Multipart{ Bytes: bbytes, @@ -94,9 +90,7 @@ func TestFilterByBitmap(t *testing.T) { mparts := []Multipart{} versionMap := make(map[string]string) for _, subdocId := range subdocIds { - blen := rand.Intn(10) + 10 - bbytes := make([]byte, blen) - crand.Read(bbytes) + bbytes := RandomBytes(10, 20) version := strconv.Itoa(int(time.Now().Unix())) mpart := Multipart{ Bytes: bbytes, diff --git a/util/random.go b/common/random.go similarity index 91% rename from util/random.go rename to common/random.go index 4400e3f..32eebb0 100644 --- a/util/random.go +++ b/common/random.go @@ -15,17 +15,13 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package util +package common import ( + cryptorand "crypto/rand" "math/rand" - "time" ) -func init() { - rand.Seed(time.Now().Unix()) -} - func RandomDouble() float64 { return rand.Float64() } @@ -46,6 +42,7 @@ func RandomBytes(lowerBound, upperBound int) []byte { n = rand.Intn(delta) + lowerBound } bbytes := make([]byte, n) - rand.Read(bbytes) + cryptorand.Read(bbytes) + bbytes[len(bbytes)-1] = 1 return bbytes } diff --git a/util/random_test.go b/common/random_test.go similarity index 98% rename from util/random_test.go rename to common/random_test.go index ef53a06..224158b 100644 --- a/util/random_test.go +++ b/common/random_test.go @@ -15,7 +15,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package util +package common import ( "testing" diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index 8e63844..da5e588 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -14,11 +14,10 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( - "crypto/rand" "net/http" "strconv" "testing" @@ -36,9 +35,7 @@ func TestMocaSubDocument(t *testing.T) { subdocId := "moca" // prepare the source data - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -66,9 +63,7 @@ func TestPrivatessidSubDocument(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() groupId := "privatessid" - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -231,7 +226,7 @@ func TestExpirySubDocument(t *testing.T) { // prepare some subdocs subdocIds := []string{"privatessid", "lan", "wan"} for _, subdocId := range subdocIds { - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -246,7 +241,7 @@ func TestExpirySubDocument(t *testing.T) { assert.Assert(t, doc.Length() == len(subdocIds)) // add an expiry-type but not-yet-expired subdoc - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) now := time.Now() nowTs := int(now.UnixNano() / 1000000) futureT := now.AddDate(0, 0, 2) @@ -265,7 +260,7 @@ func TestExpirySubDocument(t *testing.T) { assert.Assert(t, doc.Length() == len(subdocIds)+1) // set an expired subdoc - srcBytes = util.RandomBytes(100, 150) + srcBytes = common.RandomBytes(100, 150) past := now.Add(time.Duration(-1) * time.Hour) pastTs := int(past.UnixNano() / 1000000) srcVersion = strconv.Itoa(nowTs) diff --git a/db/cassandra/refsubdocument_test.go b/db/cassandra/refsubdocument_test.go index 22e0ba8..fca3fad 100644 --- a/db/cassandra/refsubdocument_test.go +++ b/db/cassandra/refsubdocument_test.go @@ -14,16 +14,15 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( - "crypto/rand" "testing" + "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/google/uuid" "gotest.tools/assert" ) @@ -31,9 +30,7 @@ func TestRefSubDocumentOperation(t *testing.T) { refId := uuid.New().String() // prepare the source data - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) // verify empty before start diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index e9bdae6..5c9e1a0 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -35,7 +35,7 @@ func TestStateUpdate1(t *testing.T) { // setup a doct groupId := "privatessid" - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -88,7 +88,7 @@ func TestStateUpdate2(t *testing.T) { // setup a doct groupId := "privatessid" - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload diff --git a/db/service_test.go b/db/service_test.go index c0226b0..65edaf8 100644 --- a/db/service_test.go +++ b/db/service_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( @@ -53,12 +53,12 @@ func TestUpdateRootVersion(t *testing.T) { doc := common.NewDocument(nil) tt := int(123) - bbytes1 := util.RandomBytes(100, 150) + bbytes1 := common.RandomBytes(100, 150) v1 := util.GetMurmur3Hash(bbytes1) subdoc1 := common.NewSubDocument(bbytes1, &v1, nil, &tt, nil, nil) doc.SetSubDocument("advsecurity", subdoc1) - bbytes2 := util.RandomBytes(100, 150) + bbytes2 := common.RandomBytes(100, 150) v2 := util.GetMurmur3Hash(bbytes2) subdoc2 := common.NewSubDocument(bbytes2, &v2, nil, &tt, nil, nil) doc.SetSubDocument("mesh", subdoc2) diff --git a/db/sqlite/refsubdocument_test.go b/db/sqlite/refsubdocument_test.go index 01478f9..8df729c 100644 --- a/db/sqlite/refsubdocument_test.go +++ b/db/sqlite/refsubdocument_test.go @@ -14,16 +14,15 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( - "crypto/rand" "testing" + "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/google/uuid" "gotest.tools/assert" ) @@ -31,9 +30,7 @@ func TestRefSubDocumentOperation(t *testing.T) { refId := uuid.New().String() // prepare the source data - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) // verify empty before start diff --git a/http/bitmap_filter_test.go b/http/bitmap_filter_test.go index 88cb46a..f8705dc 100644 --- a/http/bitmap_filter_test.go +++ b/http/bitmap_filter_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -43,7 +43,7 @@ func TestFilterOutputByBitmap(t *testing.T) { // post subdocId := "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -122,7 +122,7 @@ func TestBitmapFilterExemptSubdocIds(t *testing.T) { // post subdocId := "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 5d274c1..dc263fc 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -105,7 +105,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -135,7 +135,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -212,7 +212,7 @@ func TestPostWithDeviceId(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -244,7 +244,7 @@ func TestPostWithDeviceId(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -281,7 +281,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId := "gwrestore" gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - gwrestoreBytes := util.RandomBytes(100, 150) + gwrestoreBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -318,7 +318,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId = "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -362,7 +362,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId = "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -441,7 +441,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId := "gwrestore" gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - gwrestoreBytes := util.RandomBytes(100, 150) + gwrestoreBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -478,7 +478,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId = "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -522,7 +522,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId = "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -595,7 +595,7 @@ func TestBadHeaderExpiryHandler(t *testing.T) { // post subdocId := "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 1271836..7f67da2 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -95,7 +95,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -121,7 +121,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -191,7 +191,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -217,7 +217,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -345,7 +345,7 @@ func TestFactoryResetUpstreamAddData(t *testing.T) { // add a subdoc from upsream mparts := []common.Multipart{ { - Bytes: util.RandomBytes(100, 150), + Bytes: common.RandomBytes(100, 150), Version: strconv.Itoa(int(time.Now().Unix())), Name: "network", State: common.PendingDownload, @@ -397,7 +397,7 @@ func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -423,7 +423,7 @@ func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -449,7 +449,7 @@ func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { // ==== group 3 webui ==== subdocId = "webui" - webuiBytes := util.RandomBytes(m, n) + webuiBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) diff --git a/http/multipart_test.go b/http/multipart_test.go index 594550d..fe5676e 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -303,7 +303,7 @@ func TestVersionFiltering(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -329,7 +329,7 @@ func TestVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -447,7 +447,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -473,7 +473,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -621,7 +621,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -647,7 +647,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -798,7 +798,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -824,7 +824,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post @@ -919,7 +919,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -945,7 +945,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post @@ -972,7 +972,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // ==== group 3 mesh ==== subdocId = "mesh" - meshBytes := util.RandomBytes(m, n) + meshBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post @@ -999,7 +999,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // ==== group 3 moca ==== subdocId = "moca" - mocaBytes := util.RandomBytes(m, n) + mocaBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post @@ -1199,7 +1199,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -1223,7 +1223,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -1246,7 +1246,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // post subdocId = "privatessid" privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - privatessidBytes := util.RandomBytes(100, 150) + privatessidBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -1354,7 +1354,7 @@ func TestValidateQueryParams(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -1380,7 +1380,7 @@ func TestValidateQueryParams(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 9a9d3a2..e9952a4 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -118,7 +118,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -150,7 +150,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -226,7 +226,7 @@ func TestBuildMqttSendDocument(t *testing.T) { assert.Equal(t, document.Length(), 2) // ==== step 7 change the subdoc again ==== - lanBytes2 := util.RandomBytes(100, 150) + lanBytes2 := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes2)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index c9a1e2f..f139ac6 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -35,7 +35,7 @@ func TestRefSubDocumentHandler(t *testing.T) { router := server.GetRouter(true) refId := uuid.New().String() - bbytes := util.RandomBytes(100, 150) + bbytes := common.RandomBytes(100, 150) // post url := fmt.Sprintf("/api/v1/reference/%v/document", refId) @@ -86,7 +86,7 @@ func TestSubDocumentWithInvalidRefDoc(t *testing.T) { // ==== step 1 setup refdoc1 and subdoc1 ==== refId1 := uuid.New().String() - bbytes1 := util.RandomBytes(100, 150) + bbytes1 := common.RandomBytes(100, 150) subdocId1 := "defaultrfc" // post @@ -123,7 +123,7 @@ func TestSubDocumentWithInvalidRefDoc(t *testing.T) { // ==== step 2 setup refdoc2 and subdoc2 ==== refId2 := uuid.New().String() - bbytes2 := util.RandomBytes(100, 150) + bbytes2 := common.RandomBytes(100, 150) subdocId2 := "defaulttelemetry" // post diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 6bc640c..a0db7a3 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -284,11 +284,11 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { assert.DeepEqual(t, rootdoc, expectedRootdoc) // ==== step 2 ==== - supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(util.RandomBytes(10, 15)) - firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(util.RandomBytes(10, 15)) - modelName2 := "CGM4331COM" + string(util.RandomBytes(10, 15)) - partner2 := "comcast" + string(util.RandomBytes(10, 15)) - schemaVersion2 := "33554433-1.3,33554434-1.3" + string(util.RandomBytes(10, 15)) + supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(common.RandomBytes(10, 15)) + firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(common.RandomBytes(10, 15)) + modelName2 := "CGM4331COM" + string(common.RandomBytes(10, 15)) + partner2 := "comcast" + string(common.RandomBytes(10, 15)) + schemaVersion2 := "33554433-1.3,33554434-1.3" + string(common.RandomBytes(10, 15)) req.Header.Set(common.HeaderSupportedDocs, supportedDocs2) req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion2) diff --git a/http/upstream_connector_test.go b/http/upstream_connector_test.go index d09e1a7..f96d986 100644 --- a/http/upstream_connector_test.go +++ b/http/upstream_connector_test.go @@ -22,6 +22,7 @@ import ( "net/http/httptest" "testing" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "gotest.tools/assert" @@ -31,7 +32,7 @@ func TestUpstreamConnector(t *testing.T) { server := NewWebconfigServer(sc, true) // setup upstream mock server - mockedUpstreamResponse := util.RandomBytes(100, 150) + mockedUpstreamResponse := common.RandomBytes(100, 150) upstreamMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) diff --git a/http/upstream_test.go b/http/upstream_test.go index 0cb0b4a..8259c09 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -42,10 +42,10 @@ func TestUpstream(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) - privatessidV14Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) + privatessidV14Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -339,9 +339,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidBytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidBytes, "lan": lanBytes, @@ -556,10 +556,10 @@ func TestUpstreamUpdatedTime(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) - privatessidV14Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) + privatessidV14Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -868,9 +868,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -898,7 +898,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { } // create a new document - pfBytes := util.RandomBytes(m, n) + pfBytes := common.RandomBytes(m, n) version := util.GetMurmur3Hash(pfBytes) newMparts := []common.Multipart{ { @@ -1161,9 +1161,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -1191,7 +1191,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { } // create a new document - pfBytes := util.RandomBytes(m, n) + pfBytes := common.RandomBytes(m, n) version := util.GetMurmur3Hash(pfBytes) newMparts := []common.Multipart{ { @@ -1449,9 +1449,9 @@ func TestUpstreamBackfill(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidBytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidBytes, "lan": lanBytes, @@ -1693,7 +1693,7 @@ func TestUpstreamNoBitmapHeader(t *testing.T) { // ==== step 2 POST group lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -1719,7 +1719,7 @@ func TestUpstreamNoBitmapHeader(t *testing.T) { // ==== step 3 POST group wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) diff --git a/http/validator_test.go b/http/validator_test.go index aafc71c..0b2687d 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -271,7 +271,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -300,7 +300,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) diff --git a/util/mock.go b/util/mock.go index 1c5e564..6f5229d 100644 --- a/util/mock.go +++ b/util/mock.go @@ -14,11 +14,10 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( - "crypto/rand" "strconv" "strings" @@ -29,12 +28,10 @@ func GetMockMultiparts(queryStr string) []common.Multipart { groupIds := strings.Split(queryStr, ",") mparts := []common.Multipart{} for _, g := range groupIds { - slen := RandomInt(100) + 16 - bbytes := make([]byte, slen) - rand.Read(bbytes) + bbytes := common.RandomBytes(16, 116) mpart := common.Multipart{ Bytes: bbytes, - Version: strconv.Itoa(RandomInt(100000000)), + Version: strconv.Itoa(common.RandomInt(100000000)), Name: g, } mparts = append(mparts, mpart) diff --git a/util/murmur3_test.go b/util/murmur3_test.go index c0203f5..e173f96 100644 --- a/util/murmur3_test.go +++ b/util/murmur3_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( "testing" + "github.com/rdkcentral/webconfig/common" tmur "github.com/twmb/murmur3" "gotest.tools/assert" ) @@ -46,7 +47,7 @@ func TestSomeMurmur3(t *testing.T) { s1 := GetMurmur3Hash(nil) assert.Equal(t, s1, "0") - bbytes := RandomBytes(10, 20) + bbytes := common.RandomBytes(10, 20) s2 := GetMurmur3Hash(bbytes) assert.Assert(t, len(s2) > 0) } diff --git a/util/string_test.go b/util/string_test.go index 3c6df1a..e25b200 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -62,7 +62,7 @@ func TestIsValidUTF8(t *testing.T) { b1 := []byte(`{"foo":"bar","hello":123,"world":true}`) assert.Assert(t, IsValidUTF8(b1)) - b2 := RandomBytes(100, 150) + b2 := common.RandomBytes(100, 150) assert.Assert(t, !IsValidUTF8(b2)) } From 30f979ab76c4feb123948c6c551ef41d9d390f2b Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 2 Dec 2025 19:36:45 -0800 Subject: [PATCH 185/215] fix a bug that bitmap filtering was not applied for NONE/NONE-REBOOT call route --- http/factory_reset_upstream_test.go | 183 ++++++++++++++++++++++++++++ http/multipart.go | 10 +- http/webconfig_server.go | 2 +- util/list.go | 5 +- 4 files changed, 195 insertions(+), 5 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 5fc3c3a..1271836 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -388,3 +388,186 @@ func TestFactoryResetUpstreamAddData(t *testing.T) { assert.NilError(t, err) assert.Equal(t, rootDocument.Bitmap, 32479) } + +func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== group 3 webui ==== + subdocId = "webui" + webuiBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(webuiBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, webuiBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderProductClass, "XB8") + req.Header.Set(common.HeaderFirmwareVersion, "CGM4981COM_8.0p4s1_PROD_sey") + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderModelName, "CGM4981COM") + req.Header.Set(common.HeaderSupportedDocs, "16777439,33554435,50331649,67108865,83886081,100663423,117440513,134217735,201326594,218103809,251658241,268435457,285212673") + req.Header.Set(common.HeaderSchemaVersion, "16777232-1.3,33554433-1.3,33554434-1.3") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + // lanVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + mpart, ok = mparts["webui"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, webuiBytes) + + // ==== setup mock upstream server ==== + fields := make(log.Fields) + mockDoc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + + mockRootDoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + mockDoc.SetRootDocument(mockRootDoc) + + // mockDoc.DeleteSubDocument("wan") + // assert.NilError(t, err) + + mockBytes, err := mockDoc.Bytes() + assert.NilError(t, err) + + db.RefreshRootDocumentVersion(mockDoc) + refreshedRootVersion := mockDoc.RootVersion() + + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.Header().Set(common.HeaderContentLength, strconv.Itoa(len(mockBytes))) + ifNoneMatch := refreshedRootVersion + w.Header().Set(common.HeaderEtag, ifNoneMatch) + w.WriteHeader(http.StatusOK) + w.Write(mockBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(true) + server.SetFilterOutputByBitmapEnabled(true) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderProductClass, "XB8") + req.Header.Set(common.HeaderFirmwareVersion, "CGM4981COM_8.0p4s1_PROD_sey") + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderModelName, "CGM4981COM") + req.Header.Set(common.HeaderSupportedDocs, "16777439,33554435,50331649,67108865,83886081,100663423,117440513,134217735,201326594,218103809,251658241,268435457,285212673") + req.Header.Set(common.HeaderSchemaVersion, "16777232-1.3,33554433-1.3,33554434-1.3") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + + // verify Document is now empty + doc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + assert.Equal(t, doc.Length(), 3) +} diff --git a/http/multipart.go b/http/multipart.go index efcb7b7..72ca709 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -440,9 +440,9 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusInternalServerError, respHeader, oldDocBytes, common.NewError(err) } upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) + finalRootDocument := rootDocument.Clone() + finalRootDocument.Version = upstreamRespEtag - // filter by versionMap and filter by blockedIds - finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) for _, subdocId := range c.BlockedSubdocIds() { @@ -466,6 +466,12 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l if err != nil { return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) } + + // filter by bitmaps and blockedIds + if s.FilterOutputByBitmapEnabled() { + finalDocument = finalDocument.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) + } + finalBytes, err := finalDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index dd24fe5..9828851 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -403,7 +403,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if !util.ValidateMac(mac) { err := *common.NewHttp400Error("invalid mac") Error(w, http.StatusBadRequest, common.NewError(err)) - return + return } } diff --git a/util/list.go b/util/list.go index f58b2b6..41e7641 100644 --- a/util/list.go +++ b/util/list.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -52,7 +52,8 @@ func Contains(collection interface{}, element interface{}) bool { } // TODO keep it for backward compatibility in "webconfig" for now -// plan to remove it later +// +// plan to remove it later func ContainsInt(data []int, x int) bool { for _, d := range data { if d == x { From f2df8b14150988c9fbaf03fa3edf2f74143d5135 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 2 Dec 2025 22:26:22 -0800 Subject: [PATCH 186/215] modify the use patterns of rand package --- common/document_test.go | 10 ++---- {util => common}/random.go | 11 +++---- {util => common}/random_test.go | 2 +- db/cassandra/document_test.go | 17 ++++------ db/cassandra/refsubdocument_test.go | 9 ++---- db/cassandra/state_update_test.go | 4 +-- db/service_test.go | 6 ++-- db/sqlite/refsubdocument_test.go | 9 ++---- http/bitmap_filter_test.go | 6 ++-- http/document_handler_test.go | 22 ++++++------- http/factory_reset_upstream_test.go | 16 +++++----- http/multipart_test.go | 34 ++++++++++---------- http/poke_handler_test.go | 6 ++-- http/refsubdocument_handler_test.go | 6 ++-- http/rootdocument_handler_test.go | 10 +++--- http/upstream_connector_test.go | 3 +- http/upstream_test.go | 48 ++++++++++++++--------------- http/validator_test.go | 4 +-- util/mock.go | 9 ++---- util/murmur3_test.go | 5 +-- util/string_test.go | 2 +- 21 files changed, 109 insertions(+), 130 deletions(-) rename {util => common}/random.go (91%) rename {util => common}/random_test.go (98%) diff --git a/common/document_test.go b/common/document_test.go index 9a63919..c074aba 100644 --- a/common/document_test.go +++ b/common/document_test.go @@ -18,8 +18,6 @@ package common import ( - crand "crypto/rand" - "math/rand" "strconv" "testing" "time" @@ -44,9 +42,7 @@ func TestDocument(t *testing.T) { mparts := []Multipart{} versionMap := make(map[string]string) for _, subdocId := range subdocIds { - blen := rand.Intn(10) + 10 - bbytes := make([]byte, blen) - crand.Read(bbytes) + bbytes := RandomBytes(10, 20) version := strconv.Itoa(int(time.Now().Unix())) mpart := Multipart{ Bytes: bbytes, @@ -94,9 +90,7 @@ func TestFilterByBitmap(t *testing.T) { mparts := []Multipart{} versionMap := make(map[string]string) for _, subdocId := range subdocIds { - blen := rand.Intn(10) + 10 - bbytes := make([]byte, blen) - crand.Read(bbytes) + bbytes := RandomBytes(10, 20) version := strconv.Itoa(int(time.Now().Unix())) mpart := Multipart{ Bytes: bbytes, diff --git a/util/random.go b/common/random.go similarity index 91% rename from util/random.go rename to common/random.go index 4400e3f..32eebb0 100644 --- a/util/random.go +++ b/common/random.go @@ -15,17 +15,13 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package util +package common import ( + cryptorand "crypto/rand" "math/rand" - "time" ) -func init() { - rand.Seed(time.Now().Unix()) -} - func RandomDouble() float64 { return rand.Float64() } @@ -46,6 +42,7 @@ func RandomBytes(lowerBound, upperBound int) []byte { n = rand.Intn(delta) + lowerBound } bbytes := make([]byte, n) - rand.Read(bbytes) + cryptorand.Read(bbytes) + bbytes[len(bbytes)-1] = 1 return bbytes } diff --git a/util/random_test.go b/common/random_test.go similarity index 98% rename from util/random_test.go rename to common/random_test.go index ef53a06..224158b 100644 --- a/util/random_test.go +++ b/common/random_test.go @@ -15,7 +15,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package util +package common import ( "testing" diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index 8e63844..da5e588 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -14,11 +14,10 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( - "crypto/rand" "net/http" "strconv" "testing" @@ -36,9 +35,7 @@ func TestMocaSubDocument(t *testing.T) { subdocId := "moca" // prepare the source data - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -66,9 +63,7 @@ func TestPrivatessidSubDocument(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() groupId := "privatessid" - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -231,7 +226,7 @@ func TestExpirySubDocument(t *testing.T) { // prepare some subdocs subdocIds := []string{"privatessid", "lan", "wan"} for _, subdocId := range subdocIds { - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -246,7 +241,7 @@ func TestExpirySubDocument(t *testing.T) { assert.Assert(t, doc.Length() == len(subdocIds)) // add an expiry-type but not-yet-expired subdoc - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) now := time.Now() nowTs := int(now.UnixNano() / 1000000) futureT := now.AddDate(0, 0, 2) @@ -265,7 +260,7 @@ func TestExpirySubDocument(t *testing.T) { assert.Assert(t, doc.Length() == len(subdocIds)+1) // set an expired subdoc - srcBytes = util.RandomBytes(100, 150) + srcBytes = common.RandomBytes(100, 150) past := now.Add(time.Duration(-1) * time.Hour) pastTs := int(past.UnixNano() / 1000000) srcVersion = strconv.Itoa(nowTs) diff --git a/db/cassandra/refsubdocument_test.go b/db/cassandra/refsubdocument_test.go index 22e0ba8..fca3fad 100644 --- a/db/cassandra/refsubdocument_test.go +++ b/db/cassandra/refsubdocument_test.go @@ -14,16 +14,15 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( - "crypto/rand" "testing" + "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/google/uuid" "gotest.tools/assert" ) @@ -31,9 +30,7 @@ func TestRefSubDocumentOperation(t *testing.T) { refId := uuid.New().String() // prepare the source data - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) // verify empty before start diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index e9bdae6..5c9e1a0 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -35,7 +35,7 @@ func TestStateUpdate1(t *testing.T) { // setup a doct groupId := "privatessid" - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -88,7 +88,7 @@ func TestStateUpdate2(t *testing.T) { // setup a doct groupId := "privatessid" - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload diff --git a/db/service_test.go b/db/service_test.go index c0226b0..65edaf8 100644 --- a/db/service_test.go +++ b/db/service_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( @@ -53,12 +53,12 @@ func TestUpdateRootVersion(t *testing.T) { doc := common.NewDocument(nil) tt := int(123) - bbytes1 := util.RandomBytes(100, 150) + bbytes1 := common.RandomBytes(100, 150) v1 := util.GetMurmur3Hash(bbytes1) subdoc1 := common.NewSubDocument(bbytes1, &v1, nil, &tt, nil, nil) doc.SetSubDocument("advsecurity", subdoc1) - bbytes2 := util.RandomBytes(100, 150) + bbytes2 := common.RandomBytes(100, 150) v2 := util.GetMurmur3Hash(bbytes2) subdoc2 := common.NewSubDocument(bbytes2, &v2, nil, &tt, nil, nil) doc.SetSubDocument("mesh", subdoc2) diff --git a/db/sqlite/refsubdocument_test.go b/db/sqlite/refsubdocument_test.go index 01478f9..8df729c 100644 --- a/db/sqlite/refsubdocument_test.go +++ b/db/sqlite/refsubdocument_test.go @@ -14,16 +14,15 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( - "crypto/rand" "testing" + "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/google/uuid" "gotest.tools/assert" ) @@ -31,9 +30,7 @@ func TestRefSubDocumentOperation(t *testing.T) { refId := uuid.New().String() // prepare the source data - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) // verify empty before start diff --git a/http/bitmap_filter_test.go b/http/bitmap_filter_test.go index 88cb46a..f8705dc 100644 --- a/http/bitmap_filter_test.go +++ b/http/bitmap_filter_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -43,7 +43,7 @@ func TestFilterOutputByBitmap(t *testing.T) { // post subdocId := "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -122,7 +122,7 @@ func TestBitmapFilterExemptSubdocIds(t *testing.T) { // post subdocId := "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 5d274c1..dc263fc 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -105,7 +105,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -135,7 +135,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -212,7 +212,7 @@ func TestPostWithDeviceId(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -244,7 +244,7 @@ func TestPostWithDeviceId(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -281,7 +281,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId := "gwrestore" gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - gwrestoreBytes := util.RandomBytes(100, 150) + gwrestoreBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -318,7 +318,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId = "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -362,7 +362,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId = "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -441,7 +441,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId := "gwrestore" gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - gwrestoreBytes := util.RandomBytes(100, 150) + gwrestoreBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -478,7 +478,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId = "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) @@ -522,7 +522,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId = "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -595,7 +595,7 @@ func TestBadHeaderExpiryHandler(t *testing.T) { // post subdocId := "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 1271836..7f67da2 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -95,7 +95,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -121,7 +121,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -191,7 +191,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -217,7 +217,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -345,7 +345,7 @@ func TestFactoryResetUpstreamAddData(t *testing.T) { // add a subdoc from upsream mparts := []common.Multipart{ { - Bytes: util.RandomBytes(100, 150), + Bytes: common.RandomBytes(100, 150), Version: strconv.Itoa(int(time.Now().Unix())), Name: "network", State: common.PendingDownload, @@ -397,7 +397,7 @@ func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -423,7 +423,7 @@ func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -449,7 +449,7 @@ func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { // ==== group 3 webui ==== subdocId = "webui" - webuiBytes := util.RandomBytes(m, n) + webuiBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) diff --git a/http/multipart_test.go b/http/multipart_test.go index 594550d..fe5676e 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -303,7 +303,7 @@ func TestVersionFiltering(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -329,7 +329,7 @@ func TestVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -447,7 +447,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -473,7 +473,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -621,7 +621,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -647,7 +647,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -798,7 +798,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -824,7 +824,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post @@ -919,7 +919,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -945,7 +945,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post @@ -972,7 +972,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // ==== group 3 mesh ==== subdocId = "mesh" - meshBytes := util.RandomBytes(m, n) + meshBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post @@ -999,7 +999,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // ==== group 3 moca ==== subdocId = "moca" - mocaBytes := util.RandomBytes(m, n) + mocaBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post @@ -1199,7 +1199,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -1223,7 +1223,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -1246,7 +1246,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // post subdocId = "privatessid" privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - privatessidBytes := util.RandomBytes(100, 150) + privatessidBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -1354,7 +1354,7 @@ func TestValidateQueryParams(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -1380,7 +1380,7 @@ func TestValidateQueryParams(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 9a9d3a2..e9952a4 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -118,7 +118,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -150,7 +150,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -226,7 +226,7 @@ func TestBuildMqttSendDocument(t *testing.T) { assert.Equal(t, document.Length(), 2) // ==== step 7 change the subdoc again ==== - lanBytes2 := util.RandomBytes(100, 150) + lanBytes2 := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes2)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index c9a1e2f..f139ac6 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -35,7 +35,7 @@ func TestRefSubDocumentHandler(t *testing.T) { router := server.GetRouter(true) refId := uuid.New().String() - bbytes := util.RandomBytes(100, 150) + bbytes := common.RandomBytes(100, 150) // post url := fmt.Sprintf("/api/v1/reference/%v/document", refId) @@ -86,7 +86,7 @@ func TestSubDocumentWithInvalidRefDoc(t *testing.T) { // ==== step 1 setup refdoc1 and subdoc1 ==== refId1 := uuid.New().String() - bbytes1 := util.RandomBytes(100, 150) + bbytes1 := common.RandomBytes(100, 150) subdocId1 := "defaultrfc" // post @@ -123,7 +123,7 @@ func TestSubDocumentWithInvalidRefDoc(t *testing.T) { // ==== step 2 setup refdoc2 and subdoc2 ==== refId2 := uuid.New().String() - bbytes2 := util.RandomBytes(100, 150) + bbytes2 := common.RandomBytes(100, 150) subdocId2 := "defaulttelemetry" // post diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 6bc640c..a0db7a3 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -284,11 +284,11 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { assert.DeepEqual(t, rootdoc, expectedRootdoc) // ==== step 2 ==== - supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(util.RandomBytes(10, 15)) - firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(util.RandomBytes(10, 15)) - modelName2 := "CGM4331COM" + string(util.RandomBytes(10, 15)) - partner2 := "comcast" + string(util.RandomBytes(10, 15)) - schemaVersion2 := "33554433-1.3,33554434-1.3" + string(util.RandomBytes(10, 15)) + supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(common.RandomBytes(10, 15)) + firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(common.RandomBytes(10, 15)) + modelName2 := "CGM4331COM" + string(common.RandomBytes(10, 15)) + partner2 := "comcast" + string(common.RandomBytes(10, 15)) + schemaVersion2 := "33554433-1.3,33554434-1.3" + string(common.RandomBytes(10, 15)) req.Header.Set(common.HeaderSupportedDocs, supportedDocs2) req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion2) diff --git a/http/upstream_connector_test.go b/http/upstream_connector_test.go index d09e1a7..f96d986 100644 --- a/http/upstream_connector_test.go +++ b/http/upstream_connector_test.go @@ -22,6 +22,7 @@ import ( "net/http/httptest" "testing" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "gotest.tools/assert" @@ -31,7 +32,7 @@ func TestUpstreamConnector(t *testing.T) { server := NewWebconfigServer(sc, true) // setup upstream mock server - mockedUpstreamResponse := util.RandomBytes(100, 150) + mockedUpstreamResponse := common.RandomBytes(100, 150) upstreamMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) diff --git a/http/upstream_test.go b/http/upstream_test.go index 0cb0b4a..8259c09 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -42,10 +42,10 @@ func TestUpstream(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) - privatessidV14Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) + privatessidV14Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -339,9 +339,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidBytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidBytes, "lan": lanBytes, @@ -556,10 +556,10 @@ func TestUpstreamUpdatedTime(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) - privatessidV14Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) + privatessidV14Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -868,9 +868,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -898,7 +898,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { } // create a new document - pfBytes := util.RandomBytes(m, n) + pfBytes := common.RandomBytes(m, n) version := util.GetMurmur3Hash(pfBytes) newMparts := []common.Multipart{ { @@ -1161,9 +1161,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -1191,7 +1191,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { } // create a new document - pfBytes := util.RandomBytes(m, n) + pfBytes := common.RandomBytes(m, n) version := util.GetMurmur3Hash(pfBytes) newMparts := []common.Multipart{ { @@ -1449,9 +1449,9 @@ func TestUpstreamBackfill(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidBytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidBytes, "lan": lanBytes, @@ -1693,7 +1693,7 @@ func TestUpstreamNoBitmapHeader(t *testing.T) { // ==== step 2 POST group lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) @@ -1719,7 +1719,7 @@ func TestUpstreamNoBitmapHeader(t *testing.T) { // ==== step 3 POST group wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) diff --git a/http/validator_test.go b/http/validator_test.go index aafc71c..0b2687d 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -271,7 +271,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) @@ -300,7 +300,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) diff --git a/util/mock.go b/util/mock.go index 1c5e564..6f5229d 100644 --- a/util/mock.go +++ b/util/mock.go @@ -14,11 +14,10 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( - "crypto/rand" "strconv" "strings" @@ -29,12 +28,10 @@ func GetMockMultiparts(queryStr string) []common.Multipart { groupIds := strings.Split(queryStr, ",") mparts := []common.Multipart{} for _, g := range groupIds { - slen := RandomInt(100) + 16 - bbytes := make([]byte, slen) - rand.Read(bbytes) + bbytes := common.RandomBytes(16, 116) mpart := common.Multipart{ Bytes: bbytes, - Version: strconv.Itoa(RandomInt(100000000)), + Version: strconv.Itoa(common.RandomInt(100000000)), Name: g, } mparts = append(mparts, mpart) diff --git a/util/murmur3_test.go b/util/murmur3_test.go index c0203f5..e173f96 100644 --- a/util/murmur3_test.go +++ b/util/murmur3_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( "testing" + "github.com/rdkcentral/webconfig/common" tmur "github.com/twmb/murmur3" "gotest.tools/assert" ) @@ -46,7 +47,7 @@ func TestSomeMurmur3(t *testing.T) { s1 := GetMurmur3Hash(nil) assert.Equal(t, s1, "0") - bbytes := RandomBytes(10, 20) + bbytes := common.RandomBytes(10, 20) s2 := GetMurmur3Hash(bbytes) assert.Assert(t, len(s2) > 0) } diff --git a/util/string_test.go b/util/string_test.go index 3c6df1a..e25b200 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -62,7 +62,7 @@ func TestIsValidUTF8(t *testing.T) { b1 := []byte(`{"foo":"bar","hello":123,"world":true}`) assert.Assert(t, IsValidUTF8(b1)) - b2 := RandomBytes(100, 150) + b2 := common.RandomBytes(100, 150) assert.Assert(t, !IsValidUTF8(b2)) } From befc8ca6e6ced9690242e8bc241541bf6c4d0a24 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 16 Dec 2025 16:11:40 -0800 Subject: [PATCH 187/215] GET supplementary can return precook telemetry data --- db/cassandra/cassandra_client.go | 52 +- db/cassandra/delete_columns_test.go | 196 ++++++ db/cassandra/document.go | 38 +- db/cassandra/document_test.go | 104 +++ db/cassandra/state_update_telemetry_test.go | 229 +++++++ db/database_client.go | 7 + db/service.go | 6 + db/sqlite/delete_columns_test.go | 188 ++++++ db/sqlite/document.go | 83 ++- db/sqlite/document_test.go | 106 ++- db/sqlite/schema.go | 1 + db/sqlite/sqlite_client.go | 40 +- go.mod | 18 +- go.sum | 28 +- http/supplementary_handler.go | 95 +++ http/supplementary_precook_test.go | 711 ++++++++++++++++++++ 16 files changed, 1845 insertions(+), 57 deletions(-) create mode 100644 db/cassandra/delete_columns_test.go create mode 100644 db/cassandra/state_update_telemetry_test.go create mode 100644 db/sqlite/delete_columns_test.go create mode 100644 http/supplementary_precook_test.go diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 3804f22..a1e4a85 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -48,12 +48,14 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string - stateCorrectionEnabled bool - lockRootDocumentEnabled bool + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool + supplementaryPrecookEnabled bool + supplementaryPrecookStateTTLDays int } /* @@ -163,17 +165,21 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") + supplementaryPrecookEnabled := conf.GetBoolean("webconfig.supplementary_precook_enabled") + supplementaryPrecookStateTTLDays := int(conf.GetInt32("webconfig.supplementary_precook_state_ttl_days", 7)) return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, - lockRootDocumentEnabled: lockRootDocumentEnabled, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, + supplementaryPrecookEnabled: supplementaryPrecookEnabled, + supplementaryPrecookStateTTLDays: supplementaryPrecookStateTTLDays, }, nil } @@ -295,3 +301,19 @@ func GetTestCassandraClient(conf *configuration.Config, testOnly bool) (*Cassand } return tdbclient, nil } + +func (c *CassandraClient) SupplementaryPrecookEnabled() bool { + return c.supplementaryPrecookEnabled +} + +func (c *CassandraClient) SetSupplementaryPrecookEnabled(enabled bool) { + c.supplementaryPrecookEnabled = enabled +} + +func (c *CassandraClient) SupplementaryPrecookStateTTLDays() int { + return c.supplementaryPrecookStateTTLDays +} + +func (c *CassandraClient) SetSupplementaryPrecookStateTTLDays(days int) { + c.supplementaryPrecookStateTTLDays = days +} diff --git a/db/cassandra/delete_columns_test.go b/db/cassandra/delete_columns_test.go new file mode 100644 index 0000000..3c65e77 --- /dev/null +++ b/db/cassandra/delete_columns_test.go @@ -0,0 +1,196 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestDeleteSubDocumentColumnsExpiry(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "telemetry" + + // Create subdocument with expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Deployed + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify expiry is set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Equal(t, *fetchedSubdoc.Expiry(), futureExpiry) + + // Delete the expiry column + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry") + assert.NilError(t, err) + + // Verify expiry is now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} + +func TestDeleteSubDocumentColumnsMultiple(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "mesh" + + // Create subdocument with multiple fields + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.InDeployment + errorCode := 100 + errorDetails := "some error" + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with all fields set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errorCode, &errorDetails) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify all fields are set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Assert(t, fetchedSubdoc.ErrorCode() != nil) + assert.Assert(t, fetchedSubdoc.ErrorDetails() != nil) + + // Delete multiple columns + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry", "error_code", "error_details") + assert.NilError(t, err) + + // Verify deleted columns are now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + // Note: error_code and error_details might have default values, so we just verify no error occurred + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} + +func TestDeleteSubDocumentColumnsEmptyList(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "test" + + // Create a simple subdocument + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcState := common.PendingDownload + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + + fields := log.Fields{} + + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Call with empty column list should be no-op + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId) + assert.NilError(t, err) + + // Verify subdocument is unchanged + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *fetchedSubdoc.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc.State(), srcState) +} + +func TestDeleteSubDocumentColumnsErrorFields(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "telemetry" + + // Create subdocument with error fields and expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Failure + errorCode := 204 + errorDetails := "failed_retrying:Error unsupported namespace" + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with error fields and expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errorCode, &errorDetails) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify all fields are set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Equal(t, *fetchedSubdoc.Expiry(), futureExpiry) + assert.Assert(t, fetchedSubdoc.ErrorCode() != nil) + assert.Equal(t, *fetchedSubdoc.ErrorCode(), errorCode) + assert.Assert(t, fetchedSubdoc.ErrorDetails() != nil) + assert.Equal(t, *fetchedSubdoc.ErrorDetails(), errorDetails) + + // Delete expiry and error fields + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry", "error_code", "error_details") + assert.NilError(t, err) + + // Verify deleted columns are now cleared + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + // Note: Cassandra returns default values (0, "") for error_code and error_details after DELETE + assert.Assert(t, fetchedSubdoc2.ErrorCode() != nil) + assert.Equal(t, *fetchedSubdoc2.ErrorCode(), 0) + assert.Assert(t, fetchedSubdoc2.ErrorDetails() != nil) + assert.Equal(t, *fetchedSubdoc2.ErrorDetails(), "") + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} diff --git a/db/cassandra/document.go b/db/cassandra/document.go index df580b8..ec10cd3 100644 --- a/db/cassandra/document.go +++ b/db/cassandra/document.go @@ -14,18 +14,19 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "fmt" + "strings" "time" + "github.com/gocql/gocql" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gocql/gocql" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -56,6 +57,20 @@ func (c *CassandraClient) GetSubDocument(cpeMac string, groupId string) (*common } } + // Check if payload contains a reference to a refsubdocument + if refId, ok := db.GetRefId(payload); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + if !c.IsDbNotFound(err) { + return nil, common.NewError(err) + } + // If refsubdocument not found, continue with the reference payload + } else { + // Replace payload with the actual payload from refsubdocument + payload = refsubdocument.Payload() + } + } + if x := int(updatedTime.UnixNano() / 1000000); x > 0 { updatedTimeTsPtr = &x } @@ -195,6 +210,23 @@ func (c *CassandraClient) DeleteSubDocument(cpeMac string, groupId string) error return nil } +func (c *CassandraClient) DeleteSubDocumentColumns(cpeMac string, groupId string, columns ...string) error { + if len(columns) == 0 { + return nil + } + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // Build DELETE statement for specific columns + // In Cassandra: DELETE col1, col2 FROM table WHERE conditions + stmt := fmt.Sprintf("DELETE %v FROM xpc_group_config WHERE cpe_mac=? AND group_id=?", strings.Join(columns, ",")) + if err := c.Query(stmt, cpeMac, groupId).Exec(); err != nil { + return common.NewError(err) + } + return nil +} + func (c *CassandraClient) DeleteDocument(cpeMac string) error { c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index da5e588..9dd9bf0 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -277,3 +277,107 @@ func TestExpirySubDocument(t *testing.T) { assert.NilError(t, err) assert.Assert(t, doc.Length() == len(subdocIds)+1) } + +func TestGetSubDocumentWithReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "lan" + refId := util.GetMurmur3Hash([]byte(cpeMac + subdocId)) + + // Step 1: Create a reference subdocument with actual payload + actualPayload := common.RandomBytes(100, 200) + actualVersion := util.GetMurmur3Hash(actualPayload) + refSubdoc := common.NewRefSubDocument(actualPayload, &actualVersion) + + err := tdbclient.SetRefSubDocument(refId, refSubdoc) + assert.NilError(t, err) + + // Step 2: Create a subdocument with reference payload (4 zero bytes + refId) + referencePayload := append(make([]byte, 4), []byte(refId)...) + refVersion := util.GetMurmur3Hash(referencePayload) + refState := common.InDeployment + refUpdatedTime := int(time.Now().UnixMilli()) + + subdocWithRef := common.NewSubDocument(referencePayload, &refVersion, &refState, &refUpdatedTime, nil, nil) + fields := log.Fields{} + err = tdbclient.SetSubDocument(cpeMac, subdocId, subdocWithRef, fields) + assert.NilError(t, err) + + // Step 3: Call GetSubDocument and verify it returns the actual payload, not the reference + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is the actual payload from refsubdocument, not the reference + assert.DeepEqual(t, fetchedSubdoc.Payload(), actualPayload) + + // Verify other fields remain unchanged + assert.Equal(t, *fetchedSubdoc.Version(), refVersion) + assert.Equal(t, *fetchedSubdoc.State(), refState) + assert.Equal(t, *fetchedSubdoc.UpdatedTime(), refUpdatedTime) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) +} + +func TestGetSubDocumentWithMissingReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "wan" + refId := util.GetMurmur3Hash([]byte(cpeMac + subdocId + "nonexistent")) + + // Create a subdocument with reference payload pointing to non-existent refsubdocument + referencePayload := append(make([]byte, 4), []byte(refId)...) + refVersion := util.GetMurmur3Hash(referencePayload) + refState := common.InDeployment + refUpdatedTime := int(time.Now().UnixMilli()) + + subdocWithRef := common.NewSubDocument(referencePayload, &refVersion, &refState, &refUpdatedTime, nil, nil) + fields := log.Fields{} + err := tdbclient.SetSubDocument(cpeMac, subdocId, subdocWithRef, fields) + assert.NilError(t, err) + + // Call GetSubDocument - should return the reference payload since refsubdocument doesn't exist + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is the reference payload (since refsubdocument was not found) + assert.DeepEqual(t, fetchedSubdoc.Payload(), referencePayload) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) +} + +func TestGetSubDocumentWithoutReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "mesh" + + // Create a regular subdocument without any reference + regularPayload := common.RandomBytes(100, 200) + regularVersion := util.GetMurmur3Hash(regularPayload) + regularState := common.Deployed + regularUpdatedTime := int(time.Now().UnixMilli()) + + subdoc := common.NewSubDocument(regularPayload, ®ularVersion, ®ularState, ®ularUpdatedTime, nil, nil) + fields := log.Fields{} + err := tdbclient.SetSubDocument(cpeMac, subdocId, subdoc, fields) + assert.NilError(t, err) + + // Call GetSubDocument - should return the regular payload unchanged + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is unchanged + assert.DeepEqual(t, fetchedSubdoc.Payload(), regularPayload) + assert.Equal(t, *fetchedSubdoc.Version(), regularVersion) + assert.Equal(t, *fetchedSubdoc.State(), regularState) + assert.Equal(t, *fetchedSubdoc.UpdatedTime(), regularUpdatedTime) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) +} diff --git a/db/cassandra/state_update_telemetry_test.go b/db/cassandra/state_update_telemetry_test.go new file mode 100644 index 0000000..1593a68 --- /dev/null +++ b/db/cassandra/state_update_telemetry_test.go @@ -0,0 +1,229 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestTelemetryStateUpdateWithTTL(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // setup a telemetry subdoc + groupId := "telemetry" + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixMilli()) + srcState := common.PendingDownload + + srcDoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + fields := make(log.Fields) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcDoc, fields) + assert.NilError(t, err) + + fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) + + // Update to success with TTL set to 7 days + template := `{"application_status": "success", "device_id": "mac:%v", "namespace": "telemetry", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes := []byte(fmt.Sprintf(template, cpeMac, srcVersion)) + var m common.EventMessage + err = json.Unmarshal(bbytes, &m) + assert.NilError(t, err) + + // Enable supplementary precook and set TTL to 7 days on database client + supplementaryPrecookStateTTLDays := 7 + tdbclient.SetSupplementaryPrecookEnabled(true) + tdbclient.SetSupplementaryPrecookStateTTLDays(supplementaryPrecookStateTTLDays) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + + subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + assert.Equal(t, *subdoc.ErrorCode(), 0) + assert.Equal(t, *subdoc.ErrorDetails(), "") + + // Verify that expiry was set + assert.Assert(t, subdoc.Expiry() != nil) + expiryTime := *subdoc.Expiry() + currentTime := int(time.Now().UnixMilli()) + expectedExpiry := int(time.Now().Add(time.Duration(supplementaryPrecookStateTTLDays) * 24 * time.Hour).UnixMilli()) + + // Allow some margin for execution time (10 seconds) + margin := int(10 * time.Second / time.Millisecond) + assert.Assert(t, expiryTime >= currentTime) + assert.Assert(t, expiryTime <= expectedExpiry+margin) +} + +func TestTelemetryStateUpdateWithoutTTL(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // setup a telemetry subdoc + groupId := "telemetry" + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixMilli()) + srcState := common.PendingDownload + + srcDoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + fields := make(log.Fields) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcDoc, fields) + assert.NilError(t, err) + + fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) + + // Update to success with TTL set to 0 (no TTL) + template := `{"application_status": "success", "device_id": "mac:%v", "namespace": "telemetry", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes := []byte(fmt.Sprintf(template, cpeMac, srcVersion)) + var m common.EventMessage + err = json.Unmarshal(bbytes, &m) + assert.NilError(t, err) + + // Enable supplementary precook and set TTL to 0 (no expiry) + supplementaryPrecookStateTTLDays := 0 + tdbclient.SetSupplementaryPrecookEnabled(true) + tdbclient.SetSupplementaryPrecookStateTTLDays(supplementaryPrecookStateTTLDays) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + + subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + assert.Equal(t, *subdoc.ErrorCode(), 0) + assert.Equal(t, *subdoc.ErrorDetails(), "") + + // Verify that expiry was NOT set + assert.Assert(t, subdoc.Expiry() == nil) +} + +func TestNonTelemetryStateUpdateNoTTL(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // setup a non-telemetry subdoc (e.g., privatessid) + groupId := "privatessid" + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixMilli()) + srcState := common.PendingDownload + + srcDoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + fields := make(log.Fields) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcDoc, fields) + assert.NilError(t, err) + + fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) + + // Update to success with TTL configured but for non-telemetry subdoc + template := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes := []byte(fmt.Sprintf(template, cpeMac, srcVersion)) + var m common.EventMessage + err = json.Unmarshal(bbytes, &m) + assert.NilError(t, err) + + // Enable supplementary precook and set TTL to 7 days + // but since it's not telemetry, TTL should NOT be set + supplementaryPrecookStateTTLDays := 7 + tdbclient.SetSupplementaryPrecookEnabled(true) + tdbclient.SetSupplementaryPrecookStateTTLDays(supplementaryPrecookStateTTLDays) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + + subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + assert.Equal(t, *subdoc.ErrorCode(), 0) + assert.Equal(t, *subdoc.ErrorDetails(), "") + + // Verify that expiry was NOT set (since it's not telemetry) + assert.Assert(t, subdoc.Expiry() == nil) +} + +func TestTelemetryStateUpdateFailureNoTTL(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // setup a telemetry subdoc + groupId := "telemetry" + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixMilli()) + srcState := common.PendingDownload + + srcDoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + fields := make(log.Fields) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcDoc, fields) + assert.NilError(t, err) + + fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) + + // Update to failure - TTL should NOT be set even with supplementaryPrecookStateTTLDays configured + template := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "telemetry", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template, cpeMac, srcVersion)) + var m common.EventMessage + err = json.Unmarshal(bbytes, &m) + assert.NilError(t, err) + + // Enable supplementary precook and set TTL to 7 days + // But since state is failure, TTL should NOT be set + supplementaryPrecookStateTTLDays := 7 + tdbclient.SetSupplementaryPrecookEnabled(true) + tdbclient.SetSupplementaryPrecookStateTTLDays(supplementaryPrecookStateTTLDays) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + + subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Failure) + assert.Equal(t, *subdoc.ErrorCode(), 204) + assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") + + // Verify that expiry was NOT set (since state is failure, not success) + assert.Assert(t, subdoc.Expiry() == nil) +} diff --git a/db/database_client.go b/db/database_client.go index cc325f1..0548a81 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -30,6 +30,7 @@ type DatabaseClient interface { GetSubDocument(string, string) (*common.SubDocument, error) SetSubDocument(string, string, *common.SubDocument, ...interface{}) error DeleteSubDocument(string, string) error + DeleteSubDocumentColumns(string, string, ...string) error GetDocument(string, ...interface{}) (*common.Document, error) SetDocument(string, *common.Document) error @@ -71,4 +72,10 @@ type DatabaseClient interface { LockRootDocumentEnabled() bool SetLockRootDocumentEnabled(bool) + + // supplementary precook + SupplementaryPrecookEnabled() bool + SetSupplementaryPrecookEnabled(bool) + SupplementaryPrecookStateTTLDays() int + SetSupplementaryPrecookStateTTLDays(int) } diff --git a/db/service.go b/db/service.go index dab1732..80efde1 100644 --- a/db/service.go +++ b/db/service.go @@ -480,6 +480,12 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage newSubdoc := common.NewSubDocument(nil, nil, &state, &updatedTime, errorCodePtr, errorDetailsPtr) + // Set expiry for telemetry subdoc if supplementary precook is enabled, success state, and TTL is configured + if c.SupplementaryPrecookEnabled() && targetGroupId == "telemetry" && state == common.Deployed && c.SupplementaryPrecookStateTTLDays() > 0 { + expiryTime := int(time.Now().Add(time.Duration(c.SupplementaryPrecookStateTTLDays()) * 24 * time.Hour).UnixMilli()) + newSubdoc.SetExpiry(&expiryTime) + } + // metricsAgent handling if m.MetricsAgent != nil { labels["client"] = *m.MetricsAgent diff --git a/db/sqlite/delete_columns_test.go b/db/sqlite/delete_columns_test.go new file mode 100644 index 0000000..c7a6995 --- /dev/null +++ b/db/sqlite/delete_columns_test.go @@ -0,0 +1,188 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestDeleteSubDocumentColumnsExpiry(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "telemetry" + + // Create subdocument with expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Deployed + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify expiry is set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Equal(t, *fetchedSubdoc.Expiry(), futureExpiry) + + // Delete the expiry column + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry") + assert.NilError(t, err) + + // Verify expiry is now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} + +func TestDeleteSubDocumentColumnsMultiple(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "mesh" + + // Create subdocument with expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.InDeployment + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify expiry is set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + + // Delete expiry column + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry") + assert.NilError(t, err) + + // Verify expiry is now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} + +func TestDeleteSubDocumentColumnsEmptyList(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "test" + + // Create a simple subdocument + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcState := common.PendingDownload + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + + fields := log.Fields{} + + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Call with empty column list should be no-op + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId) + assert.NilError(t, err) + + // Verify subdocument is unchanged + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *fetchedSubdoc.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc.State(), srcState) +} + +func TestDeleteSubDocumentColumnsErrorFields(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "telemetry" + + // Create subdocument with error fields and expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Failure + errorCode := 204 + errorDetails := "failed_retrying:Error unsupported namespace" + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with error fields and expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errorCode, &errorDetails) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify all fields are set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Equal(t, *fetchedSubdoc.Expiry(), futureExpiry) + assert.Assert(t, fetchedSubdoc.ErrorCode() != nil) + assert.Equal(t, *fetchedSubdoc.ErrorCode(), errorCode) + assert.Assert(t, fetchedSubdoc.ErrorDetails() != nil) + assert.Equal(t, *fetchedSubdoc.ErrorDetails(), errorDetails) + + // Delete expiry and error fields + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry", "error_code", "error_details") + assert.NilError(t, err) + + // Verify deleted columns are now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + assert.Assert(t, fetchedSubdoc2.ErrorCode() == nil) + assert.Assert(t, fetchedSubdoc2.ErrorDetails() == nil) + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} diff --git a/db/sqlite/document.go b/db/sqlite/document.go index 1831df0..8ac879d 100644 --- a/db/sqlite/document.go +++ b/db/sqlite/document.go @@ -14,17 +14,18 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( "database/sql" "fmt" + "strings" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/db" _ "github.com/mattn/go-sqlite3" "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" log "github.com/sirupsen/logrus" ) @@ -32,20 +33,20 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - rows, err := c.Query("SELECT payload,state,updated_time,version,error_code,error_details FROM xpc_group_config WHERE cpe_mac=? AND group_id=?", cpeMac, groupId) + rows, err := c.Query("SELECT payload,state,updated_time,version,error_code,error_details,expiry FROM xpc_group_config WHERE cpe_mac=? AND group_id=?", cpeMac, groupId) if err != nil { return nil, common.NewError(err) } var ns1, ns2 sql.NullString var b1 []byte - var nt1 sql.NullTime + var nt1, nt2 sql.NullTime var ni1, ni2 sql.NullInt64 if !rows.Next() { return nil, sql.ErrNoRows } - err = rows.Scan(&b1, &ni1, &nt1, &ns1, &ni2, &ns2) + err = rows.Scan(&b1, &ni1, &nt1, &ns1, &ni2, &ns2, &nt2) defer rows.Close() if err != nil { return nil, common.NewError(err) @@ -53,7 +54,7 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su var s1, s2 *string var i1, i2 *int - var ts *int + var ts, expiry *int if ns1.Valid { s1 = &(ns1.String) } @@ -65,6 +66,11 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su tt := int(t1.UnixNano() / 1000000) ts = &tt } + if nt2.Valid { + t2 := nt2.Time + tt := int(t2.UnixNano() / 1000000) + expiry = &tt + } if ni1.Valid { ii := int(ni1.Int64) i1 = &ii @@ -74,7 +80,24 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su i2 = &ii } + // Check if payload contains a reference to a refsubdocument + if refId, ok := db.GetRefId(b1); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + if !c.IsDbNotFound(err) { + return nil, common.NewError(err) + } + // If refsubdocument not found, continue with the reference payload + } else { + // Replace payload with the actual payload from refsubdocument + b1 = refsubdocument.Payload() + } + } + doc := common.NewSubDocument(b1, s1, i1, ts, i2, s2) + if expiry != nil { + doc.SetExpiry(expiry) + } return doc, nil } @@ -101,6 +124,18 @@ func (c *SqliteClient) insertSubDocument(cpeMac string, groupId string, doc *com columns = append(columns, "updated_time") values = append(values, doc.UpdatedTime()) } + if doc.Expiry() != nil { + columns = append(columns, "expiry") + values = append(values, doc.Expiry()) + } + if doc.ErrorCode() != nil { + columns = append(columns, "error_code") + values = append(values, doc.ErrorCode()) + } + if doc.ErrorDetails() != nil { + columns = append(columns, "error_details") + values = append(values, doc.ErrorDetails()) + } qstr := fmt.Sprintf("INSERT INTO xpc_group_config(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) stmt, err := c.Prepare(qstr) if err != nil { @@ -137,6 +172,14 @@ func (c *SqliteClient) updateSubDocument(cpeMac string, groupId string, doc *com columns = append(columns, "updated_time") values = append(values, doc.UpdatedTime()) } + if doc.ErrorCode() != nil { + columns = append(columns, "error_code") + values = append(values, doc.ErrorCode()) + } + if doc.ErrorDetails() != nil { + columns = append(columns, "error_details") + values = append(values, doc.ErrorDetails()) + } values = append(values, cpeMac) values = append(values, groupId) qstr := fmt.Sprintf("UPDATE xpc_group_config SET %v WHERE cpe_mac=? AND group_id=?", db.GetSetColumnsStr(columns)) @@ -219,6 +262,32 @@ func (c *SqliteClient) DeleteSubDocument(cpeMac string, groupId string) error { return nil } +func (c *SqliteClient) DeleteSubDocumentColumns(cpeMac string, groupId string, columns ...string) error { + if len(columns) == 0 { + return nil + } + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // Build UPDATE statement to set columns to NULL + updateExprs := make([]string, len(columns)) + for i, col := range columns { + updateExprs[i] = col + "=NULL" + } + qstr := fmt.Sprintf("UPDATE xpc_group_config SET %v WHERE cpe_mac=? AND group_id=?", strings.Join(updateExprs, ",")) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(cpeMac, groupId) + if err != nil { + return common.NewError(err) + } + return nil +} + func (c *SqliteClient) DeleteDocument(cpeMac string) error { c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() diff --git a/db/sqlite/document_test.go b/db/sqlite/document_test.go index ea6e15e..78d99ed 100644 --- a/db/sqlite/document_test.go +++ b/db/sqlite/document_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -123,3 +123,107 @@ func TestDbReadDocument(t *testing.T) { _, err = tdbclient.GetDocument(cpeMac) assert.Assert(t, tdbclient.IsDbNotFound(err)) } + +func TestGetSubDocumentWithReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "lan" + refId := util.GetMurmur3Hash([]byte(cpeMac + subdocId)) + + // Step 1: Create a reference subdocument with actual payload + actualPayload := common.RandomBytes(100, 200) + actualVersion := util.GetMurmur3Hash(actualPayload) + refSubdoc := common.NewRefSubDocument(actualPayload, &actualVersion) + + err := tdbclient.SetRefSubDocument(refId, refSubdoc) + assert.NilError(t, err) + + // Step 2: Create a subdocument with reference payload (4 zero bytes + refId) + referencePayload := append(make([]byte, 4), []byte(refId)...) + refVersion := util.GetMurmur3Hash(referencePayload) + refState := common.InDeployment + refUpdatedTime := int(time.Now().UnixMilli()) + + subdocWithRef := common.NewSubDocument(referencePayload, &refVersion, &refState, &refUpdatedTime, nil, nil) + fields := log.Fields{} + err = tdbclient.SetSubDocument(cpeMac, subdocId, subdocWithRef, fields) + assert.NilError(t, err) + + // Step 3: Call GetSubDocument and verify it returns the actual payload, not the reference + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is the actual payload from refsubdocument, not the reference + assert.DeepEqual(t, fetchedSubdoc.Payload(), actualPayload) + + // Verify other fields remain unchanged + assert.Equal(t, *fetchedSubdoc.Version(), refVersion) + assert.Equal(t, *fetchedSubdoc.State(), refState) + assert.Equal(t, *fetchedSubdoc.UpdatedTime(), refUpdatedTime) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) +} + +func TestGetSubDocumentWithMissingReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "wan" + refId := util.GetMurmur3Hash([]byte(cpeMac + subdocId + "nonexistent")) + + // Create a subdocument with reference payload pointing to non-existent refsubdocument + referencePayload := append(make([]byte, 4), []byte(refId)...) + refVersion := util.GetMurmur3Hash(referencePayload) + refState := common.InDeployment + refUpdatedTime := int(time.Now().UnixMilli()) + + subdocWithRef := common.NewSubDocument(referencePayload, &refVersion, &refState, &refUpdatedTime, nil, nil) + fields := log.Fields{} + err := tdbclient.SetSubDocument(cpeMac, subdocId, subdocWithRef, fields) + assert.NilError(t, err) + + // Call GetSubDocument - should return the reference payload since refsubdocument doesn't exist + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is the reference payload (since refsubdocument was not found) + assert.DeepEqual(t, fetchedSubdoc.Payload(), referencePayload) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) +} + +func TestGetSubDocumentWithoutReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "mesh" + + // Create a regular subdocument without any reference + regularPayload := common.RandomBytes(100, 200) + regularVersion := util.GetMurmur3Hash(regularPayload) + regularState := common.Deployed + regularUpdatedTime := int(time.Now().UnixMilli()) + + subdoc := common.NewSubDocument(regularPayload, ®ularVersion, ®ularState, ®ularUpdatedTime, nil, nil) + fields := log.Fields{} + err := tdbclient.SetSubDocument(cpeMac, subdocId, subdoc, fields) + assert.NilError(t, err) + + // Call GetSubDocument - should return the regular payload unchanged + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is unchanged + assert.DeepEqual(t, fetchedSubdoc.Payload(), regularPayload) + assert.Equal(t, *fetchedSubdoc.Version(), regularVersion) + assert.Equal(t, *fetchedSubdoc.State(), regularState) + assert.Equal(t, *fetchedSubdoc.UpdatedTime(), regularUpdatedTime) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) +} diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 81c368a..de7ff10 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -39,6 +39,7 @@ var ( state int, error_code int, error_details text, + expiry timestamp, PRIMARY KEY (cpe_mac, group_id) )`, `CREATE TABLE IF NOT EXISTS root_document ( diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 56ee4f5..9b9b86c 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -43,10 +43,12 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string - stateCorrectionEnabled bool - lockRootDocumentEnabled bool + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool + supplementaryPrecookEnabled bool + supplementaryPrecookStateTTLDays int } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -62,6 +64,8 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") + supplementaryPrecookEnabled := conf.GetBoolean("webconfig.supplementary_precook_enabled") + supplementaryPrecookStateTTLDays := int(conf.GetInt32("webconfig.supplementary_precook_state_ttl_days", 7)) db, err := sql.Open("sqlite3", dbfile) if err != nil { @@ -69,11 +73,13 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, - lockRootDocumentEnabled: lockRootDocumentEnabled, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, + supplementaryPrecookEnabled: supplementaryPrecookEnabled, + supplementaryPrecookStateTTLDays: supplementaryPrecookStateTTLDays, }, nil } @@ -182,3 +188,19 @@ func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClie return tdbclient, nil } + +func (c *SqliteClient) SupplementaryPrecookEnabled() bool { + return c.supplementaryPrecookEnabled +} + +func (c *SqliteClient) SetSupplementaryPrecookEnabled(enabled bool) { + c.supplementaryPrecookEnabled = enabled +} + +func (c *SqliteClient) SupplementaryPrecookStateTTLDays() int { + return c.supplementaryPrecookStateTTLDays +} + +func (c *SqliteClient) SetSupplementaryPrecookStateTTLDays(days int) { + c.supplementaryPrecookStateTTLDays = days +} diff --git a/go.mod b/go.mod index 95108f9..8b8a232 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,21 @@ module github.com/rdkcentral/webconfig -go 1.21 +go 1.24.0 + +toolchain go1.24.11 require ( github.com/IBM/sarama v1.42.1 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 - github.com/golang-jwt/jwt/v5 v5.0.0 + github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.0 github.com/mattn/go-sqlite3 v1.14.15 github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.3 github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 @@ -24,7 +26,7 @@ require ( go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.18.0 gotest.tools v2.2.0+incompatible ) @@ -63,10 +65,10 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index 74c2010..a8f6d45 100644 --- a/go.sum +++ b/go.sum @@ -101,8 +101,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -269,8 +269,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -333,8 +333,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -401,8 +401,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -420,8 +420,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -464,8 +464,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -478,8 +478,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 8dace6d..5561b85 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" @@ -53,6 +54,100 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r return } + // Check if supplementary_precook feature is enabled + if s.SupplementaryPrecookEnabled() { + // Check state from xpc_group_config by cpe_mac and group_id=telemetry + telemetrySubdoc, err := s.GetSubDocument(mac, "telemetry") + if err != nil && !s.IsDbNotFound(err) { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + + if telemetrySubdoc != nil && telemetrySubdoc.State() != nil { + state := *telemetrySubdoc.State() + + // If state=1 (Deployed), check expiry first before returning 304 + if state == common.Deployed { + // Check if expiry has passed + if telemetrySubdoc.Expiry() != nil { + currentTime := int(time.Now().UnixNano() / 1000000) + if *telemetrySubdoc.Expiry() <= currentTime { + // Expiry has passed, continue with normal xconf flow + // Do not return 304 + } else { + // Expiry has not passed, return 304 + w.WriteHeader(http.StatusNotModified) + return + } + } else { + // No expiry set, return 304 + w.WriteHeader(http.StatusNotModified) + return + } + } + + // If state in (2, 3, 4), use the data from the "telemetry" row as response + if state == common.PendingDownload || state == common.InDeployment || state == common.Failure { + if telemetrySubdoc.Payload() != nil && len(telemetrySubdoc.Payload()) > 0 { // Update state to InDeployment (3) to indicate the data is being delivered + newState := common.InDeployment + updatedTime := int(time.Now().UnixNano() / 1000000) + errorCode := 0 + errorDetails := "" + newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) + // Note: Not setting expiry here means it won't be updated in the database + + labels, err := s.DatabaseClient.GetRootDocumentLabels(mac) + if err == nil { + labels["client"] = "default" + _ = s.DatabaseClient.SetSubDocument(mac, "telemetry", newSubdoc, state, labels, fields) + // Clear expiry and error fields when transitioning to InDeployment + columnsToDelete := []string{} + if telemetrySubdoc.Expiry() != nil { + columnsToDelete = append(columnsToDelete, "expiry") + } + if telemetrySubdoc.ErrorCode() != nil { + columnsToDelete = append(columnsToDelete, "error_code") + } + if telemetrySubdoc.ErrorDetails() != nil { + columnsToDelete = append(columnsToDelete, "error_details") + } + if len(columnsToDelete) > 0 { + _ = s.DatabaseClient.DeleteSubDocumentColumns(mac, "telemetry", columnsToDelete...) + } + } + // Build multipart response from stored payload (already in msgpack format) + version := "" + if telemetrySubdoc.Version() != nil { + version = *telemetrySubdoc.Version() + } + mpart := common.Multipart{ + Bytes: telemetrySubdoc.Payload(), + Version: version, + Name: "telemetry", + State: state, + } + mparts := []common.Multipart{mpart} + fields["telemetry_version"] = version + respBytes, err := common.WriteMultipartBytes(mparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + rootVersion := util.GetRandomRootVersion() + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + w.Header().Set(common.HeaderEtag, rootVersion) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + return + } + } + + // If state == 0 or no payload, continue with existing code + } + } + // append the extra query_params if any var rootdoc *common.RootDocument var queryParams string diff --git a/http/supplementary_precook_test.go b/http/supplementary_precook_test.go new file mode 100644 index 0000000..186d8f5 --- /dev/null +++ b/http/supplementary_precook_test.go @@ -0,0 +1,711 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestSupplementaryPrecookConfigFlags(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + + // Test default values + assert.Equal(t, server.SupplementaryPrecookEnabled(), false) + assert.Equal(t, server.SupplementaryPrecookStateTTLDays(), 7) + + // Test setter for SupplementaryPrecookEnabled + server.SetSupplementaryPrecookEnabled(true) + assert.Equal(t, server.SupplementaryPrecookEnabled(), true) + + server.SetSupplementaryPrecookEnabled(false) + assert.Equal(t, server.SupplementaryPrecookEnabled(), false) + + // Test setter for SupplementaryPrecookStateTTLDays + server.SetSupplementaryPrecookStateTTLDays(14) + assert.Equal(t, server.SupplementaryPrecookStateTTLDays(), 14) + + server.SetSupplementaryPrecookStateTTLDays(0) + assert.Equal(t, server.SupplementaryPrecookStateTTLDays(), 0) +} + +func TestSupplementaryPrecookStateDeployed(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup telemetry subdoc with state=Deployed ==== + telemetryBytes := common.RandomBytes(100, 150) + telemetryVersion := util.GetMurmur3Hash(telemetryBytes) + telemetryState := common.Deployed + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(telemetryBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler returns 304 ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) +} + +func TestSupplementaryPrecookStatePendingDownload(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload from mock profile response ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.PendingDownload + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler returns cached data ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) +} + +func TestSupplementaryPrecookStateInDeployment(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload from mock profile response ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.InDeployment + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler returns cached data ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) +} + +func TestSupplementaryPrecookStateFailure(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload from mock profile response ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.Failure + telemetryUpdatedTime := int(time.Now().UnixMilli()) + errorCode := 500 + errorDetails := "test error" + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, &errorCode, &errorDetails) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler returns cached data ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) +} + +func TestSupplementaryPrecookDisabled(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Ensure supplementary precook feature is disabled + server.SetSupplementaryPrecookEnabled(false) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup telemetry subdoc with state=Deployed ==== + telemetryBytes := common.RandomBytes(100, 150) + telemetryVersion := util.GetMurmur3Hash(telemetryBytes) + telemetryState := common.Deployed + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(telemetryBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler uses normal flow (not 304) ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + + // Should not return 304, should fetch from mock server + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) +} + +func TestSupplementaryPrecookStateUpdate(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload with state=PendingDownload ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.PendingDownload + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify initial state is PendingDownload ==== + subdoc, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.PendingDownload) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + + // ==== verify state is updated to InDeployment ==== + subdoc, err = server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.InDeployment) +} + +func TestSupplementaryPrecookStateUpdateFromFailure(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload with state=Failure ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.Failure + telemetryUpdatedTime := int(time.Now().UnixMilli()) + errorCode := 500 + errorDetails := "test error" + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, &errorCode, &errorDetails) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify initial state is Failure ==== + subdoc, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Failure) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + + // ==== verify state is updated to InDeployment ==== + subdoc, err = server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.InDeployment) +} + +func TestSupplementaryPrecookStateNotUpdatedWhenDeployed(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload with state=Deployed ==== + telemetryBytes := common.RandomBytes(100, 150) + telemetryVersion := util.GetMurmur3Hash(telemetryBytes) + telemetryState := common.Deployed + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(telemetryBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify initial state is Deployed ==== + subdoc, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // ==== verify state is still Deployed (not updated) ==== + subdoc, err = server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) +} + +func TestSupplementaryPrecookStateDeployedWithExpiredTTL(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup mock xconf server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup telemetry subdoc with state=Deployed but expired TTL ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.Deployed + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + // Set expiry to a past time (1 hour ago in milliseconds) + expiredTime := int(time.Now().Add(-1 * time.Hour).UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + telemetrySubdoc.SetExpiry(&expiredTime) + + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify the subdoc has state=Deployed and expired expiry ==== + subdoc, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + assert.Assert(t, subdoc.Expiry() != nil) + assert.Assert(t, *subdoc.Expiry() < int(time.Now().UnixMilli())) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + + // ==== expectation: should NOT return 304, should fetch from xconf ==== + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart from xconf + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) +} + +func TestSupplementaryPrecookExpiryDeletedOnStateTransition(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create root document for labels ==== + queryParams := "model=TG1682G&partner=comcast" + fwVersion := "TG1682_3.14p9s6_PROD_sey" + modelName := "TG1682G" + partnerId := "comcast" + rootDoc := common.NewRootDocument(0, fwVersion, modelName, partnerId, "", "", queryParams) + err := server.SetRootDocument(cpeMac, rootDoc) + assert.NilError(t, err) + + // ==== create telemetry payload from mock profile response with state=PendingDownload and expiry set ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.PendingDownload + telemetryUpdatedTime := int(time.Now().UnixMilli()) + futureExpiry := int(time.Now().Add(24 * time.Hour).UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + telemetrySubdoc.SetExpiry(&futureExpiry) + fields := make(log.Fields) + err = server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // Verify expiry is set before the request + subdocBefore, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Assert(t, subdocBefore.Expiry() != nil) + assert.Equal(t, *subdocBefore.Expiry(), futureExpiry) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) + + // ==== verify state was updated to InDeployment ==== + subdocAfter, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdocAfter.State(), common.InDeployment) + + // ==== verify expiry was deleted ==== + assert.Assert(t, subdocAfter.Expiry() == nil) +} + +func TestSupplementaryPrecookErrorFieldsDeletedOnStateTransition(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create root document for labels ==== + queryParams := "model=TG1682G&partner=comcast" + fwVersion := "TG1682_3.14p9s6_PROD_sey" + modelName := "TG1682G" + partnerId := "comcast" + rootDoc := common.NewRootDocument(0, fwVersion, modelName, partnerId, "", "", queryParams) + err := server.SetRootDocument(cpeMac, rootDoc) + assert.NilError(t, err) + + // ==== create telemetry payload with state=Failure (state=4) and error fields set ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.Failure + telemetryUpdatedTime := int(time.Now().UnixMilli()) + errorCode := 204 + errorDetails := "failed_retrying:Error unsupported namespace" + futureExpiry := int(time.Now().Add(24 * time.Hour).UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, &errorCode, &errorDetails) + telemetrySubdoc.SetExpiry(&futureExpiry) + fields := make(log.Fields) + err = server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // Verify expiry and error fields are set before the request + subdocBefore, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Assert(t, subdocBefore.Expiry() != nil) + assert.Equal(t, *subdocBefore.Expiry(), futureExpiry) + assert.Assert(t, subdocBefore.ErrorCode() != nil) + assert.Equal(t, *subdocBefore.ErrorCode(), errorCode) + assert.Assert(t, subdocBefore.ErrorDetails() != nil) + assert.Equal(t, *subdocBefore.ErrorDetails(), errorDetails) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) + + // ==== verify state was updated to InDeployment ==== + subdocAfter, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdocAfter.State(), common.InDeployment) + + // ==== verify expiry and error fields were deleted ==== + assert.Assert(t, subdocAfter.Expiry() == nil) + // Note: Cassandra returns default values (0, "") for error_code and error_details after DELETE + assert.Assert(t, subdocAfter.ErrorCode() != nil) + assert.Equal(t, *subdocAfter.ErrorCode(), 0) + assert.Assert(t, subdocAfter.ErrorDetails() != nil) + assert.Equal(t, *subdocAfter.ErrorDetails(), "") +} From d880dad6ea119a17fd99ad4ea1deddf7152dc96f Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Feb 2026 16:07:12 -0800 Subject: [PATCH 188/215] Add support for kafka secure connection --- .gitignore | 6 +- README.md | 92 ++++++++ common/kafka_tls.go | 135 ++++++++++++ common/kafka_tls_test.go | 403 ++++++++++++++++++++++++++++++++++ config/sample_webconfig.conf | 49 +++++ http/webconfig_server.go | 11 + kafka/kafka_consumer_group.go | 14 +- 7 files changed, 707 insertions(+), 3 deletions(-) create mode 100644 common/kafka_tls.go create mode 100644 common/kafka_tls_test.go diff --git a/.gitignore b/.gitignore index 85981f5..8e767f7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,12 @@ # Dependency directories (remove the comment below to include it) # vendor/ -# ##################### +# ignored files bin/* .vscode/* .env *.sh + +# Ignore OpenSpec generated files and folders +openspec/* +.github/* diff --git a/README.md b/README.md index c489fb1..7ef6617 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,98 @@ The actual kafka topic names can be any. Below is just an example. "topics" shou } ``` +#### Kafka TLS/SSL Configuration + +Webconfig supports secure TLS/SSL connections to Kafka brokers for both consumers and producers. This is recommended for production environments to ensure data encryption in transit and proper authentication. + +**TLS Configuration Options:** + +- `tls.enabled` - Enable/disable TLS for Kafka connections (default: false) +- `tls.cert_file` - Path to client certificate file for mTLS authentication (optional) +- `tls.key_file` - Path to client private key file for mTLS authentication (optional) +- `tls.ca_cert_file` - Path to CA certificate file for broker verification (optional) +- `tls.insecure_skip_verify` - Skip certificate verification (insecure, for testing only, default: false) + +**Consumer TLS Configuration Example:** + +```shell + kafka { + enabled = true + brokers = "kafka-broker:9093" # Use secure port + topics = "config-version-report" + consumer_group = "webconfig" + + tls { + enabled = true + cert_file = "/etc/webconfig/kafka/client.crt" + key_file = "/etc/webconfig/kafka/client.key" + ca_cert_file = "/etc/webconfig/kafka/ca.crt" + insecure_skip_verify = false + } + + # Per-cluster TLS configuration + clusters { + mesh { + enabled = true + brokers = "kafka-mesh:9093" + topics = "staging-chi-onewifi-from-device" + + tls { + enabled = true + cert_file = "/etc/webconfig/kafka/mesh-client.crt" + key_file = "/etc/webconfig/kafka/mesh-client.key" + ca_cert_file = "/etc/webconfig/kafka/mesh-ca.crt" + } + } + } + } +``` + +**Producer TLS Configuration Example:** + +```shell + kafka_producer { + enabled = true + brokers = "kafka-broker:9093" + topic = "webconfig_downstream" + + tls { + enabled = true + cert_file = "/etc/webconfig/kafka/producer-client.crt" + key_file = "/etc/webconfig/kafka/producer-client.key" + ca_cert_file = "/etc/webconfig/kafka/ca.crt" + } + } +``` + +**Certificate Requirements:** + +1. **Client Certificate (mTLS)**: If `cert_file` and `key_file` are provided, mutual TLS authentication is enabled. The certificate and key must be in PEM format. + +2. **CA Certificate**: If `ca_cert_file` is provided, it will be used to verify the Kafka broker's certificate. This is useful when using self-signed certificates or internal CAs. + +3. **Certificate Validation**: All certificate files are validated at startup. The application will fail to start with clear error messages if: + - Certificate files are missing or unreadable + - Certificates are in invalid format + - Certificate and key don't match + +**TLS Security Best Practices:** + +1. **Use TLS in Production**: Always enable TLS for production Kafka connections to encrypt data in transit +2. **Use mTLS**: Provide client certificates (`cert_file` and `key_file`) for mutual authentication +3. **Verify Certificates**: Never use `insecure_skip_verify = true` in production - it disables certificate verification and is insecure +4. **Protect Certificate Files**: Set appropriate file permissions (0600) on certificate and key files +5. **Use Secure Ports**: Configure Kafka brokers to listen on secure ports (typically 9093 for TLS) +6. **Certificate Rotation**: Plan for certificate rotation - the service must be restarted to pick up new certificates + +**Troubleshooting TLS Issues:** + +- Check logs for TLS-related errors during startup +- Verify certificate file paths are correct and files are readable +- Ensure Kafka brokers are configured to accept TLS connections +- Test certificate validity: `openssl x509 -in client.crt -text -noout` +- Verify certificate and key match: `openssl x509 -noout -modulus -in client.crt | openssl md5` vs `openssl rsa -noout -modulus -in client.key | openssl md5` + ### Configuration for database The main database operations are defined as an interface. Any driver that implements the interface should work. We has implemented using sqlite, cassandra and yugabytedb. After the db is properly configured, the dbinit.cql can be used to create the tables for cassandra. diff --git a/common/kafka_tls.go b/common/kafka_tls.go new file mode 100644 index 0000000..2619ad3 --- /dev/null +++ b/common/kafka_tls.go @@ -0,0 +1,135 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package common + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + + "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" +) + +// LoadKafkaTLSConfig loads TLS configuration from HOCON config at the specified prefix. +// Prefix should be like "webconfig.kafka" or "webconfig.kafka.clusters.mesh" or "webconfig.kafka_producer". +// Returns nil if TLS is not enabled. +// Returns error if TLS is enabled but configuration is invalid. +func LoadKafkaTLSConfig(conf *configuration.Config, prefix string) (*tls.Config, error) { + tlsEnabled := conf.GetBoolean(prefix + ".tls.enabled") + if !tlsEnabled { + return nil, nil + } + + tlsConfig := &tls.Config{} + + // Check insecure_skip_verify flag first + insecureSkipVerify := conf.GetBoolean(prefix + ".tls.insecure_skip_verify") + + // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) + certFile := conf.GetString(prefix + ".tls.cert_file") + keyFile := conf.GetString(prefix + ".tls.key_file") + + // When insecure_skip_verify is true and no cert files configured, skip loading certificates + // This allows TLS without client authentication (server-only TLS) + if insecureSkipVerify && (len(certFile) == 0 || len(keyFile) == 0) { + // Insecure mode without client certificates - skip cert loading + } else if len(certFile) > 0 && len(keyFile) > 0 { + // Only validate cert files exist when verification is enabled + if !insecureSkipVerify { + // Validate certificate file exists + if _, err := os.Stat(certFile); os.IsNotExist(err) { + return nil, NewError(fmt.Errorf("TLS certificate file does not exist: %s", certFile)) + } + + // Validate key file exists + if _, err := os.Stat(keyFile); os.IsNotExist(err) { + return nil, NewError(fmt.Errorf("TLS key file does not exist: %s", keyFile)) + } + } + + // Load and parse the certificate and key + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, NewError(fmt.Errorf("failed to load TLS certificate and key from %s and %s: %v", certFile, keyFile, err)) + } + + tlsConfig.Certificates = []tls.Certificate{cert} + log.WithFields(log.Fields{ + "prefix": prefix, + "cert_file": certFile, + "key_file": keyFile, + }).Info("Loaded TLS client certificate for mTLS") + } else if len(certFile) > 0 || len(keyFile) > 0 { + // Partial cert configuration detected - require both cert and key when verification is enabled + if !insecureSkipVerify { + return nil, NewError(fmt.Errorf("TLS enabled with verification but incomplete certificate configuration (cert: %s, key: %s)", certFile, keyFile)) + } + } + + // Load CA certificate if provided (optional when insecure_skip_verify is true) + caCertFile := conf.GetString(prefix + ".tls.ca_cert_file") + // When insecure_skip_verify is true and no CA file configured, skip loading CA cert + // This allows TLS without broker verification (insecure mode) + if insecureSkipVerify && len(caCertFile) == 0 { + // Insecure mode without CA cert - skip CA loading + } else if len(caCertFile) > 0 { + // Only validate CA cert file exists when verification is enabled + if !insecureSkipVerify { + // Validate CA certificate file exists + if _, err := os.Stat(caCertFile); os.IsNotExist(err) { + return nil, NewError(fmt.Errorf("TLS CA certificate file does not exist: %s", caCertFile)) + } + } + + // Load CA certificate + caCert, err := os.ReadFile(caCertFile) + if err != nil { + return nil, NewError(fmt.Errorf("failed to read TLS CA certificate from %s: %v", caCertFile, err)) + } + + // Parse CA certificate + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, NewError(fmt.Errorf("failed to parse TLS CA certificate from %s", caCertFile)) + } + + tlsConfig.RootCAs = caCertPool + log.WithFields(log.Fields{ + "prefix": prefix, + "ca_cert_file": caCertFile, + }).Info("Loaded TLS CA certificate for broker verification") + } + + if insecureSkipVerify { + tlsConfig.InsecureSkipVerify = true + log.WithFields(log.Fields{ + "prefix": prefix, + }).Warn("TLS certificate verification is disabled (insecure_skip_verify=true). This is insecure and should only be used for testing.") + } + + log.WithFields(log.Fields{ + "prefix": prefix, + "has_client_cert": len(tlsConfig.Certificates) > 0, + "has_ca_cert": tlsConfig.RootCAs != nil, + "insecure_skip_verify": insecureSkipVerify, + }).Info("TLS configuration loaded for Kafka connection") + + return tlsConfig, nil +} diff --git a/common/kafka_tls_test.go b/common/kafka_tls_test.go new file mode 100644 index 0000000..5be2cd6 --- /dev/null +++ b/common/kafka_tls_test.go @@ -0,0 +1,403 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package common + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-akka/configuration" + "gotest.tools/assert" +) + +// Helper function to generate a self-signed certificate for testing +func generateTestCertificate(t *testing.T, certFile, keyFile string) { + // Generate private key + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + assert.NilError(t, err) + + // Create certificate template + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Test Org"}, + CommonName: "localhost", + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + + // Create self-signed certificate + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + assert.NilError(t, err) + + // Write certificate to file + certOut, err := os.Create(certFile) + assert.NilError(t, err) + defer certOut.Close() + err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + assert.NilError(t, err) + + // Write private key to file + keyOut, err := os.Create(keyFile) + assert.NilError(t, err) + defer keyOut.Close() + privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey) + err = pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: privateKeyBytes}) + assert.NilError(t, err) +} + +func TestLoadKafkaTLSConfig_Disabled(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls { + enabled = false + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig == nil, "TLS config should be nil when TLS is disabled") +} + +func TestLoadKafkaTLSConfig_EnabledWithoutCerts(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls { + enabled = true + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil when TLS is enabled") + assert.Assert(t, len(tlsConfig.Certificates) == 0, "Should have no client certificates") + assert.Assert(t, tlsConfig.RootCAs == nil, "Should have no custom CA") + assert.Assert(t, !tlsConfig.InsecureSkipVerify, "Should not skip verification by default") +} + +func TestLoadKafkaTLSConfig_WithClientCertificates(t *testing.T) { + // Create temp directory for test certificates + tempDir := t.TempDir() + certFile := filepath.Join(tempDir, "client.crt") + keyFile := filepath.Join(tempDir, "client.key") + + // Generate test certificate + generateTestCertificate(t, certFile, keyFile) + + confStr := ` +webconfig { + kafka { + tls { + enabled = true + cert_file = "` + certFile + `" + key_file = "` + keyFile + `" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, len(tlsConfig.Certificates) == 1, "Should have one client certificate") + assert.Assert(t, tlsConfig.RootCAs == nil, "Should have no custom CA") +} + +func TestLoadKafkaTLSConfig_WithCACertificate(t *testing.T) { + // Create temp directory for test certificates + tempDir := t.TempDir() + caCertFile := filepath.Join(tempDir, "ca.crt") + caKeyFile := filepath.Join(tempDir, "ca.key") + + // Generate CA certificate + generateTestCertificate(t, caCertFile, caKeyFile) + + confStr := ` +webconfig { + kafka { + tls { + enabled = true + ca_cert_file = "` + caCertFile + `" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, len(tlsConfig.Certificates) == 0, "Should have no client certificates") + assert.Assert(t, tlsConfig.RootCAs != nil, "Should have custom CA") +} + +func TestLoadKafkaTLSConfig_WithInsecureSkipVerify(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls { + enabled = true + insecure_skip_verify = true + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, tlsConfig.InsecureSkipVerify, "Should skip verification when configured") +} + +func TestLoadKafkaTLSConfig_MissingCertFile(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls { + enabled = true + cert_file = "/nonexistent/client.crt" + key_file = "/nonexistent/client.key" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.Assert(t, err != nil, "Should return error for missing certificate file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "does not exist") +} + +func TestLoadKafkaTLSConfig_MissingCertFileWithInsecureSkipVerify(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls { + enabled = true + insecure_skip_verify = true + cert_file = "/nonexistent/client.crt" + key_file = "/nonexistent/client.key" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + // When insecure_skip_verify is true, missing cert files should not cause validation errors on file check, + // but will still fail when trying to actually load them + assert.Assert(t, err != nil, "Should return error trying to load non-existent files") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "failed to load TLS certificate") +} + +func TestLoadKafkaTLSConfig_InvalidCertFile(t *testing.T) { + // Create temp directory for test files + tempDir := t.TempDir() + certFile := filepath.Join(tempDir, "invalid.crt") + keyFile := filepath.Join(tempDir, "invalid.key") + + // Create invalid certificate files + err := os.WriteFile(certFile, []byte("invalid certificate content"), 0644) + assert.NilError(t, err) + err = os.WriteFile(keyFile, []byte("invalid key content"), 0644) + assert.NilError(t, err) + + confStr := ` +webconfig { + kafka { + tls { + enabled = true + cert_file = "` + certFile + `" + key_file = "` + keyFile + `" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.Assert(t, err != nil, "Should return error for invalid certificate file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "failed to load TLS certificate") +} + +func TestLoadKafkaTLSConfig_MissingCACertFile(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls { + enabled = true + ca_cert_file = "/nonexistent/ca.crt" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.Assert(t, err != nil, "Should return error for missing CA certificate file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "does not exist") +} + +func TestLoadKafkaTLSConfig_MissingCACertFileWithInsecureSkipVerify(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls { + enabled = true + insecure_skip_verify = true + ca_cert_file = "/nonexistent/ca.crt" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + // When insecure_skip_verify is true, missing CA file will not cause validation error on file check, + // but will still fail when trying to actually read it + assert.Assert(t, err != nil, "Should return error trying to load non-existent CA file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "failed to read TLS CA certificate") +} + +func TestLoadKafkaTLSConfig_InvalidCACertFile(t *testing.T) { + // Create temp directory for test files + tempDir := t.TempDir() + caCertFile := filepath.Join(tempDir, "invalid_ca.crt") + + // Create invalid CA certificate file + err := os.WriteFile(caCertFile, []byte("invalid CA certificate content"), 0644) + assert.NilError(t, err) + + confStr := ` +webconfig { + kafka { + tls { + enabled = true + ca_cert_file = "` + caCertFile + `" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.Assert(t, err != nil, "Should return error for invalid CA certificate file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "failed to parse TLS CA certificate") +} + +func TestLoadKafkaTLSConfig_FullConfiguration(t *testing.T) { + // Create temp directory for test certificates + tempDir := t.TempDir() + certFile := filepath.Join(tempDir, "client.crt") + keyFile := filepath.Join(tempDir, "client.key") + caCertFile := filepath.Join(tempDir, "ca.crt") + caKeyFile := filepath.Join(tempDir, "ca.key") + + // Generate test certificates + generateTestCertificate(t, certFile, keyFile) + generateTestCertificate(t, caCertFile, caKeyFile) + + confStr := ` +webconfig { + kafka { + tls { + enabled = true + cert_file = "` + certFile + `" + key_file = "` + keyFile + `" + ca_cert_file = "` + caCertFile + `" + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, len(tlsConfig.Certificates) == 1, "Should have one client certificate") + assert.Assert(t, tlsConfig.RootCAs != nil, "Should have custom CA") + assert.Assert(t, !tlsConfig.InsecureSkipVerify, "Should not skip verification") +} + +func TestLoadKafkaTLSConfig_DifferentPrefixes(t *testing.T) { + // Test with producer prefix + confStr := ` +webconfig { + kafka_producer { + tls { + enabled = true + insecure_skip_verify = true + } + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka_producer") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, tlsConfig.InsecureSkipVerify) + + // Test with cluster prefix + confStr2 := ` +webconfig { + kafka { + clusters { + mesh { + tls { + enabled = true + } + } + } + } +} +` + conf2 := configuration.ParseString(confStr2) + tlsConfig2, err := LoadKafkaTLSConfig(conf2, "webconfig.kafka.clusters.mesh") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig2 != nil, "TLS config should not be nil for cluster") +} diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 33f3bf0..aa1caef 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -206,6 +206,22 @@ webconfig { messages_per_second = 10 } + // TLS configuration for secure Kafka connections + tls { + enabled = false + // Path to client certificate file (PEM format) + // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) + // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) + cert_file = "/etc/webconfig/kafka/client-cert.pem" + // Path to client private key file (PEM format) + key_file = "/etc/webconfig/kafka/client-key.pem" + // Path to CA certificate file for broker verification (PEM format) + ca_cert_file = "/etc/webconfig/kafka/ca-cert.pem" + // Skip certificate verification (WARNING: use ONLY in dev/test environments, NEVER in production) + // When true, certificate files become optional (TLS without client authentication) + insecure_skip_verify = false + } + // if we want to use more than 1 cluster clusters { mesh { @@ -220,6 +236,17 @@ webconfig { ratelimit { messages_per_second = 10 } + + // TLS configuration for mesh cluster + tls { + enabled = false + // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) + // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) + cert_file = "/etc/webconfig/kafka/mesh-client-cert.pem" + key_file = "/etc/webconfig/kafka/mesh-client-key.pem" + ca_cert_file = "/etc/webconfig/kafka/mesh-ca-cert.pem" + insecure_skip_verify = false + } } east { enabled = false @@ -233,6 +260,17 @@ webconfig { ratelimit { messages_per_second = 10 } + + // TLS configuration for east cluster + tls { + enabled = false + // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) + // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) + cert_file = "/etc/webconfig/kafka/east-client-cert.pem" + key_file = "/etc/webconfig/kafka/east-client-key.pem" + ca_cert_file = "/etc/webconfig/kafka/east-ca-cert.pem" + insecure_skip_verify = false + } } } } @@ -257,6 +295,17 @@ webconfig { enabled = false brokers = "localhost:9092" topic = "webconfig_downstream" + + // TLS configuration for Kafka producer + tls { + enabled = false + // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) + // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) + cert_file = "/etc/webconfig/kafka/producer-client-cert.pem" + key_file = "/etc/webconfig/kafka/producer-client-key.pem" + ca_cert_file = "/etc/webconfig/kafka/producer-ca-cert.pem" + insecure_skip_verify = false + } } // this allows the root document locked if needed diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 420dcd9..0309cea 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -277,6 +277,17 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer saramaConfig := sarama.NewConfig() saramaConfig.Producer.Return.Errors = true + + // Load TLS configuration for producer + tlsConfig, err := common.LoadKafkaTLSConfig(conf, "webconfig.kafka_producer") + if err != nil { + panic(fmt.Errorf("failed to load TLS configuration for Kafka producer: %v", err)) + } + if tlsConfig != nil { + saramaConfig.Net.TLS.Enable = true + saramaConfig.Net.TLS.Config = tlsConfig + } + kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) if err != nil { panic(err) diff --git a/kafka/kafka_consumer_group.go b/kafka/kafka_consumer_group.go index c768af7..9cc24e9 100644 --- a/kafka/kafka_consumer_group.go +++ b/kafka/kafka_consumer_group.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package kafka import ( @@ -23,10 +23,10 @@ import ( "time" "github.com/IBM/sarama" + "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" wchttp "github.com/rdkcentral/webconfig/http" - "github.com/go-akka/configuration" ) type KafkaConsumerGroup struct { @@ -103,6 +103,16 @@ func NewKafkaConsumerGroup(conf *configuration.Config, s *wchttp.WebconfigServer } } + // Load TLS configuration + tlsConfig, err := common.LoadKafkaTLSConfig(conf, prefix) + if err != nil { + return nil, common.NewError(fmt.Errorf("failed to load TLS configuration for %s: %v", prefix, err)) + } + if tlsConfig != nil { + sconfig.Net.TLS.Enable = true + sconfig.Net.TLS.Config = tlsConfig + } + consumer := NewConsumer(s, ratelimitMessagesPerSecond, m, clusterName, offsetEnum, topicPartitionsMap) client, err := sarama.NewConsumerGroup(brokers, group, sconfig) From ed01365a1876ca3789482feb08512fea49e10ada Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 9 Feb 2026 14:26:30 -0800 Subject: [PATCH 189/215] Add support to allow skipping table creation during test --- db/cassandra/cassandra_client.go | 23 +++++++++++++++++------ db/sqlite/sqlite_client.go | 26 ++++++++++++++++++-------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index a1e4a85..17e50fe 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "strings" "time" "github.com/go-akka/configuration" @@ -291,14 +292,24 @@ func GetTestCassandraClient(conf *configuration.Config, testOnly bool) (*Cassand if err != nil { return nil, common.NewError(err) } - err = tdbclient.SetUp() - if err != nil { - return nil, common.NewError(err) + + // Check if SKIP_TABLE_CREATION environment variable is set (case-insensitive) + skipTableCreation := false + if skipEnv, exists := os.LookupEnv("SKIP_TABLE_CREATION"); exists { + skipTableCreation = strings.EqualFold(skipEnv, "true") || strings.EqualFold(skipEnv, "1") || strings.EqualFold(skipEnv, "yes") } - err = tdbclient.TearDown() - if err != nil { - return nil, common.NewError(err) + + if !skipTableCreation { + err = tdbclient.SetUp() + if err != nil { + return nil, common.NewError(err) + } + err = tdbclient.TearDown() + if err != nil { + return nil, common.NewError(err) + } } + return tdbclient, nil } diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 9b9b86c..43bf6ee 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -21,6 +21,8 @@ import ( "database/sql" "errors" "fmt" + "os" + "strings" "github.com/go-akka/configuration" _ "github.com/mattn/go-sqlite3" @@ -175,15 +177,23 @@ func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClie return nil, common.NewError(err) } - // need to do it this way to sure we have correct schema but empty tables - if err = tdbclient.SetUp(); err != nil { - return nil, common.NewError(err) + // Check if SKIP_TABLE_CREATION environment variable is set (case-insensitive) + skipTableCreation := false + if skipEnv, exists := os.LookupEnv("SKIP_TABLE_CREATION"); exists { + skipTableCreation = strings.EqualFold(skipEnv, "true") || strings.EqualFold(skipEnv, "1") || strings.EqualFold(skipEnv, "yes") } - if err = tdbclient.TearDown(); err != nil { - return nil, common.NewError(err) - } - if err = tdbclient.SetUp(); err != nil { - return nil, common.NewError(err) + + // need to do it this way to sure we have correct schema but empty tables + if !skipTableCreation { + if err = tdbclient.SetUp(); err != nil { + return nil, common.NewError(err) + } + if err = tdbclient.TearDown(); err != nil { + return nil, common.NewError(err) + } + if err = tdbclient.SetUp(); err != nil { + return nil, common.NewError(err) + } } return tdbclient, nil From 742d2e201d9fe9ec0001233ffbfc4a3aac6da7fe Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 9 Feb 2026 15:25:19 -0800 Subject: [PATCH 190/215] Rollback changes for sqlite. Skipping table creation during tests is only allowed in cassandra db --- db/sqlite/sqlite_client.go | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 43bf6ee..9b9b86c 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -21,8 +21,6 @@ import ( "database/sql" "errors" "fmt" - "os" - "strings" "github.com/go-akka/configuration" _ "github.com/mattn/go-sqlite3" @@ -177,23 +175,15 @@ func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClie return nil, common.NewError(err) } - // Check if SKIP_TABLE_CREATION environment variable is set (case-insensitive) - skipTableCreation := false - if skipEnv, exists := os.LookupEnv("SKIP_TABLE_CREATION"); exists { - skipTableCreation = strings.EqualFold(skipEnv, "true") || strings.EqualFold(skipEnv, "1") || strings.EqualFold(skipEnv, "yes") - } - // need to do it this way to sure we have correct schema but empty tables - if !skipTableCreation { - if err = tdbclient.SetUp(); err != nil { - return nil, common.NewError(err) - } - if err = tdbclient.TearDown(); err != nil { - return nil, common.NewError(err) - } - if err = tdbclient.SetUp(); err != nil { - return nil, common.NewError(err) - } + if err = tdbclient.SetUp(); err != nil { + return nil, common.NewError(err) + } + if err = tdbclient.TearDown(); err != nil { + return nil, common.NewError(err) + } + if err = tdbclient.SetUp(); err != nil { + return nil, common.NewError(err) } return tdbclient, nil From 1e2d01ccb5902d124940abf9c5524ef0d3baa613 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 9 Feb 2026 22:16:44 -0800 Subject: [PATCH 191/215] Update the sample Cassandra configuration to use a non-SSL connection --- config/sample_webconfig.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index aa1caef..b9f4d55 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -170,7 +170,7 @@ webconfig { page_size = 50 user = "dbuser" test_keyspace = "test_webconfig" - is_ssl_enabled = true + is_ssl_enabled = false } yugabyte { @@ -189,7 +189,7 @@ webconfig { test_keyspace = "test_yugabytedb" // TODO change to false for CI/CD - is_ssl_enabled = true + is_ssl_enabled = false } } From 72dee333c31af0f2b7588dd50a233b08a94da37a Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 9 Feb 2026 22:41:37 -0800 Subject: [PATCH 192/215] Revert "Update the sample Cassandra configuration to use a non-SSL connection" This reverts commit 1e2d01ccb5902d124940abf9c5524ef0d3baa613. --- config/sample_webconfig.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index b9f4d55..aa1caef 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -170,7 +170,7 @@ webconfig { page_size = 50 user = "dbuser" test_keyspace = "test_webconfig" - is_ssl_enabled = false + is_ssl_enabled = true } yugabyte { @@ -189,7 +189,7 @@ webconfig { test_keyspace = "test_yugabytedb" // TODO change to false for CI/CD - is_ssl_enabled = false + is_ssl_enabled = true } } From bc8cf479efe67e2d6f442f0a16f61d73ce8bb97f Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Feb 2026 16:11:26 -0800 Subject: [PATCH 193/215] Add cassandra config for tls as we upgraded the dependency libs in go.mod --- config/sample_webconfig.conf | 30 ++++++++ db/cassandra/cassandra_client.go | 116 +++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index aa1caef..84c6849 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -171,6 +171,21 @@ webconfig { user = "dbuser" test_keyspace = "test_webconfig" is_ssl_enabled = true + + // TLS/SSL configuration (optional when is_ssl_enabled = true) + // If tls block is not provided or incomplete, will use insecure TLS + tls { + // Client certificate for mutual TLS (mTLS) - optional + cert_file = "/path/to/client-cert.pem" + key_file = "/path/to/client-key.pem" + + // CA certificate for server verification - optional + ca_cert_file = "/path/to/ca-cert.pem" + + // Skip certificate verification (INSECURE - for testing only) + // When true, allows TLS without certificates or CA validation + insecure_skip_verify = false + } } yugabyte { @@ -190,6 +205,21 @@ webconfig { // TODO change to false for CI/CD is_ssl_enabled = true + + // TLS/SSL configuration (optional when is_ssl_enabled = true) + // If tls block is not provided or incomplete, will use insecure TLS + tls { + // Client certificate for mutual TLS (mTLS) - optional + cert_file = "/path/to/client-cert.pem" + key_file = "/path/to/client-key.pem" + + // CA certificate for server verification - optional + ca_cert_file = "/path/to/ca-cert.pem" + + // Skip certificate verification (INSECURE - for testing only) + // When true, allows TLS without certificates or CA validation + insecure_skip_verify = false + } } } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 17e50fe..65335e3 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -18,6 +18,8 @@ package cassandra import ( + "crypto/tls" + "crypto/x509" "errors" "fmt" "os" @@ -30,6 +32,7 @@ import ( "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" ) const ( @@ -142,7 +145,13 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl } if isSslEnabled { + tlsConfig, err := loadCassandraTLSConfig(dbconf, dbdriver) + if err != nil { + return nil, common.NewError(err) + } + sslOpts := &gocql.SslOptions{ + Config: tlsConfig, EnableHostVerification: false, } cluster.SslOpts = sslOpts @@ -184,6 +193,113 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl }, nil } +// loadCassandraTLSConfig loads TLS configuration for Cassandra connection. +// Returns a tls.Config with certificates loaded from the configuration. +// The function expects tls.{} block under the database driver config (cassandra or yugabyte). +func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls.Config, error) { + tlsConfig := &tls.Config{} + + // Check insecure_skip_verify flag first + insecureSkipVerify := dbconf.GetBoolean("tls.insecure_skip_verify") + + // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) + certFile := dbconf.GetString("tls.cert_file") + keyFile := dbconf.GetString("tls.key_file") + + // When insecure_skip_verify is true and no cert files configured, skip loading certificates + // This allows TLS without client authentication (server-only TLS) + if insecureSkipVerify && (len(certFile) == 0 || len(keyFile) == 0) { + // Insecure mode without client certificates - skip cert loading + log.WithFields(log.Fields{ + "driver": dbdriver, + }).Warn("Cassandra TLS enabled in insecure mode without client certificates") + } else if len(certFile) > 0 && len(keyFile) > 0 { + // Only validate cert files exist when verification is enabled + if !insecureSkipVerify { + // Validate certificate file exists + if _, err := os.Stat(certFile); os.IsNotExist(err) { + return nil, fmt.Errorf("Cassandra TLS certificate file does not exist: %s", certFile) + } + + // Validate key file exists + if _, err := os.Stat(keyFile); os.IsNotExist(err) { + return nil, fmt.Errorf("Cassandra TLS key file does not exist: %s", keyFile) + } + } + + // Load and parse the certificate and key + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("failed to load Cassandra TLS certificate and key from %s and %s: %v", certFile, keyFile, err) + } + + tlsConfig.Certificates = []tls.Certificate{cert} + log.WithFields(log.Fields{ + "driver": dbdriver, + "cert_file": certFile, + "key_file": keyFile, + }).Info("Loaded Cassandra TLS client certificate for mTLS") + } else if len(certFile) > 0 || len(keyFile) > 0 { + // Partial cert configuration detected - require both cert and key when verification is enabled + if !insecureSkipVerify { + return nil, fmt.Errorf("Cassandra TLS enabled with verification but incomplete certificate configuration (cert: %s, key: %s)", certFile, keyFile) + } + } + + // Load CA certificate if provided (optional when insecure_skip_verify is true) + caCertFile := dbconf.GetString("tls.ca_cert_file") + // When insecure_skip_verify is true and no CA file configured, skip loading CA cert + // This allows TLS without server verification (insecure mode) + if insecureSkipVerify && len(caCertFile) == 0 { + // Insecure mode without CA cert - skip CA loading + log.WithFields(log.Fields{ + "driver": dbdriver, + }).Warn("Cassandra TLS enabled in insecure mode without CA certificate") + } else if len(caCertFile) > 0 { + // Only validate CA cert file exists when verification is enabled + if !insecureSkipVerify { + // Validate CA certificate file exists + if _, err := os.Stat(caCertFile); os.IsNotExist(err) { + return nil, fmt.Errorf("Cassandra TLS CA certificate file does not exist: %s", caCertFile) + } + } + + // Load CA certificate + caCert, err := os.ReadFile(caCertFile) + if err != nil { + return nil, fmt.Errorf("failed to read Cassandra TLS CA certificate from %s: %v", caCertFile, err) + } + + // Parse CA certificate + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to parse Cassandra TLS CA certificate from %s", caCertFile) + } + + tlsConfig.RootCAs = caCertPool + log.WithFields(log.Fields{ + "driver": dbdriver, + "ca_cert_file": caCertFile, + }).Info("Loaded Cassandra TLS CA certificate for server verification") + } + + if insecureSkipVerify { + tlsConfig.InsecureSkipVerify = true + log.WithFields(log.Fields{ + "driver": dbdriver, + }).Warn("Cassandra TLS certificate verification is disabled (insecure_skip_verify=true). This is insecure and should only be used for testing.") + } + + log.WithFields(log.Fields{ + "driver": dbdriver, + "has_client_cert": len(tlsConfig.Certificates) > 0, + "has_ca_cert": tlsConfig.RootCAs != nil, + "insecure_skip_verify": insecureSkipVerify, + }).Info("TLS configuration loaded for Cassandra connection") + + return tlsConfig, nil +} + func (c *CassandraClient) Codec() *security.AesCodec { return c.AesCodec } From 524862ba0af04040e3b0ede543d37c7653747594 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Feb 2026 18:14:50 -0800 Subject: [PATCH 194/215] Add tls configs to work with a targeted cassandra env --- db/cassandra/cassandra_client.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 65335e3..1fd5531 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -154,6 +154,7 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl Config: tlsConfig, EnableHostVerification: false, } + cluster.SslOpts = sslOpts } @@ -197,14 +198,22 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl // Returns a tls.Config with certificates loaded from the configuration. // The function expects tls.{} block under the database driver config (cassandra or yugabyte). func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls.Config, error) { - tlsConfig := &tls.Config{} - // Check insecure_skip_verify flag first insecureSkipVerify := dbconf.GetBoolean("tls.insecure_skip_verify") // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) certFile := dbconf.GetString("tls.cert_file") keyFile := dbconf.GetString("tls.key_file") + caCertFile := dbconf.GetString("tls.ca_cert_file") + + // Create TLS config with Cassandra 3.11.x compatible cipher suite + // Based on working example that successfully connects to Cassandra 3.11.15 + tlsConfig := &tls.Config{ + CipherSuites: []uint16{ + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + }, + InsecureSkipVerify: insecureSkipVerify, + } // When insecure_skip_verify is true and no cert files configured, skip loading certificates // This allows TLS without client authentication (server-only TLS) @@ -247,7 +256,6 @@ func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls } // Load CA certificate if provided (optional when insecure_skip_verify is true) - caCertFile := dbconf.GetString("tls.ca_cert_file") // When insecure_skip_verify is true and no CA file configured, skip loading CA cert // This allows TLS without server verification (insecure mode) if insecureSkipVerify && len(caCertFile) == 0 { @@ -284,7 +292,6 @@ func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls } if insecureSkipVerify { - tlsConfig.InsecureSkipVerify = true log.WithFields(log.Fields{ "driver": dbdriver, }).Warn("Cassandra TLS certificate verification is disabled (insecure_skip_verify=true). This is insecure and should only be used for testing.") @@ -295,6 +302,7 @@ func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls "has_client_cert": len(tlsConfig.Certificates) > 0, "has_ca_cert": tlsConfig.RootCAs != nil, "insecure_skip_verify": insecureSkipVerify, + "cipher_suites": len(tlsConfig.CipherSuites), }).Info("TLS configuration loaded for Cassandra connection") return tlsConfig, nil From 0892bd0d9cdeeeb92ce86127d69f0de507d77179 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Feb 2026 19:18:40 -0800 Subject: [PATCH 195/215] Fix a bug that wronge hocon paths wwere used to read kafka tls configs --- common/kafka_tls.go | 10 ++-- common/kafka_tls_test.go | 94 +++++++++++--------------------- db/cassandra/cassandra_client.go | 6 +- 3 files changed, 39 insertions(+), 71 deletions(-) diff --git a/common/kafka_tls.go b/common/kafka_tls.go index 2619ad3..bcc182a 100644 --- a/common/kafka_tls.go +++ b/common/kafka_tls.go @@ -32,7 +32,7 @@ import ( // Returns nil if TLS is not enabled. // Returns error if TLS is enabled but configuration is invalid. func LoadKafkaTLSConfig(conf *configuration.Config, prefix string) (*tls.Config, error) { - tlsEnabled := conf.GetBoolean(prefix + ".tls.enabled") + tlsEnabled := conf.GetBoolean(prefix + ".tls_enabled") if !tlsEnabled { return nil, nil } @@ -40,11 +40,11 @@ func LoadKafkaTLSConfig(conf *configuration.Config, prefix string) (*tls.Config, tlsConfig := &tls.Config{} // Check insecure_skip_verify flag first - insecureSkipVerify := conf.GetBoolean(prefix + ".tls.insecure_skip_verify") + insecureSkipVerify := conf.GetBoolean(prefix + ".tls_insecure_skip_verify") // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) - certFile := conf.GetString(prefix + ".tls.cert_file") - keyFile := conf.GetString(prefix + ".tls.key_file") + certFile := conf.GetString(prefix + ".tls_cert_file") + keyFile := conf.GetString(prefix + ".tls_key_file") // When insecure_skip_verify is true and no cert files configured, skip loading certificates // This allows TLS without client authentication (server-only TLS) @@ -84,7 +84,7 @@ func LoadKafkaTLSConfig(conf *configuration.Config, prefix string) (*tls.Config, } // Load CA certificate if provided (optional when insecure_skip_verify is true) - caCertFile := conf.GetString(prefix + ".tls.ca_cert_file") + caCertFile := conf.GetString(prefix + ".tls_ca_cert_file") // When insecure_skip_verify is true and no CA file configured, skip loading CA cert // This allows TLS without broker verification (insecure mode) if insecureSkipVerify && len(caCertFile) == 0 { diff --git a/common/kafka_tls_test.go b/common/kafka_tls_test.go index 5be2cd6..688d07d 100644 --- a/common/kafka_tls_test.go +++ b/common/kafka_tls_test.go @@ -77,9 +77,7 @@ func TestLoadKafkaTLSConfig_Disabled(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = false - } + tls_enabled = false } } ` @@ -94,9 +92,7 @@ func TestLoadKafkaTLSConfig_EnabledWithoutCerts(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - } + tls_enabled = true } } ` @@ -122,11 +118,9 @@ func TestLoadKafkaTLSConfig_WithClientCertificates(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - cert_file = "` + certFile + `" - key_file = "` + keyFile + `" - } + tls_enabled = true + tls_cert_file = "` + certFile + `" + tls_key_file = "` + keyFile + `" } } ` @@ -151,10 +145,8 @@ func TestLoadKafkaTLSConfig_WithCACertificate(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - ca_cert_file = "` + caCertFile + `" - } + tls_enabled = true + tls_ca_cert_file = "` + caCertFile + `" } } ` @@ -171,10 +163,8 @@ func TestLoadKafkaTLSConfig_WithInsecureSkipVerify(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - insecure_skip_verify = true - } + tls_enabled = true + tls_insecure_skip_verify = true } } ` @@ -190,11 +180,9 @@ func TestLoadKafkaTLSConfig_MissingCertFile(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - cert_file = "/nonexistent/client.crt" - key_file = "/nonexistent/client.key" - } + tls_enabled = true + tls_cert_file = "/nonexistent/client.crt" + tls_key_file = "/nonexistent/client.key" } } ` @@ -210,12 +198,10 @@ func TestLoadKafkaTLSConfig_MissingCertFileWithInsecureSkipVerify(t *testing.T) confStr := ` webconfig { kafka { - tls { - enabled = true - insecure_skip_verify = true - cert_file = "/nonexistent/client.crt" - key_file = "/nonexistent/client.key" - } + tls_enabled = true + tls_insecure_skip_verify = true + tls_cert_file = "/nonexistent/client.crt" + tls_key_file = "/nonexistent/client.key" } } ` @@ -244,11 +230,9 @@ func TestLoadKafkaTLSConfig_InvalidCertFile(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - cert_file = "` + certFile + `" - key_file = "` + keyFile + `" - } + tls_enabled = true + tls_cert_file = "` + certFile + `" + tls_key_file = "` + keyFile + `" } } ` @@ -264,10 +248,8 @@ func TestLoadKafkaTLSConfig_MissingCACertFile(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - ca_cert_file = "/nonexistent/ca.crt" - } + tls_enabled = true + tls_ca_cert_file = "/nonexistent/ca.crt" } } ` @@ -283,11 +265,9 @@ func TestLoadKafkaTLSConfig_MissingCACertFileWithInsecureSkipVerify(t *testing.T confStr := ` webconfig { kafka { - tls { - enabled = true - insecure_skip_verify = true - ca_cert_file = "/nonexistent/ca.crt" - } + tls_enabled = true + tls_insecure_skip_verify = true + tls_ca_cert_file = "/nonexistent/ca.crt" } } ` @@ -313,10 +293,8 @@ func TestLoadKafkaTLSConfig_InvalidCACertFile(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - ca_cert_file = "` + caCertFile + `" - } + tls_enabled = true + tls_ca_cert_file = "` + caCertFile + `" } } ` @@ -343,12 +321,10 @@ func TestLoadKafkaTLSConfig_FullConfiguration(t *testing.T) { confStr := ` webconfig { kafka { - tls { - enabled = true - cert_file = "` + certFile + `" - key_file = "` + keyFile + `" - ca_cert_file = "` + caCertFile + `" - } + tls_enabled = true + tls_cert_file = "` + certFile + `" + tls_key_file = "` + keyFile + `" + tls_ca_cert_file = "` + caCertFile + `" } } ` @@ -367,10 +343,8 @@ func TestLoadKafkaTLSConfig_DifferentPrefixes(t *testing.T) { confStr := ` webconfig { kafka_producer { - tls { - enabled = true - insecure_skip_verify = true - } + tls_enabled = true + tls_insecure_skip_verify = true } } ` @@ -387,9 +361,7 @@ webconfig { kafka { clusters { mesh { - tls { - enabled = true - } + tls_enabled = true } } } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 1fd5531..b91d8b2 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -206,12 +206,8 @@ func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls keyFile := dbconf.GetString("tls.key_file") caCertFile := dbconf.GetString("tls.ca_cert_file") - // Create TLS config with Cassandra 3.11.x compatible cipher suite - // Based on working example that successfully connects to Cassandra 3.11.15 + // Create TLS config for Cassandra connection tlsConfig := &tls.Config{ - CipherSuites: []uint16{ - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - }, InsecureSkipVerify: insecureSkipVerify, } From 5a41e7864cd52a11b88de47ea1ece3a4592b52eb Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Feb 2026 20:27:53 -0800 Subject: [PATCH 196/215] Add back cassandra ciphersuite configs removed by mistake --- db/cassandra/cassandra_client.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index b91d8b2..d46eb15 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -206,8 +206,12 @@ func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls keyFile := dbconf.GetString("tls.key_file") caCertFile := dbconf.GetString("tls.ca_cert_file") - // Create TLS config for Cassandra connection + // Create TLS config for Cassandra connection with compatible cipher suite + // Cassandra 3.11.x requires specific cipher suites that are disabled by default in newer Go crypto tlsConfig := &tls.Config{ + CipherSuites: []uint16{ + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + }, InsecureSkipVerify: insecureSkipVerify, } From 2cf1f2390296df88954d9780e86bbff3a59d8605 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 18 Mar 2026 16:20:16 -0700 Subject: [PATCH 197/215] fix a bug that error_code and error_details columns were not reset when POST new payload and state changed from 4 to 2 --- db/service.go | 4 ++ db/sqlite/state_update_test.go | 79 ++++++++++++++++++++++++++++++++++ http/document_handler.go | 4 +- http/document_handler_test.go | 44 +++++++++++++++++++ 4 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 db/sqlite/state_update_test.go diff --git a/db/service.go b/db/service.go index 80efde1..a9eccdb 100644 --- a/db/service.go +++ b/db/service.go @@ -528,6 +528,10 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old newState := common.InDeployment newSubdoc.SetState(&newState) + zeroErrorCode := 0 + emptyErrorDetails := "" + newSubdoc.SetErrorCode(&zeroErrorCode) + newSubdoc.SetErrorDetails(&emptyErrorDetails) err = c.SetSubDocument(cpeMac, subdocId, newSubdoc, oldState, labels, fields) if err != nil { diff --git a/db/sqlite/state_update_test.go b/db/sqlite/state_update_test.go new file mode 100644 index 0000000..6cc8008 --- /dev/null +++ b/db/sqlite/state_update_test.go @@ -0,0 +1,79 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package sqlite + +import ( + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +// TestUpdateSubDocumentResetsErrorFields verifies that transitioning a subdocument +// to InDeployment (state 3) via UpdateSubDocument clears any stale error_code and +// error_details left from a prior Failure (state 4). +func TestUpdateSubDocumentResetsErrorFields(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "privatessid" + + // step 1: seed a root document so GetRootDocumentLabels succeeds + rootdoc := common.NewRootDocument(0, "", "", "", "", "", "", "", "") + err := tdbclient.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + // step 2: write a subdoc in Failure state with non-zero error fields + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Failure + errCode := 204 + errDetails := "failed_retrying:Error unsupported namespace" + failureSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errCode, &errDetails) + fields := log.Fields{} + err = tdbclient.SetSubDocument(cpeMac, groupId, failureSubdoc, fields) + assert.NilError(t, err) + + // verify failure state and error fields persisted + fetched, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *fetched.State(), common.Failure) + assert.Equal(t, *fetched.ErrorCode(), 204) + assert.Equal(t, *fetched.ErrorDetails(), "failed_retrying:Error unsupported namespace") + + // step 3: call UpdateSubDocument (simulating upstream fetch, 2→3 transition) + // newSubdoc represents fresh config from upstream — no state/error fields set + newBytes := common.RandomBytes(100, 150) + newVersion := util.GetMurmur3Hash(newBytes) + newSubdoc := common.NewSubDocument(newBytes, &newVersion, nil, nil, nil, nil) + + // empty versionMap so UpdateSubDocument does not skip via early-return path + versionMap := make(map[string]string) + err = db.UpdateSubDocument(tdbclient, cpeMac, groupId, newSubdoc, failureSubdoc, versionMap, fields) + assert.NilError(t, err) + + // step 4: verify state advanced to InDeployment and error fields are reset + fetched, err = tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *fetched.State(), common.InDeployment) + assert.Equal(t, *fetched.ErrorCode(), 0) + assert.Equal(t, *fetched.ErrorDetails(), "") +} diff --git a/http/document_handler.go b/http/document_handler.go index e07890f..e833c8a 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -135,7 +135,9 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. } updatedTime := int(time.Now().UnixNano() / 1000000) - subdoc := common.NewSubDocument(bbytes, &version, statePtr, &updatedTime, nil, nil) + zeroErrorCode := 0 + emptyErrorDetails := "" + subdoc := common.NewSubDocument(bbytes, &version, statePtr, &updatedTime, &zeroErrorCode, &emptyErrorDetails) // handle expiry header expiryTmsStr := r.Header.Get(common.HeaderSubdocumentExpiry) diff --git a/http/document_handler_test.go b/http/document_handler_test.go index dc263fc..0be8cf0 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -613,3 +613,47 @@ func TestBadHeaderExpiryHandler(t *testing.T) { assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) } + +func TestPostSubDocumentResetsErrorFields(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + subdocId := "lan" + + // step 1: write a subdoc in Failure state with non-zero error fields + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Failure + errCode := 204 + errDetails := "failed_retrying:Error unsupported namespace" + failureSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errCode, &errDetails) + err := server.SetSubDocument(cpeMac, subdocId, failureSubdoc) + assert.NilError(t, err) + + // verify failure state persisted correctly + fetched, err := server.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Equal(t, *fetched.State(), common.Failure) + assert.Equal(t, *fetched.ErrorCode(), 204) + assert.Equal(t, *fetched.ErrorDetails(), "failed_retrying:Error unsupported namespace") + + // step 2: POST new config via HTTP handler (4 → 2 transition) + newBytes := common.RandomBytes(100, 150) + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(newBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // step 3: verify state is PendingDownload and error fields are reset to zero + fetched, err = server.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Equal(t, *fetched.State(), common.PendingDownload) + assert.Equal(t, *fetched.ErrorCode(), 0) + assert.Equal(t, *fetched.ErrorDetails(), "") +} From 826c144b96f15ba38c640146a694581178421a02 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 19 Mar 2026 16:48:28 -0700 Subject: [PATCH 198/215] handle failed sqlite related tests during root_document schema change --- Makefile | 3 ++ db/sqlite/schema.go | 68 ++++++++++++++++++++++++++++++++++ db/sqlite/sqlite_client.go | 62 +++++++++++++++++++++++++++++++ db/sqlite/state_update_test.go | 2 +- 4 files changed, 134 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 605e139..4c6d513 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,9 @@ BUILDTIME := $(shell date -u +"%F_%T_%Z") all: build testall: test testsqlite testyuga +testenv: ## Run all tests sourcing env.sh for TEST_CONFIG_FILE + bash -c "source env.sh && go test ./... -cover -count=1" + build: ## Build a version go build -v -ldflags="-X ${REPO}/common.BinaryBranch=${BRANCH} -X ${REPO}/common.BinaryVersion=${Version} -X ${REPO}/common.BinaryBuildTime=${BUILDTIME}" -o bin/${PROJ}-${GOOS}-${GOARCH} main.go diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index de7ff10..0d2d4af 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -18,7 +18,10 @@ package sqlite import ( + "fmt" "regexp" + "strings" + "unicode" ) const ( @@ -72,3 +75,68 @@ func init() { } } } + +// parseCreateTable extracts the table name and a map of column-name→type from a +// "CREATE TABLE IF NOT EXISTS" DDL string. It is used by SyncSchema to diff +// expected columns against what PRAGMA table_info reports. +func parseCreateTable(stmt string) (string, map[string]string, error) { + reTableName := regexp.MustCompile(`(?i)CREATE TABLE IF NOT EXISTS (\w+)`) + m := reTableName.FindStringSubmatch(stmt) + if len(m) < 2 { + return "", nil, fmt.Errorf("parseCreateTable: cannot find table name in DDL") + } + tableName := m[1] + + colDefs := make(map[string]string) + for _, rawLine := range strings.Split(stmt, "\n") { + line := strings.TrimSpace(rawLine) + line = strings.TrimRight(line, ",") + line = strings.TrimSpace(line) + if line == "" { + continue + } + upper := strings.ToUpper(line) + // skip the CREATE TABLE header, lone parens, and table-level constraints + if strings.HasPrefix(upper, "CREATE") || + line == "(" || line == ")" || + strings.HasPrefix(upper, "PRIMARY") || + strings.HasPrefix(upper, "UNIQUE") || + strings.HasPrefix(upper, "FOREIGN") || + strings.HasPrefix(upper, "CHECK") || + strings.HasPrefix(upper, "CONSTRAINT") { + continue + } + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + colName := parts[0] + if !isSQLiteIdentifier(colName) { + continue + } + var colType string + if len(parts) > 1 { + t := parts[1] + u := strings.ToUpper(t) + if u != "PRIMARY" && u != "NOT" && u != "UNIQUE" && u != "REFERENCES" { + colType = t + } + } + colDefs[colName] = colType + } + return tableName, colDefs, nil +} + +// isSQLiteIdentifier reports whether s is a valid plain SQL identifier +// (ASCII letters, digits, and underscores only). +func isSQLiteIdentifier(s string) bool { + if len(s) == 0 { + return false + } + for _, ch := range s { + if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' { + return false + } + } + return true +} diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 9b9b86c..78182d6 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -179,6 +179,10 @@ func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClie if err = tdbclient.SetUp(); err != nil { return nil, common.NewError(err) } + // add any new columns that are missing from an existing DB file + if err = tdbclient.SyncSchema(); err != nil { + return nil, common.NewError(err) + } if err = tdbclient.TearDown(); err != nil { return nil, common.NewError(err) } @@ -204,3 +208,61 @@ func (c *SqliteClient) SupplementaryPrecookStateTTLDays() int { func (c *SqliteClient) SetSupplementaryPrecookStateTTLDays(days int) { c.supplementaryPrecookStateTTLDays = days } + +// SyncSchema adds any columns that are defined in SqliteCreateTableStatements but +// missing from the existing tables. It is safe to call on both new and existing DB +// files, and is idempotent — it is a no-op when the schema is already current. +func (c *SqliteClient) SyncSchema() error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + for _, stmt := range SqliteCreateTableStatements { + tableName, colDefs, err := parseCreateTable(stmt) + if err != nil { + return common.NewError(err) + } + + existing, err := c.pragmaTableInfo(tableName) + if err != nil { + return common.NewError(err) + } + + for colName, colType := range colDefs { + if _, ok := existing[colName]; ok { + continue + } + var alterSQL string + if colType == "" { + alterSQL = fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s", tableName, colName) + } else { + alterSQL = fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", tableName, colName, colType) + } + if _, err := c.Exec(alterSQL); err != nil { + return common.NewError(fmt.Errorf("SyncSchema %s.%s: %w", tableName, colName, err)) + } + } + } + return nil +} + +// pragmaTableInfo returns a map of column name → type for an existing table. +// Returns an empty map (not an error) if the table does not exist. +func (c *SqliteClient) pragmaTableInfo(tableName string) (map[string]string, error) { + rows, err := c.Query(fmt.Sprintf("PRAGMA table_info(%s)", tableName)) + if err != nil { + return nil, err + } + defer rows.Close() + + cols := make(map[string]string) + for rows.Next() { + var cid, notNull, pk int + var name, colType string + var dfltValue sql.NullString + if err := rows.Scan(&cid, &name, &colType, ¬Null, &dfltValue, &pk); err != nil { + return nil, err + } + cols[name] = colType + } + return cols, rows.Err() +} diff --git a/db/sqlite/state_update_test.go b/db/sqlite/state_update_test.go index 6cc8008..96a2ac6 100644 --- a/db/sqlite/state_update_test.go +++ b/db/sqlite/state_update_test.go @@ -36,7 +36,7 @@ func TestUpdateSubDocumentResetsErrorFields(t *testing.T) { groupId := "privatessid" // step 1: seed a root document so GetRootDocumentLabels succeeds - rootdoc := common.NewRootDocument(0, "", "", "", "", "", "", "", "") + rootdoc := &common.RootDocument{} err := tdbclient.SetRootDocument(cpeMac, rootdoc) assert.NilError(t, err) From 261241b77bdc826cf36461bae4c42c8a9eaeb804 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Mar 2026 12:25:53 -0700 Subject: [PATCH 199/215] Store new headers X-System-Product-Class and X-System-Type into root_document columns product_class and customer_type --- .gitignore | 5 +- common/const_var.go | 1 + common/document_test.go | 4 +- common/root_document.go | 36 +++++++++++++- common/root_document_test.go | 18 +++---- db/cassandra/root_document.go | 4 +- db/cassandra/root_document_test.go | 12 ++--- db/service.go | 16 +++++- db/service_test.go | 2 +- db/sqlite/root_document.go | 28 +++++++---- db/sqlite/root_document_test.go | 52 ++++++++++++++++++-- db/sqlite/schema.go | 2 + http/multipart.go | 5 +- http/multipart_test.go | 79 ++++++++++++++++++++++++++++++ http/rootdocument_handler_test.go | 70 ++++++++++++++++++++++++-- http/supplementary_handler_test.go | 10 ++-- http/supplementary_precook_test.go | 6 +-- 17 files changed, 295 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index 8e767f7..07683a2 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,6 @@ bin/* *.sh # Ignore OpenSpec generated files and folders -openspec/* -.github/* +openspec/ +.github/ +.claude/ diff --git a/common/const_var.go b/common/const_var.go index 2590d3b..28234bb 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -77,6 +77,7 @@ const ( HeaderPartnerID = "X-System-PartnerID" HeaderAccountID = "X-System-AccountID" HeaderProductClass = "X-System-Product-Class" + HeaderCustomerType = "X-System-Type" HeaderUserAgent = "User-Agent" HeaderSchemaVersion = "X-System-Schema-Version" HeaderMetricsAgent = "X-Metrics-Agent" diff --git a/common/document_test.go b/common/document_test.go index c074aba..c7ea662 100644 --- a/common/document_test.go +++ b/common/document_test.go @@ -35,7 +35,7 @@ func TestDocument(t *testing.T) { modelName := "bar" partnerId := "cox" firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") document = NewDocument(rootdoc) subdocIds := []string{"red", "orange", "yellow", "green"} @@ -77,7 +77,7 @@ func TestFilterByBitmap(t *testing.T) { modelName := "bar" partnerId := "cox" firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") document = NewDocument(rootdoc) subdocIds := []string{ diff --git a/common/root_document.go b/common/root_document.go index 54cf801..925c563 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -38,10 +38,12 @@ type RootDocument struct { Version string `json:"version"` QueryParams string `json:"query_params"` LockedTill int `json:"locked_till"` + ProductClass string `json:"product_class"` + CustomerType string `json:"customer_type"` } -// (bitmap, firmware_version, model_name, partner_id, schema_version, version), nil -func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVersion, version, query_params string) *RootDocument { +// (bitmap, firmware_version, model_name, partner_id, schema_version, version, query_params, product_class, customer_type), nil +func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVersion, version, query_params, productClass, customerType string) *RootDocument { return &RootDocument{ Bitmap: bitmap, FirmwareVersion: firmwareVersion, @@ -50,6 +52,8 @@ func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVe SchemaVersion: schemaVersion, Version: version, QueryParams: query_params, + ProductClass: productClass, + CustomerType: customerType, } } @@ -62,6 +66,8 @@ func (d *RootDocument) ColumnMap() map[string]interface{} { "schema_version": d.SchemaVersion, "version": d.Version, "query_params": d.QueryParams, + "product_class": d.ProductClass, + "customer_type": d.CustomerType, } return dict } @@ -82,6 +88,8 @@ func (d *RootDocument) NonEmptyColumnMap() map[string]interface{} { "schema_version": d.SchemaVersion, "version": d.Version, "query_params": d.QueryParams, + "product_class": d.ProductClass, + "customer_type": d.CustomerType, } for k, v := range tempDict { @@ -109,6 +117,12 @@ func (d *RootDocument) Compare(r *RootDocument) int { if d.SchemaVersion != r.SchemaVersion { return RootDocumentMetaChanged } + if d.ProductClass != r.ProductClass { + return RootDocumentMetaChanged + } + if d.CustomerType != r.CustomerType { + return RootDocumentMetaChanged + } if d.Version != r.Version { return RootDocumentVersionOnlyChanged } @@ -134,6 +148,12 @@ func (d *RootDocument) Equals(r *RootDocument) bool { if d.SchemaVersion != r.SchemaVersion { return false } + if d.ProductClass != r.ProductClass { + return false + } + if d.CustomerType != r.CustomerType { + return false + } return true } @@ -160,6 +180,12 @@ func (d *RootDocument) Update(r *RootDocument) { if len(r.QueryParams) > 0 { d.QueryParams = r.QueryParams } + if len(r.ProductClass) > 0 { + d.ProductClass = r.ProductClass + } + if len(r.CustomerType) > 0 { + d.CustomerType = r.CustomerType + } } func (d *RootDocument) UpdateMetadata(r *RootDocument) { @@ -179,6 +205,12 @@ func (d *RootDocument) UpdateMetadata(r *RootDocument) { if len(r.SchemaVersion) > 0 { d.SchemaVersion = r.SchemaVersion } + if len(r.ProductClass) > 0 { + d.ProductClass = r.ProductClass + } + if len(r.CustomerType) > 0 { + d.CustomerType = r.CustomerType + } } func (d *RootDocument) String() string { diff --git a/common/root_document_test.go b/common/root_document_test.go index 50c149f..41ceaff 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -32,18 +32,18 @@ func TestRootDocumentCompare(t *testing.T) { modelName := "bar" partnerId := "cox" firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") rootdoc2 := rootdoc1.Clone() enum := rootdoc1.Compare(rootdoc2) assert.Equal(t, enum, RootDocumentEquals) firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "") + rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "", "", "") enum = rootdoc1.Compare(rootdoc3) assert.Equal(t, enum, RootDocumentMetaChanged) version4 := "3456" - rootdoc4 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version4, "") + rootdoc4 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version4, "", "", "") enum = rootdoc1.Compare(rootdoc4) assert.Equal(t, enum, RootDocumentVersionOnlyChanged) } @@ -55,7 +55,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName1 := "TG4482" partnerId1 := "" firmwareVersion1 := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc1 := NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "") + rootdoc1 := NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", "", "") bitmap2 := 123 version2 := "bar" @@ -63,7 +63,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName2 := "TG4482" partnerId2 := "cox" firmwareVersion2 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc2 := NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, "") + rootdoc2 := NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, "", "", "") bitmap3 := 123 version3 := "bar" @@ -71,7 +71,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName3 := "TG4482" partnerId3 := "cox" firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc3 := NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, "") + rootdoc3 := NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, "", "", "") rootdoc1.Update(rootdoc2) assert.Equal(t, *rootdoc1, *rootdoc3) @@ -88,13 +88,13 @@ func TestRootDocumentEquals(t *testing.T) { modelName := "bar" partnerId := "cox" firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") rootdoc2 := rootdoc1.Clone() ok := rootdoc1.Equals(rootdoc2) assert.Assert(t, ok) firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "") + rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "", "", "") ok = rootdoc1.Equals(rootdoc3) assert.Assert(t, !ok) } @@ -106,7 +106,7 @@ func TestRootDocumentLocked(t *testing.T) { modelName := "bar" partnerId := "cox" firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") assert.Assert(t, !rootdoc.Locked()) epoch := int(time.Now().UnixMilli()) rootdoc.LockedTill = epoch + 1000 diff --git a/db/cassandra/root_document.go b/db/cassandra/root_document.go index ff9321a..412e4c3 100644 --- a/db/cassandra/root_document.go +++ b/db/cassandra/root_document.go @@ -34,8 +34,8 @@ func (c *CassandraClient) GetRootDocument(cpeMac string) (*common.RootDocument, var rd common.RootDocument var tobj time.Time - stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till FROM root_document WHERE cpe_mac=?" - err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj) + stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till,product_class,customer_type FROM root_document WHERE cpe_mac=?" + err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj, &rd.ProductClass, &rd.CustomerType) if err != nil { return nil, common.NewError(err) } diff --git a/db/cassandra/root_document_test.go b/db/cassandra/root_document_test.go index 98ffe38..d9e5d5f 100644 --- a/db/cassandra/root_document_test.go +++ b/db/cassandra/root_document_test.go @@ -30,7 +30,7 @@ func TestRootDocumentOperations(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() bitmap := 123 version := "foo" - rdoc := common.NewRootDocument(bitmap, "", "", "", "", version, "") + rdoc := common.NewRootDocument(bitmap, "", "", "", "", version, "", "", "") err := tdbclient.SetRootDocument(cpeMac, rdoc) assert.NilError(t, err) @@ -86,7 +86,7 @@ func TestRootDocumentDb(t *testing.T) { // set by a RootDocument version4 := "indigo violet" bitmap4 := 67 - rdoc4 := common.NewRootDocument(bitmap4, "", "", "", "", version4, "") + rdoc4 := common.NewRootDocument(bitmap4, "", "", "", "", version4, "", "", "") err = tdbclient.SetRootDocument(cpeMac, rdoc4) assert.NilError(t, err) fetched, err := tdbclient.GetRootDocument(cpeMac) @@ -134,7 +134,7 @@ func TestRootDocumentUpdate(t *testing.T) { partnerId1 := "" firmwareVersion1 := "TG4482PC2_4.12p7s3_PROD_sey" queryParams1 := "stormReadyWifi=true" - srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, queryParams1) + srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, queryParams1, "", "") err = tdbclient.SetRootDocument(cpeMac, srcRootdoc1) assert.NilError(t, err) @@ -151,7 +151,7 @@ func TestRootDocumentUpdate(t *testing.T) { partnerId2 := "cox" firmwareVersion2 := "TG4482PC2_4.14p7s3_PROD_sey" queryParams2 := "stormReadyWifi=true" - rootdoc2 := common.NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, queryParams2) + rootdoc2 := common.NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, queryParams2, "", "") err = tdbclient.SetRootDocument(cpeMac, rootdoc2) assert.NilError(t, err) @@ -164,7 +164,7 @@ func TestRootDocumentUpdate(t *testing.T) { partnerId3 := "cox" firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" queryParams3 := "stormReadyWifi=true" - rootdoc3 := common.NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, queryParams3) + rootdoc3 := common.NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, queryParams3, "", "") tgtRootdoc3, err := tdbclient.GetRootDocument(cpeMac) assert.NilError(t, err) @@ -181,7 +181,7 @@ func TestRootDocumentLocked(t *testing.T) { partnerId := "cox" firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + rootdoc := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") epoch := int(time.Now().UnixMilli()) rootdoc.LockedTill = epoch + 1000 diff --git a/db/service.go b/db/service.go index a9eccdb..1f51ed4 100644 --- a/db/service.go +++ b/db/service.go @@ -94,8 +94,18 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Warn(err) } + productClass, err := rHeader.Get(common.HeaderProductClass) + if err != nil { + log.WithFields(tfields).Warn(err) + } + + customerType, err := rHeader.Get(common.HeaderCustomerType) + if err != nil { + log.WithFields(tfields).Warn(err) + } + // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers - deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") + deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "", productClass, customerType) // ==== parse mac ==== mac, err := rHeader.Get(common.HeaderDeviceId) @@ -647,9 +657,11 @@ func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerI schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) modelName := rHeader.Get(common.HeaderModelName) firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + productClass := rHeader.Get(common.HeaderProductClass) + customerType := rHeader.Get(common.HeaderCustomerType) // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers - deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") + deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "", productClass, customerType) // ==== read the cloudRootDocument from db ==== cloudRootDocument, err := c.GetRootDocument(mac) diff --git a/db/service_test.go b/db/service_test.go index 65edaf8..79d712b 100644 --- a/db/service_test.go +++ b/db/service_test.go @@ -65,7 +65,7 @@ func TestUpdateRootVersion(t *testing.T) { rootVersion := HashRootVersion(doc.VersionMap()) assert.Assert(t, rootVersion != "0") - rootDoc := common.NewRootDocument(123, "fw_ver_123", "model_123", "partner_123", "", rootVersion, "") + rootDoc := common.NewRootDocument(123, "fw_ver_123", "model_123", "partner_123", "", rootVersion, "", "", "") doc.SetRootDocument(rootDoc) doc.DeleteSubDocument("mesh") diff --git a/db/sqlite/root_document.go b/db/sqlite/root_document.go index d67fbe6..8674966 100644 --- a/db/sqlite/root_document.go +++ b/db/sqlite/root_document.go @@ -14,21 +14,21 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( "database/sql" - "github.com/rdkcentral/webconfig/common" "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" ) func (c *SqliteClient) GetRootDocument(cpeMac string) (*common.RootDocument, error) { c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - rows, err := c.Query("SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version FROM root_document WHERE cpe_mac=?", cpeMac) + rows, err := c.Query("SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,product_class,customer_type FROM root_document WHERE cpe_mac=?", cpeMac) if err != nil { return nil, common.NewError(err) } @@ -38,8 +38,8 @@ func (c *SqliteClient) GetRootDocument(cpeMac string) (*common.RootDocument, err } var ni sql.NullInt64 - var ns1, ns2, ns3, ns4, ns5 sql.NullString - err = rows.Scan(&ni, &ns1, &ns2, &ns3, &ns4, &ns5) + var ns1, ns2, ns3, ns4, ns5, ns6, ns7 sql.NullString + err = rows.Scan(&ni, &ns1, &ns2, &ns3, &ns4, &ns5, &ns6, &ns7) defer rows.Close() if err != nil { return nil, common.NewError(err) @@ -67,7 +67,15 @@ func (c *SqliteClient) GetRootDocument(cpeMac string) (*common.RootDocument, err version = ns5.String } - return common.NewRootDocument(bitmap, firmware_version, model_name, partner_id, schema_version, version, ""), nil + var product_class, customer_type string + if ns6.Valid { + product_class = ns6.String + } + if ns7.Valid { + customer_type = ns7.String + } + + return common.NewRootDocument(bitmap, firmware_version, model_name, partner_id, schema_version, version, "", product_class, customer_type), nil } func (c *SqliteClient) insertRootDocumentVersion(cpeMac, version string) error { @@ -206,12 +214,12 @@ func (c *SqliteClient) insertRootDocument(cpeMac string, rd *common.RootDocument c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - stmt, err := c.Prepare("INSERT INTO root_document(cpe_mac,bitmap,firmware_version,model_name,partner_id,schema_version,version) VALUES(?,?,?,?,?,?,?)") + stmt, err := c.Prepare("INSERT INTO root_document(cpe_mac,bitmap,firmware_version,model_name,partner_id,schema_version,version,product_class,customer_type) VALUES(?,?,?,?,?,?,?,?,?)") if err != nil { return common.NewError(err) } - _, err = stmt.Exec(cpeMac, rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version) + _, err = stmt.Exec(cpeMac, rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, rd.ProductClass, rd.CustomerType) if err != nil { return common.NewError(err) } @@ -222,11 +230,11 @@ func (c *SqliteClient) updateRootDocument(cpeMac string, rd *common.RootDocument c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - stmt, err := c.Prepare("UPDATE root_document SET bitmap=?,firmware_version=?,model_name=?,partner_id=?,schema_version=?,version=? WHERE cpe_mac=?") + stmt, err := c.Prepare("UPDATE root_document SET bitmap=?,firmware_version=?,model_name=?,partner_id=?,schema_version=?,version=?,product_class=?,customer_type=? WHERE cpe_mac=?") if err != nil { return common.NewError(err) } - _, err = stmt.Exec(rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, cpeMac) + _, err = stmt.Exec(rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, rd.ProductClass, rd.CustomerType, cpeMac) if err != nil { return common.NewError(err) } diff --git a/db/sqlite/root_document_test.go b/db/sqlite/root_document_test.go index fdc7933..8632519 100644 --- a/db/sqlite/root_document_test.go +++ b/db/sqlite/root_document_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -71,7 +71,7 @@ func TestRootDocumentDb(t *testing.T) { // set by a RootDocument version4 := "indigo violet" bitmap4 := 67 - rdoc4 := common.NewRootDocument(bitmap4, "", "", "", "", version4, "") + rdoc4 := common.NewRootDocument(bitmap4, "", "", "", "", version4, "", "", "") err = tdbclient.SetRootDocument(cpeMac, rdoc4) assert.NilError(t, err) fetched, err := tdbclient.GetRootDocument(cpeMac) @@ -147,7 +147,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName1 := "TG4482" partnerId1 := "" firmwareVersion1 := "TG4482PC2_4.12p7s3_PROD_sey" - srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "") + srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", "", "") err = tdbclient.SetRootDocument(cpeMac, srcRootdoc1) assert.NilError(t, err) @@ -163,7 +163,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName2 := "TG4482" partnerId2 := "cox" firmwareVersion2 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc2 := common.NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, "") + rootdoc2 := common.NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, "", "", "") err = tdbclient.SetRootDocument(cpeMac, rootdoc2) assert.NilError(t, err) @@ -175,9 +175,51 @@ func TestRootDocumentUpdate(t *testing.T) { modelName3 := "TG4482" partnerId3 := "cox" firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc3 := common.NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, "") + rootdoc3 := common.NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, "", "", "") tgtRootdoc3, err := tdbclient.GetRootDocument(cpeMac) assert.NilError(t, err) assert.DeepEqual(t, tgtRootdoc3, rootdoc3) } + +func TestRootDocumentProductClassCustomerType(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // verify starting empty + _, err := tdbclient.GetRootDocument(cpeMac) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // ==== step 1: set rootdoc with product_class and customer_type ==== + bitmap1 := 100 + version1 := "v1" + schemaVersion1 := "33554433-1.3" + modelName1 := "TG3482G" + partnerId1 := "comcast" + firmwareVersion1 := "TG3482G_4.10p7s1_PROD_sey" + productClass1 := "rg" + customerType1 := "residential" + srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", productClass1, customerType1) + + err = tdbclient.SetRootDocument(cpeMac, srcRootdoc1) + assert.NilError(t, err) + + // read from db and verify product_class and customer_type are stored + tgtRootdoc1, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass1, tgtRootdoc1.ProductClass) + assert.Equal(t, customerType1, tgtRootdoc1.CustomerType) + + // ==== step 2: update with new product_class and customer_type ==== + productClass2 := "xb" + customerType2 := "business" + rootdoc2 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", productClass2, customerType2) + + err = tdbclient.SetRootDocument(cpeMac, rootdoc2) + assert.NilError(t, err) + + // verify the updated values are stored + tgtRootdoc2, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass2, tgtRootdoc2.ProductClass) + assert.Equal(t, customerType2, tgtRootdoc2.CustomerType) +} diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 0d2d4af..ea068a7 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -48,10 +48,12 @@ var ( `CREATE TABLE IF NOT EXISTS root_document ( cpe_mac text PRIMARY KEY, bitmap bigint, + customer_type text, firmware_version text, locked_till timestamp, model_name text, partner_id text, + product_class text, query_params text, route text, schema_version, diff --git a/http/multipart.go b/http/multipart.go index e579760..bd154ba 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -38,6 +38,7 @@ var ( "X-System-Schema-Version", "X-System-Supported-Docs", "X-System-Product-Class", + "X-System-Type", "Transaction-Id", } ) @@ -201,7 +202,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } } if document == nil { - rootDocument := common.NewRootDocument(0, "", "", "", "", "", "") + rootDocument := common.NewRootDocument(0, "", "", "", "", "", "", "", "") document = common.NewDocument(rootDocument) } @@ -321,7 +322,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin bitmap = oldRootDocument.Bitmap } // filter by versionMap and filter by blockedIds - finalRootDocument := common.NewRootDocument(bitmap, "", "", "", "", upstreamRespEtag, "") + finalRootDocument := common.NewRootDocument(bitmap, "", "", "", "", upstreamRespEtag, "", "", "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) diff --git a/http/multipart_test.go b/http/multipart_test.go index fe5676e..0eb70f2 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1513,3 +1513,82 @@ func TestValidateQueryParams(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) } + +func TestMultipartConfigHandlerNewHeaders(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== POST lan subdoc ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== GET /config with X-System-Product-Class and X-System-Type headers ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partnerId1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + productClass1 := "rg" + customerType1 := "residential" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partnerId1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + req.Header.Set(common.HeaderProductClass, productClass1) + req.Header.Set(common.HeaderCustomerType, customerType1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // read from db and verify product_class and customer_type are stored + rdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass1, rdoc.ProductClass) + assert.Equal(t, customerType1, rdoc.CustomerType) + + // ==== GET /config again with updated X-System-Product-Class and X-System-Type headers ==== + productClass2 := "xb8" + customerType2 := "business" + + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partnerId1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + req.Header.Set(common.HeaderProductClass, productClass2) + req.Header.Set(common.HeaderCustomerType, customerType2) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + + // verify updated values are persisted in the root_document table + rdoc, err = server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass2, rdoc.ProductClass) + assert.Equal(t, customerType2, rdoc.CustomerType) +} diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index a0db7a3..837f07a 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -81,7 +81,7 @@ func TestRootDocumentHandler(t *testing.T) { expectedBitmap1, err := common.GetCpeBitmap(supportedDocs1) assert.NilError(t, err) - expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") + expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "", "", "") assert.DeepEqual(t, rootdoc, expectedRootdoc) // ==== step 2 build lan subdoc ==== @@ -190,7 +190,7 @@ func TestRootDocumentHandler(t *testing.T) { err = json.Unmarshal(rbytes, &getResp) assert.NilError(t, err) - expectedRootdoc = common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, "") + expectedRootdoc = common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, "", "", "") assert.Equal(t, getResp.Data, *expectedRootdoc) } @@ -207,7 +207,7 @@ func TestPostRootDocumentHandler(t *testing.T) { schemaVersion1 := "33554433-1.3,33554434-1.3" etag := strconv.Itoa(int(time.Now().Unix())) queryParams1 := "stormReadyWifi=true" - srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1) + srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1, "", "") bbytes, err := json.Marshal(srcDoc1) assert.NilError(t, err) @@ -280,7 +280,7 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { expectedBitmap1, err := common.GetCpeBitmap(supportedDocs1) assert.NilError(t, err) - expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") + expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "", "", "") assert.DeepEqual(t, rootdoc, expectedRootdoc) // ==== step 2 ==== @@ -307,3 +307,65 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { assert.NilError(t, err) assert.Assert(t, rootdoc.Equals(rootdoc2)) } + +func TestMultipartConfigHandlerStoresProductClassAndCustomerType(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // make a GET /config request with X-System-Product-Class and X-System-Type headers + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + productClass1 := "rg" + customerType1 := "residential" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + req.Header.Set(common.HeaderProductClass, productClass1) + req.Header.Set(common.HeaderCustomerType, customerType1) + + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + // expect 404 since no subdocs are stored + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db and verify product_class and customer_type are stored + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass1, rootdoc.ProductClass) + assert.Equal(t, customerType1, rootdoc.CustomerType) + + // ==== make a second request with different product_class and customer_type ==== + productClass2 := "xb8" + customerType2 := "business" + req2, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req2.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req2.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req2.Header.Set(common.HeaderModelName, modelName1) + req2.Header.Set(common.HeaderPartnerID, partner1) + req2.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + req2.Header.Set(common.HeaderProductClass, productClass2) + req2.Header.Set(common.HeaderCustomerType, customerType2) + + res2 := ExecuteRequest(req2, router).Result() + _, err = io.ReadAll(res2.Body) + assert.NilError(t, err) + + // verify updated values are stored + rootdoc2, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass2, rootdoc2.ProductClass) + assert.Equal(t, customerType2, rootdoc2.CustomerType) +} diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index f99bd95..533626f 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -459,7 +459,7 @@ func TestSupplementaryWithExtraQueryParams(t *testing.T) { schemaVersion1 := "33554433-1.3,33554434-1.3" etag := strconv.Itoa(int(time.Now().Unix())) queryParams1 := "stormReadyWifi=true" - srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1) + srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1, "", "") bbytes, err := json.Marshal(srcDoc1) assert.NilError(t, err) @@ -649,7 +649,7 @@ func TestSupplementaryAppendingFlag(t *testing.T) { schemaVersion1 := "33554433-1.3,33554434-1.3" etag := strconv.Itoa(int(time.Now().Unix())) queryParams1 := "stormReadyWifi=true" - srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1) + srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1, "", "") bbytes, err := json.Marshal(srcDoc1) assert.NilError(t, err) @@ -1171,7 +1171,7 @@ func TestSupplementaryUpstreamProfilesNotFoundNotDefaultEmptyProfile(t *testing. firmwareVersion := "TG1682_3.14p9s6_PROD_sey" // set up root_document table - rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true") + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true", "", "") err = server.SetRootDocument(cpeMac, rdoc) assert.NilError(t, err) getRdoc, err := server.GetRootDocument(cpeMac) @@ -1235,7 +1235,7 @@ func TestSupplementaryUpstreamProfilesNotFoundDefaultEmptyProfile(t *testing.T) firmwareVersion := "TG1682_3.14p9s6_PROD_sey" // set up root_document table - rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true") + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true", "", "") err = server.SetRootDocument(cpeMac, rdoc) assert.NilError(t, err) getRdoc, err := server.GetRootDocument(cpeMac) @@ -1299,7 +1299,7 @@ func TestSupplementaryDefaultEmptyProfile(t *testing.T) { firmwareVersion := "TG1682_3.14p9s6_PROD_sey" // set up root_document table - rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true") + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true", "", "") err = server.SetRootDocument(cpeMac, rdoc) assert.NilError(t, err) getRdoc, err := server.GetRootDocument(cpeMac) diff --git a/http/supplementary_precook_test.go b/http/supplementary_precook_test.go index 186d8f5..17d0a73 100644 --- a/http/supplementary_precook_test.go +++ b/http/supplementary_precook_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -566,7 +566,7 @@ func TestSupplementaryPrecookExpiryDeletedOnStateTransition(t *testing.T) { fwVersion := "TG1682_3.14p9s6_PROD_sey" modelName := "TG1682G" partnerId := "comcast" - rootDoc := common.NewRootDocument(0, fwVersion, modelName, partnerId, "", "", queryParams) + rootDoc := common.NewRootDocument(0, fwVersion, modelName, partnerId, "", "", queryParams, "", "") err := server.SetRootDocument(cpeMac, rootDoc) assert.NilError(t, err) @@ -641,7 +641,7 @@ func TestSupplementaryPrecookErrorFieldsDeletedOnStateTransition(t *testing.T) { fwVersion := "TG1682_3.14p9s6_PROD_sey" modelName := "TG1682G" partnerId := "comcast" - rootDoc := common.NewRootDocument(0, fwVersion, modelName, partnerId, "", "", queryParams) + rootDoc := common.NewRootDocument(0, fwVersion, modelName, partnerId, "", "", queryParams, "", "") err := server.SetRootDocument(cpeMac, rootDoc) assert.NilError(t, err) From cdb653036b8613bd96efa02f4316f318a540544d Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 31 Mar 2026 14:57:16 -0700 Subject: [PATCH 200/215] Add missing code for customer_type and product_class support --- db/cassandra/schema.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index 6953e84..a6f1bc8 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -38,10 +38,12 @@ var ( `CREATE TABLE IF NOT EXISTS root_document ( cpe_mac text PRIMARY KEY, bitmap bigint, + customer_type text, firmware_version text, locked_till timestamp, model_name text, partner_id text, + product_class text, query_params text, route text, schema_version text, @@ -69,9 +71,11 @@ var ( "root_document": { "cpe_mac": gocql.TypeText, "bitmap": gocql.TypeBigInt, + "customer_type": gocql.TypeText, "firmware_version": gocql.TypeText, "model_name": gocql.TypeText, "partner_id": gocql.TypeText, + "product_class": gocql.TypeText, "route": gocql.TypeText, "schema_version": gocql.TypeText, "version": gocql.TypeText, From c8f8113b41594ab30601cd3b309f31e4866c8331 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 17 Apr 2026 16:50:37 -0700 Subject: [PATCH 201/215] rename column name from customer_type to account_type --- common/const_var.go | 2 +- common/root_document.go | 24 ++++++++++++------------ db/cassandra/root_document.go | 4 ++-- db/cassandra/schema.go | 4 ++-- db/service.go | 8 ++++---- db/sqlite/root_document.go | 16 ++++++++-------- db/sqlite/root_document_test.go | 20 ++++++++++---------- db/sqlite/schema.go | 2 +- http/multipart_test.go | 14 +++++++------- http/rootdocument_handler_test.go | 18 +++++++++--------- 10 files changed, 56 insertions(+), 56 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 28234bb..776cda6 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -77,7 +77,7 @@ const ( HeaderPartnerID = "X-System-PartnerID" HeaderAccountID = "X-System-AccountID" HeaderProductClass = "X-System-Product-Class" - HeaderCustomerType = "X-System-Type" + HeaderAccountType = "X-System-Type" HeaderUserAgent = "User-Agent" HeaderSchemaVersion = "X-System-Schema-Version" HeaderMetricsAgent = "X-Metrics-Agent" diff --git a/common/root_document.go b/common/root_document.go index 925c563..b40296b 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -39,11 +39,11 @@ type RootDocument struct { QueryParams string `json:"query_params"` LockedTill int `json:"locked_till"` ProductClass string `json:"product_class"` - CustomerType string `json:"customer_type"` + AccountType string `json:"account_type"` } -// (bitmap, firmware_version, model_name, partner_id, schema_version, version, query_params, product_class, customer_type), nil -func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVersion, version, query_params, productClass, customerType string) *RootDocument { +// (bitmap, firmware_version, model_name, partner_id, schema_version, version, query_params, product_class, account_type), nil +func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVersion, version, query_params, productClass, accountType string) *RootDocument { return &RootDocument{ Bitmap: bitmap, FirmwareVersion: firmwareVersion, @@ -53,7 +53,7 @@ func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVe Version: version, QueryParams: query_params, ProductClass: productClass, - CustomerType: customerType, + AccountType: accountType, } } @@ -67,7 +67,7 @@ func (d *RootDocument) ColumnMap() map[string]interface{} { "version": d.Version, "query_params": d.QueryParams, "product_class": d.ProductClass, - "customer_type": d.CustomerType, + "account_type": d.AccountType, } return dict } @@ -89,7 +89,7 @@ func (d *RootDocument) NonEmptyColumnMap() map[string]interface{} { "version": d.Version, "query_params": d.QueryParams, "product_class": d.ProductClass, - "customer_type": d.CustomerType, + "account_type": d.AccountType, } for k, v := range tempDict { @@ -120,7 +120,7 @@ func (d *RootDocument) Compare(r *RootDocument) int { if d.ProductClass != r.ProductClass { return RootDocumentMetaChanged } - if d.CustomerType != r.CustomerType { + if d.AccountType != r.AccountType { return RootDocumentMetaChanged } if d.Version != r.Version { @@ -151,7 +151,7 @@ func (d *RootDocument) Equals(r *RootDocument) bool { if d.ProductClass != r.ProductClass { return false } - if d.CustomerType != r.CustomerType { + if d.AccountType != r.AccountType { return false } return true @@ -183,8 +183,8 @@ func (d *RootDocument) Update(r *RootDocument) { if len(r.ProductClass) > 0 { d.ProductClass = r.ProductClass } - if len(r.CustomerType) > 0 { - d.CustomerType = r.CustomerType + if len(r.AccountType) > 0 { + d.AccountType = r.AccountType } } @@ -208,8 +208,8 @@ func (d *RootDocument) UpdateMetadata(r *RootDocument) { if len(r.ProductClass) > 0 { d.ProductClass = r.ProductClass } - if len(r.CustomerType) > 0 { - d.CustomerType = r.CustomerType + if len(r.AccountType) > 0 { + d.AccountType = r.AccountType } } diff --git a/db/cassandra/root_document.go b/db/cassandra/root_document.go index 412e4c3..3a9b0be 100644 --- a/db/cassandra/root_document.go +++ b/db/cassandra/root_document.go @@ -34,8 +34,8 @@ func (c *CassandraClient) GetRootDocument(cpeMac string) (*common.RootDocument, var rd common.RootDocument var tobj time.Time - stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till,product_class,customer_type FROM root_document WHERE cpe_mac=?" - err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj, &rd.ProductClass, &rd.CustomerType) + stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till,product_class,account_type FROM root_document WHERE cpe_mac=?" + err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj, &rd.ProductClass, &rd.AccountType) if err != nil { return nil, common.NewError(err) } diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index a6f1bc8..d216277 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -38,7 +38,7 @@ var ( `CREATE TABLE IF NOT EXISTS root_document ( cpe_mac text PRIMARY KEY, bitmap bigint, - customer_type text, + account_type text, firmware_version text, locked_till timestamp, model_name text, @@ -71,7 +71,7 @@ var ( "root_document": { "cpe_mac": gocql.TypeText, "bitmap": gocql.TypeBigInt, - "customer_type": gocql.TypeText, + "account_type": gocql.TypeText, "firmware_version": gocql.TypeText, "model_name": gocql.TypeText, "partner_id": gocql.TypeText, diff --git a/db/service.go b/db/service.go index 1f51ed4..c4de467 100644 --- a/db/service.go +++ b/db/service.go @@ -99,13 +99,13 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Warn(err) } - customerType, err := rHeader.Get(common.HeaderCustomerType) + accountType, err := rHeader.Get(common.HeaderAccountType) if err != nil { log.WithFields(tfields).Warn(err) } // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers - deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "", productClass, customerType) + deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "", productClass, accountType) // ==== parse mac ==== mac, err := rHeader.Get(common.HeaderDeviceId) @@ -658,10 +658,10 @@ func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerI modelName := rHeader.Get(common.HeaderModelName) firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) productClass := rHeader.Get(common.HeaderProductClass) - customerType := rHeader.Get(common.HeaderCustomerType) + accountType := rHeader.Get(common.HeaderAccountType) // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers - deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "", productClass, customerType) + deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "", productClass, accountType) // ==== read the cloudRootDocument from db ==== cloudRootDocument, err := c.GetRootDocument(mac) diff --git a/db/sqlite/root_document.go b/db/sqlite/root_document.go index 8674966..13a13eb 100644 --- a/db/sqlite/root_document.go +++ b/db/sqlite/root_document.go @@ -28,7 +28,7 @@ func (c *SqliteClient) GetRootDocument(cpeMac string) (*common.RootDocument, err c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - rows, err := c.Query("SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,product_class,customer_type FROM root_document WHERE cpe_mac=?", cpeMac) + rows, err := c.Query("SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,product_class,account_type FROM root_document WHERE cpe_mac=?", cpeMac) if err != nil { return nil, common.NewError(err) } @@ -67,15 +67,15 @@ func (c *SqliteClient) GetRootDocument(cpeMac string) (*common.RootDocument, err version = ns5.String } - var product_class, customer_type string + var product_class, account_type string if ns6.Valid { product_class = ns6.String } if ns7.Valid { - customer_type = ns7.String + account_type = ns7.String } - return common.NewRootDocument(bitmap, firmware_version, model_name, partner_id, schema_version, version, "", product_class, customer_type), nil + return common.NewRootDocument(bitmap, firmware_version, model_name, partner_id, schema_version, version, "", product_class, account_type), nil } func (c *SqliteClient) insertRootDocumentVersion(cpeMac, version string) error { @@ -214,12 +214,12 @@ func (c *SqliteClient) insertRootDocument(cpeMac string, rd *common.RootDocument c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - stmt, err := c.Prepare("INSERT INTO root_document(cpe_mac,bitmap,firmware_version,model_name,partner_id,schema_version,version,product_class,customer_type) VALUES(?,?,?,?,?,?,?,?,?)") + stmt, err := c.Prepare("INSERT INTO root_document(cpe_mac,bitmap,firmware_version,model_name,partner_id,schema_version,version,product_class,account_type) VALUES(?,?,?,?,?,?,?,?,?)") if err != nil { return common.NewError(err) } - _, err = stmt.Exec(cpeMac, rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, rd.ProductClass, rd.CustomerType) + _, err = stmt.Exec(cpeMac, rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, rd.ProductClass, rd.AccountType) if err != nil { return common.NewError(err) } @@ -230,11 +230,11 @@ func (c *SqliteClient) updateRootDocument(cpeMac string, rd *common.RootDocument c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - stmt, err := c.Prepare("UPDATE root_document SET bitmap=?,firmware_version=?,model_name=?,partner_id=?,schema_version=?,version=?,product_class=?,customer_type=? WHERE cpe_mac=?") + stmt, err := c.Prepare("UPDATE root_document SET bitmap=?,firmware_version=?,model_name=?,partner_id=?,schema_version=?,version=?,product_class=?,account_type=? WHERE cpe_mac=?") if err != nil { return common.NewError(err) } - _, err = stmt.Exec(rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, rd.ProductClass, rd.CustomerType, cpeMac) + _, err = stmt.Exec(rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, rd.ProductClass, rd.AccountType, cpeMac) if err != nil { return common.NewError(err) } diff --git a/db/sqlite/root_document_test.go b/db/sqlite/root_document_test.go index 8632519..dec2f00 100644 --- a/db/sqlite/root_document_test.go +++ b/db/sqlite/root_document_test.go @@ -182,14 +182,14 @@ func TestRootDocumentUpdate(t *testing.T) { assert.DeepEqual(t, tgtRootdoc3, rootdoc3) } -func TestRootDocumentProductClassCustomerType(t *testing.T) { +func TestRootDocumentProductClassAccountType(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() // verify starting empty _, err := tdbclient.GetRootDocument(cpeMac) assert.Assert(t, tdbclient.IsDbNotFound(err)) - // ==== step 1: set rootdoc with product_class and customer_type ==== + // ==== step 1: set rootdoc with product_class and account_type ==== bitmap1 := 100 version1 := "v1" schemaVersion1 := "33554433-1.3" @@ -197,22 +197,22 @@ func TestRootDocumentProductClassCustomerType(t *testing.T) { partnerId1 := "comcast" firmwareVersion1 := "TG3482G_4.10p7s1_PROD_sey" productClass1 := "rg" - customerType1 := "residential" - srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", productClass1, customerType1) + accountType1 := "residential" + srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", productClass1, accountType1) err = tdbclient.SetRootDocument(cpeMac, srcRootdoc1) assert.NilError(t, err) - // read from db and verify product_class and customer_type are stored + // read from db and verify product_class and account_type are stored tgtRootdoc1, err := tdbclient.GetRootDocument(cpeMac) assert.NilError(t, err) assert.Equal(t, productClass1, tgtRootdoc1.ProductClass) - assert.Equal(t, customerType1, tgtRootdoc1.CustomerType) + assert.Equal(t, accountType1, tgtRootdoc1.AccountType) - // ==== step 2: update with new product_class and customer_type ==== + // ==== step 2: update with new product_class and account_type ==== productClass2 := "xb" - customerType2 := "business" - rootdoc2 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", productClass2, customerType2) + accountType2 := "business" + rootdoc2 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", productClass2, accountType2) err = tdbclient.SetRootDocument(cpeMac, rootdoc2) assert.NilError(t, err) @@ -221,5 +221,5 @@ func TestRootDocumentProductClassCustomerType(t *testing.T) { tgtRootdoc2, err := tdbclient.GetRootDocument(cpeMac) assert.NilError(t, err) assert.Equal(t, productClass2, tgtRootdoc2.ProductClass) - assert.Equal(t, customerType2, tgtRootdoc2.CustomerType) + assert.Equal(t, accountType2, tgtRootdoc2.AccountType) } diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index ea068a7..e30f7ac 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -48,7 +48,7 @@ var ( `CREATE TABLE IF NOT EXISTS root_document ( cpe_mac text PRIMARY KEY, bitmap bigint, - customer_type text, + account_type text, firmware_version text, locked_till timestamp, model_name text, diff --git a/http/multipart_test.go b/http/multipart_test.go index 0eb70f2..775c1e7 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1545,7 +1545,7 @@ func TestMultipartConfigHandlerNewHeaders(t *testing.T) { partnerId1 := "comcast" schemaVersion1 := "33554433-1.3,33554434-1.3" productClass1 := "rg" - customerType1 := "residential" + accountType1 := "residential" req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) @@ -1553,7 +1553,7 @@ func TestMultipartConfigHandlerNewHeaders(t *testing.T) { req.Header.Set(common.HeaderPartnerID, partnerId1) req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) req.Header.Set(common.HeaderProductClass, productClass1) - req.Header.Set(common.HeaderCustomerType, customerType1) + req.Header.Set(common.HeaderAccountType, accountType1) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1561,15 +1561,15 @@ func TestMultipartConfigHandlerNewHeaders(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // read from db and verify product_class and customer_type are stored + // read from db and verify product_class and account_type are stored rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) assert.Equal(t, productClass1, rdoc.ProductClass) - assert.Equal(t, customerType1, rdoc.CustomerType) + assert.Equal(t, accountType1, rdoc.AccountType) // ==== GET /config again with updated X-System-Product-Class and X-System-Type headers ==== productClass2 := "xb8" - customerType2 := "business" + accountType2 := "business" req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -1579,7 +1579,7 @@ func TestMultipartConfigHandlerNewHeaders(t *testing.T) { req.Header.Set(common.HeaderPartnerID, partnerId1) req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) req.Header.Set(common.HeaderProductClass, productClass2) - req.Header.Set(common.HeaderCustomerType, customerType2) + req.Header.Set(common.HeaderAccountType, accountType2) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1590,5 +1590,5 @@ func TestMultipartConfigHandlerNewHeaders(t *testing.T) { rdoc, err = server.GetRootDocument(cpeMac) assert.NilError(t, err) assert.Equal(t, productClass2, rdoc.ProductClass) - assert.Equal(t, customerType2, rdoc.CustomerType) + assert.Equal(t, accountType2, rdoc.AccountType) } diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 837f07a..f292764 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -308,7 +308,7 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { assert.Assert(t, rootdoc.Equals(rootdoc2)) } -func TestMultipartConfigHandlerStoresProductClassAndCustomerType(t *testing.T) { +func TestMultipartConfigHandlerStoresProductClassAndAccountType(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) cpeMac := util.GenerateRandomCpeMac() @@ -324,7 +324,7 @@ func TestMultipartConfigHandlerStoresProductClassAndCustomerType(t *testing.T) { partner1 := "comcast" schemaVersion1 := "33554433-1.3,33554434-1.3" productClass1 := "rg" - customerType1 := "residential" + accountType1 := "residential" req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) @@ -332,7 +332,7 @@ func TestMultipartConfigHandlerStoresProductClassAndCustomerType(t *testing.T) { req.Header.Set(common.HeaderPartnerID, partner1) req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) req.Header.Set(common.HeaderProductClass, productClass1) - req.Header.Set(common.HeaderCustomerType, customerType1) + req.Header.Set(common.HeaderAccountType, accountType1) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -340,15 +340,15 @@ func TestMultipartConfigHandlerStoresProductClassAndCustomerType(t *testing.T) { // expect 404 since no subdocs are stored assert.Equal(t, res.StatusCode, http.StatusNotFound) - // read from db and verify product_class and customer_type are stored + // read from db and verify product_class and account_type are stored rootdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) assert.Equal(t, productClass1, rootdoc.ProductClass) - assert.Equal(t, customerType1, rootdoc.CustomerType) + assert.Equal(t, accountType1, rootdoc.AccountType) - // ==== make a second request with different product_class and customer_type ==== + // ==== make a second request with different product_class and account_type ==== productClass2 := "xb8" - customerType2 := "business" + accountType2 := "business" req2, err := http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) req2.Header.Set(common.HeaderSupportedDocs, supportedDocs1) @@ -357,7 +357,7 @@ func TestMultipartConfigHandlerStoresProductClassAndCustomerType(t *testing.T) { req2.Header.Set(common.HeaderPartnerID, partner1) req2.Header.Set(common.HeaderSchemaVersion, schemaVersion1) req2.Header.Set(common.HeaderProductClass, productClass2) - req2.Header.Set(common.HeaderCustomerType, customerType2) + req2.Header.Set(common.HeaderAccountType, accountType2) res2 := ExecuteRequest(req2, router).Result() _, err = io.ReadAll(res2.Body) @@ -367,5 +367,5 @@ func TestMultipartConfigHandlerStoresProductClassAndCustomerType(t *testing.T) { rootdoc2, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) assert.Equal(t, productClass2, rootdoc2.ProductClass) - assert.Equal(t, customerType2, rootdoc2.CustomerType) + assert.Equal(t, accountType2, rootdoc2.AccountType) } From 43ac1818a8ae7bc10fa82272feb1b37456d7b69a Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 7 May 2026 15:58:45 -0700 Subject: [PATCH 202/215] update dependencies based code scan advice --- db/sqlite/document.go | 15 ++++------ db/sqlite/refsubdocument.go | 4 +-- db/sqlite/sqlite_client.go | 4 +-- go.mod | 17 +++++++---- go.sum | 56 +++++++++++++++++++++++++++++++++---- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/db/sqlite/document.go b/db/sqlite/document.go index 8ac879d..6711aa5 100644 --- a/db/sqlite/document.go +++ b/db/sqlite/document.go @@ -22,11 +22,11 @@ import ( "fmt" "strings" - _ "github.com/mattn/go-sqlite3" "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" log "github.com/sirupsen/logrus" + _ "modernc.org/sqlite" ) func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.SubDocument, error) { @@ -40,7 +40,7 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su var ns1, ns2 sql.NullString var b1 []byte - var nt1, nt2 sql.NullTime + var nt1, nt2 sql.NullInt64 var ni1, ni2 sql.NullInt64 if !rows.Next() { @@ -62,13 +62,11 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su s2 = &(ns2.String) } if nt1.Valid { - t1 := nt1.Time - tt := int(t1.UnixNano() / 1000000) + tt := int(nt1.Int64) ts = &tt } if nt2.Valid { - t2 := nt2.Time - tt := int(t2.UnixNano() / 1000000) + tt := int(nt2.Int64) expiry = &tt } if ni1.Valid { @@ -319,7 +317,7 @@ func (c *SqliteClient) GetDocument(cpeMac string, xargs ...interface{}) (*common for rows.Next() { var ns0, ns1, ns2 sql.NullString var b1 []byte - var nt1 sql.NullTime + var nt1 sql.NullInt64 var ni1, ni2 sql.NullInt64 err = rows.Scan(&ns0, &b1, &ni1, &nt1, &ns1, &ni2, &ns2) @@ -342,8 +340,7 @@ func (c *SqliteClient) GetDocument(cpeMac string, xargs ...interface{}) (*common s2 = &(ns2.String) } if nt1.Valid { - t1 := nt1.Time - tt := int(t1.UnixNano() / 1000000) + tt := int(nt1.Int64) ts = &tt } if ni1.Valid { diff --git a/db/sqlite/refsubdocument.go b/db/sqlite/refsubdocument.go index b93d4eb..bcea8c4 100644 --- a/db/sqlite/refsubdocument.go +++ b/db/sqlite/refsubdocument.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -23,7 +23,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" ) func (c *SqliteClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 78182d6..01b1e1e 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -23,9 +23,9 @@ import ( "fmt" "github.com/go-akka/configuration" - _ "github.com/mattn/go-sqlite3" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" + _ "modernc.org/sqlite" ) const ( @@ -67,7 +67,7 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, supplementaryPrecookEnabled := conf.GetBoolean("webconfig.supplementary_precook_enabled") supplementaryPrecookStateTTLDays := int(conf.GetInt32("webconfig.supplementary_precook_state_ttl_days", 7)) - db, err := sql.Open("sqlite3", dbfile) + db, err := sql.Open("sqlite", dbfile) if err != nil { return nil, common.NewError(err) } diff --git a/go.mod b/go.mod index 8b8a232..dc4f0ca 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/rdkcentral/webconfig -go 1.24.0 - -toolchain go1.24.11 +go 1.25.0 require ( github.com/IBM/sarama v1.42.1 @@ -12,7 +10,6 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.0 - github.com/mattn/go-sqlite3 v1.14.15 github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/sirupsen/logrus v1.9.3 @@ -26,8 +23,9 @@ require ( go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.18.0 + golang.org/x/sync v0.20.0 gotest.tools v2.2.0+incompatible + modernc.org/sqlite v1.50.0 ) require ( @@ -36,6 +34,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect @@ -55,19 +54,22 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect go.opentelemetry.io/proto/otlp v1.2.0 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.31.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect @@ -75,4 +77,7 @@ require ( google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + modernc.org/libc v1.72.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index a8f6d45..18b3cc9 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= @@ -158,6 +160,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -179,7 +183,10 @@ github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= @@ -215,8 +222,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -226,6 +233,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -263,6 +272,8 @@ github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= @@ -366,6 +377,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -420,8 +433,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -464,8 +477,9 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -524,6 +538,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -640,6 +656,34 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= +modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8= +modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU= +modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM= +modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 805c5a122555873b28c2ce791373674226d98b84 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 7 May 2026 16:04:41 -0700 Subject: [PATCH 203/215] update dependencies based code scan advice --- go.mod | 89 ++++---- go.sum | 638 ++++++++++----------------------------------------------- 2 files changed, 154 insertions(+), 573 deletions(-) diff --git a/go.mod b/go.mod index dc4f0ca..abe8783 100644 --- a/go.mod +++ b/go.mod @@ -3,81 +3,80 @@ module github.com/rdkcentral/webconfig go 1.25.0 require ( - github.com/IBM/sarama v1.42.1 + github.com/IBM/sarama v1.48.0 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 - github.com/gocql/gocql v1.6.0 - github.com/golang-jwt/jwt/v5 v5.2.2 + github.com/gocql/gocql v1.7.0 + github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.0 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 - github.com/sirupsen/logrus v1.9.3 - github.com/twmb/murmur3 v1.1.6 + github.com/gorilla/mux v1.8.1 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 + github.com/sirupsen/logrus v1.9.4 + github.com/twmb/murmur3 v1.1.8 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/otel v1.27.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 - go.opentelemetry.io/otel/sdk v1.27.0 - go.opentelemetry.io/otel/trace v1.27.0 - go.uber.org/automaxprocs v1.5.1 - go.uber.org/ratelimit v0.2.0 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 + go.uber.org/automaxprocs v1.6.0 + go.uber.org/ratelimit v0.3.1 golang.org/x/sync v0.20.0 gotest.tools v2.2.0+incompatible modernc.org/sqlite v1.50.0 ) require ( - github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/eapache/go-resiliency v1.4.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect + github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.31.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 // indirect + google.golang.org/grpc v1.81.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - modernc.org/libc v1.72.0 // indirect + modernc.org/libc v1.72.2 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 18b3cc9..1adcd3b 100644 --- a/go.sum +++ b/go.sum @@ -1,193 +1,68 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= -github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= +github.com/IBM/sarama v1.48.0 h1:9LJS0VNeg/boXxT/GLAMDKX6uSQ1mr/5F/j4v9gSeBQ= +github.com/IBM/sarama v1.48.0/go.mod h1:UhvwPF8zilmLOSd6O+ENzdycCJYwMww1U9DJOZpoCro= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= -github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= -github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= +github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= +github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= -github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= +github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -200,21 +75,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -222,82 +84,50 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= +github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= -github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= @@ -305,361 +135,116 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 h1:mS47AX77OtFfKG4vtp+84kuGSFZHTyxtXIN269vChY0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0/go.mod h1:PJnsC41lAGncJlPUniSwM81gc80GkgWJWr3cu2nKEtU= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= -go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= -go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= -golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348 h1:U8orV30l6KpDsi9dxU0CoJZGbjS8EEpw+6ba+XwGPQA= +google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348/go.mod h1:Yzdzr5OOZFgSsEV2D/Xi9NL3bszpXFAg0hFJiRohcD8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 h1:pfIbyB44sWzHiCpRqIen67ZQnVXSfIxWrqUMk1qwODE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.0 h1:W3G9N3KQf3BU+YuCtGKJk0CmxQNbAISICD/9AORxLIw= +google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= -modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8= -modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU= -modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0= +modernc.org/cc/v4 v4.28.1 h1:XpLbkYVQ24E8tX5u8+yWGvaxerxkR/S4zqxI8ZoSBuc= +modernc.org/cc/v4 v4.28.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= @@ -668,14 +253,14 @@ modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= -modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= +modernc.org/libc v1.72.2 h1:HwRjrHwX7hZIFCfRyw6otVlY+BoZEHFQmHQa5B0BzDE= +modernc.org/libc v1.72.2/go.mod h1:43RZAMuEX483KwP1bW+3lTFm3dzwFpl6R8HMEutqy/w= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= -modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM= @@ -684,6 +269,3 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From e5ed63828ba7399caaaaa59dee985012147685fe Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 May 2026 14:01:07 -0700 Subject: [PATCH 204/215] add a codeql.yml file --- .github/workflow/codeql.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflow/codeql.yml diff --git a/.github/workflow/codeql.yml b/.github/workflow/codeql.yml new file mode 100644 index 0000000..a8366c9 --- /dev/null +++ b/.github/workflow/codeql.yml @@ -0,0 +1,33 @@ +name: "CodeQL" + +on: + pull_request: + push: + branches: [ main ] + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: go + + # ✅ THIS IS THE CRITICAL MISSING STEP + # ✅ IT MUST BE HERE — BETWEEN init and analyze + - name: Build Go + run: go build ./... + + - name: Analyze + uses: github/codeql-action/analyze@v3 From 3083892d84b3182dc31a677937adba13a0a9c4c1 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 May 2026 14:05:43 -0700 Subject: [PATCH 205/215] modify codeql.yml based on advice --- .github/workflow/codeql.yml | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflow/codeql.yml b/.github/workflow/codeql.yml index a8366c9..56460a3 100644 --- a/.github/workflow/codeql.yml +++ b/.github/workflow/codeql.yml @@ -1,4 +1,4 @@ -name: "CodeQL" +name: CodeQL on: pull_request: @@ -7,27 +7,19 @@ on: jobs: analyze: - name: Analyze runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5 with: go-version-file: go.mod - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + - uses: github/codeql-action/init@v3 with: - languages: go + languages: go # ✅ ONLY GO - # ✅ THIS IS THE CRITICAL MISSING STEP - # ✅ IT MUST BE HERE — BETWEEN init and analyze - - name: Build Go - run: go build ./... + - run: go build ./... - - name: Analyze - uses: github/codeql-action/analyze@v3 + - uses: github/codeql-action/analyze@v3 From 164ac43f53bde1d4a34ca4f19591e0573eef7a7e Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 May 2026 14:37:37 -0700 Subject: [PATCH 206/215] fix a bug that codeql.yml was put in a wrong directory --- .github/{workflow => workflows}/codeql.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{workflow => workflows}/codeql.yml (100%) diff --git a/.github/workflow/codeql.yml b/.github/workflows/codeql.yml similarity index 100% rename from .github/workflow/codeql.yml rename to .github/workflows/codeql.yml From 73033d1b61c3b1606e3a5f242890c8e6d2b14779 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 May 2026 14:57:13 -0700 Subject: [PATCH 207/215] update codeql.yml to pass the PR check --- .github/workflows/codeql.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 56460a3..6028744 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,3 +23,5 @@ jobs: - run: go build ./... - uses: github/codeql-action/analyze@v3 + with: + upload: false # ← ADD THIS LINE From 9df101b0547174d76d0c89665ddd1783070d3924 Mon Sep 17 00:00:00 2001 From: lstruman Date: Wed, 13 May 2026 15:04:27 -0700 Subject: [PATCH 208/215] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- common/req_header.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/req_header.go b/common/req_header.go index 1a21f23..73dae9a 100644 --- a/common/req_header.go +++ b/common/req_header.go @@ -35,7 +35,7 @@ func NewReqHeader(header http.Header) *ReqHeader { func (h *ReqHeader) Get(k string) (string, error) { v := h.Header.Get(k) if !IsPrintable([]byte(v)) { - return "", fmt.Errorf("header %v invalid value %v discarded", k, v) + return "", fmt.Errorf("header %v contains non-printable characters and was discarded", k) } return v, nil } From df0fb2f12dfd3fd0a5bc3938a0807beb3d021a8a Mon Sep 17 00:00:00 2001 From: lstruman Date: Wed, 13 May 2026 15:05:37 -0700 Subject: [PATCH 209/215] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- util/dict.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/util/dict.go b/util/dict.go index fb8d570..9ef4b97 100644 --- a/util/dict.go +++ b/util/dict.go @@ -19,6 +19,7 @@ package util import ( "net/http" + "strings" "time" log "github.com/sirupsen/logrus" @@ -210,9 +211,21 @@ func (d Dict) Update(itf interface{}) { } } +func isSensitiveHeaderKey(key string) bool { + k := strings.ToLower(strings.TrimSpace(key)) + if k == "authorization" || k == "proxy-authorization" || k == "cookie" || k == "set-cookie" { + return true + } + return strings.Contains(k, "token") || strings.Contains(k, "secret") || strings.Contains(k, "apikey") || strings.Contains(k, "api-key") || strings.Contains(k, "key") +} + func HeaderToMap(header http.Header) map[string]string { m := make(map[string]string) for k, v := range header { + if isSensitiveHeaderKey(k) { + m[k] = "****" + continue + } m[k] = v[0] } return m From d1d71813d1ae398e8f35ce01b4e63b7fc01ba29c Mon Sep 17 00:00:00 2001 From: lstruman Date: Wed, 13 May 2026 15:05:57 -0700 Subject: [PATCH 210/215] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- http/webconfig_server.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0309cea..1361860 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1063,7 +1063,7 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes for _, m := range messages { if len(m.DeviceId) != 16 { - log.WithFields(tfields).Warn("invalid device_id " + m.DeviceId) + log.WithFields(tfields).Warn("invalid device_id") continue } mac := m.DeviceId[4:] @@ -1083,8 +1083,8 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } s.Input() <- outMessage - tfields["output_key"] = mac - tfields["output_body"] = m + tfields["output_key"] = "****" + tfields["output_body"] = "omitted" log.WithFields(tfields).Info("send") } } From 386080b72d4f49db4a51d16989a39c862d048072 Mon Sep 17 00:00:00 2001 From: lstruman Date: Wed, 13 May 2026 15:06:16 -0700 Subject: [PATCH 211/215] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- http/webconfig_server.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1361860..fe101c1 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1093,6 +1093,7 @@ func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token str fields := xw.Audit() fields["logger"] = "token" tfields := common.FilterLogFields(fields) + delete(tfields, "header") var headerMap map[string]string var isObfuscated bool if itf, ok := tfields["header"]; ok { @@ -1113,7 +1114,8 @@ func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token str } if codec == nil { - tfields["plaintoken"] = token + tfields["token_present"] = len(token) > 0 + tfields["token_length"] = len(token) } else { var encToken string if encryptedB64, err := codec.Encrypt(token); err == nil { From 0c7d9a0f68a62a8dcf0e8f113ff9994a0ecdd680 Mon Sep 17 00:00:00 2001 From: lstruman Date: Wed, 13 May 2026 15:07:16 -0700 Subject: [PATCH 212/215] Potential fix for pull request finding 'CodeQL / Workflow does not contain permissions' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6028744..4fd94b9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -5,6 +5,9 @@ on: push: branches: [ main ] +permissions: + contents: read + jobs: analyze: runs-on: ubuntu-latest From f3f60c5a821f9063875da30b3e5f547aa4a72c63 Mon Sep 17 00:00:00 2001 From: lstruman Date: Wed, 13 May 2026 15:24:09 -0700 Subject: [PATCH 213/215] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- util/dict.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/util/dict.go b/util/dict.go index 9ef4b97..19e4e8d 100644 --- a/util/dict.go +++ b/util/dict.go @@ -211,21 +211,36 @@ func (d Dict) Update(itf interface{}) { } } -func isSensitiveHeaderKey(key string) bool { +func isSafeHeaderKey(key string) bool { k := strings.ToLower(strings.TrimSpace(key)) - if k == "authorization" || k == "proxy-authorization" || k == "cookie" || k == "set-cookie" { + switch k { + case "accept", + "accept-encoding", + "accept-language", + "cache-control", + "connection", + "content-length", + "content-type", + "host", + "pragma", + "user-agent": return true + default: + return false } - return strings.Contains(k, "token") || strings.Contains(k, "secret") || strings.Contains(k, "apikey") || strings.Contains(k, "api-key") || strings.Contains(k, "key") } func HeaderToMap(header http.Header) map[string]string { m := make(map[string]string) for k, v := range header { - if isSensitiveHeaderKey(k) { + if !isSafeHeaderKey(k) { m[k] = "****" continue } + if len(v) == 0 { + m[k] = "" + continue + } m[k] = v[0] } return m From dbb2aad332b9676fe224a7ba2a66c558a69ed359 Mon Sep 17 00:00:00 2001 From: lstruman Date: Wed, 13 May 2026 15:27:29 -0700 Subject: [PATCH 214/215] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- common/log_fields.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/common/log_fields.go b/common/log_fields.go index 4bbed90..da4d6ba 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -50,7 +50,15 @@ func FilterLogFields(src log.Fields, excludes ...string) log.Fields { for k, v := range src { switch ty := v.(type) { case map[string]string: - fields[k] = maps.Clone(ty) + if k == "header" { + redacted := map[string]string{} + for hk := range ty { + redacted[hk] = "****" + } + fields[k] = redacted + } else { + fields[k] = maps.Clone(ty) + } case map[string]interface{}: fields[k] = maps.Clone(ty) case string: From e98851e1b577d5432755aab4ed500d6678f590fc Mon Sep 17 00:00:00 2001 From: lstruman Date: Wed, 13 May 2026 15:38:33 -0700 Subject: [PATCH 215/215] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- common/log_fields.go | 69 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index da4d6ba..4cb96ba 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -20,6 +20,7 @@ package common import ( "maps" "slices" + "strings" log "github.com/sirupsen/logrus" ) @@ -45,32 +46,70 @@ var ( } ) +func isSensitiveLogKey(k string) bool { + key := strings.ToLower(k) + return strings.Contains(key, "passphrase") || + strings.Contains(key, "password") || + strings.Contains(key, "passwd") || + strings.Contains(key, "token") || + strings.Contains(key, "authorization") || + strings.Contains(key, "cookie") || + strings.Contains(key, "secret") || + strings.Contains(key, "apikey") || + strings.Contains(key, "api_key") +} + +func sanitizeLogValue(k string, v interface{}) interface{} { + if isSensitiveLogKey(k) { + return "****" + } + + switch ty := v.(type) { + case map[string]string: + redacted := map[string]string{} + for mk, mv := range ty { + if isSensitiveLogKey(mk) || k == "header" { + redacted[mk] = "****" + } else { + redacted[mk] = mv + } + } + return redacted + case map[string]interface{}: + redacted := map[string]interface{}{} + for mk, mv := range ty { + redacted[mk] = sanitizeLogValue(mk, mv) + } + return redacted + case []interface{}: + redacted := make([]interface{}, len(ty)) + for i, iv := range ty { + redacted[i] = sanitizeLogValue(k, iv) + } + return redacted + default: + return ty + } +} + func FilterLogFields(src log.Fields, excludes ...string) log.Fields { fields := log.Fields{} for k, v := range src { switch ty := v.(type) { - case map[string]string: - if k == "header" { - redacted := map[string]string{} - for hk := range ty { - redacted[hk] = "****" - } - fields[k] = redacted - } else { - fields[k] = maps.Clone(ty) - } - case map[string]interface{}: - fields[k] = maps.Clone(ty) case string: if slices.Contains(nonEmptyFields, k) { if len(ty) > 0 { - fields[k] = ty + fields[k] = sanitizeLogValue(k, ty) } } else { - fields[k] = ty + fields[k] = sanitizeLogValue(k, ty) } + case map[string]string: + fields[k] = sanitizeLogValue(k, maps.Clone(ty)) + case map[string]interface{}: + fields[k] = sanitizeLogValue(k, maps.Clone(ty)) default: - fields[k] = ty + fields[k] = sanitizeLogValue(k, ty) } }