diff --git a/.gitignore b/.gitignore index 85981f5..07683a2 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,13 @@ # Dependency directories (remove the comment below to include it) # vendor/ -# ##################### +# ignored files bin/* .vscode/* .env *.sh + +# Ignore OpenSpec generated files and folders +openspec/ +.github/ +.claude/ diff --git a/Makefile b/Makefile index 605e139..4c6d513 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,9 @@ BUILDTIME := $(shell date -u +"%F_%T_%Z") all: build testall: test testsqlite testyuga +testenv: ## Run all tests sourcing env.sh for TEST_CONFIG_FILE + bash -c "source env.sh && go test ./... -cover -count=1" + build: ## Build a version go build -v -ldflags="-X ${REPO}/common.BinaryBranch=${BRANCH} -X ${REPO}/common.BinaryVersion=${Version} -X ${REPO}/common.BinaryBuildTime=${BUILDTIME}" -o bin/${PROJ}-${GOOS}-${GOARCH} main.go diff --git a/README.md b/README.md index c1ea263..7ef6617 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Webconfig supports 2 types of transport between cloud and devices 2. mqtt ## Install go -This project is written and tested with Go **1.17**. +This project is written and tested with Go **1.21**. ## Build the binary ```shell @@ -78,6 +78,98 @@ The actual kafka topic names can be any. Below is just an example. "topics" shou } ``` +#### Kafka TLS/SSL Configuration + +Webconfig supports secure TLS/SSL connections to Kafka brokers for both consumers and producers. This is recommended for production environments to ensure data encryption in transit and proper authentication. + +**TLS Configuration Options:** + +- `tls.enabled` - Enable/disable TLS for Kafka connections (default: false) +- `tls.cert_file` - Path to client certificate file for mTLS authentication (optional) +- `tls.key_file` - Path to client private key file for mTLS authentication (optional) +- `tls.ca_cert_file` - Path to CA certificate file for broker verification (optional) +- `tls.insecure_skip_verify` - Skip certificate verification (insecure, for testing only, default: false) + +**Consumer TLS Configuration Example:** + +```shell + kafka { + enabled = true + brokers = "kafka-broker:9093" # Use secure port + topics = "config-version-report" + consumer_group = "webconfig" + + tls { + enabled = true + cert_file = "/etc/webconfig/kafka/client.crt" + key_file = "/etc/webconfig/kafka/client.key" + ca_cert_file = "/etc/webconfig/kafka/ca.crt" + insecure_skip_verify = false + } + + # Per-cluster TLS configuration + clusters { + mesh { + enabled = true + brokers = "kafka-mesh:9093" + topics = "staging-chi-onewifi-from-device" + + tls { + enabled = true + cert_file = "/etc/webconfig/kafka/mesh-client.crt" + key_file = "/etc/webconfig/kafka/mesh-client.key" + ca_cert_file = "/etc/webconfig/kafka/mesh-ca.crt" + } + } + } + } +``` + +**Producer TLS Configuration Example:** + +```shell + kafka_producer { + enabled = true + brokers = "kafka-broker:9093" + topic = "webconfig_downstream" + + tls { + enabled = true + cert_file = "/etc/webconfig/kafka/producer-client.crt" + key_file = "/etc/webconfig/kafka/producer-client.key" + ca_cert_file = "/etc/webconfig/kafka/ca.crt" + } + } +``` + +**Certificate Requirements:** + +1. **Client Certificate (mTLS)**: If `cert_file` and `key_file` are provided, mutual TLS authentication is enabled. The certificate and key must be in PEM format. + +2. **CA Certificate**: If `ca_cert_file` is provided, it will be used to verify the Kafka broker's certificate. This is useful when using self-signed certificates or internal CAs. + +3. **Certificate Validation**: All certificate files are validated at startup. The application will fail to start with clear error messages if: + - Certificate files are missing or unreadable + - Certificates are in invalid format + - Certificate and key don't match + +**TLS Security Best Practices:** + +1. **Use TLS in Production**: Always enable TLS for production Kafka connections to encrypt data in transit +2. **Use mTLS**: Provide client certificates (`cert_file` and `key_file`) for mutual authentication +3. **Verify Certificates**: Never use `insecure_skip_verify = true` in production - it disables certificate verification and is insecure +4. **Protect Certificate Files**: Set appropriate file permissions (0600) on certificate and key files +5. **Use Secure Ports**: Configure Kafka brokers to listen on secure ports (typically 9093 for TLS) +6. **Certificate Rotation**: Plan for certificate rotation - the service must be restarted to pick up new certificates + +**Troubleshooting TLS Issues:** + +- Check logs for TLS-related errors during startup +- Verify certificate file paths are correct and files are readable +- Ensure Kafka brokers are configured to accept TLS connections +- Test certificate validity: `openssl x509 -in client.crt -text -noout` +- Verify certificate and key match: `openssl x509 -noout -modulus -in client.crt | openssl md5` vs `openssl rsa -noout -modulus -in client.key | openssl md5` + ### Configuration for database The main database operations are defined as an interface. Any driver that implements the interface should work. We has implemented using sqlite, cassandra and yugabytedb. After the db is properly configured, the dbinit.cql can be used to create the tables for cassandra. diff --git a/common/bitmap.go b/common/bitmap.go new file mode 100644 index 0000000..ba47d7d --- /dev/null +++ b/common/bitmap.go @@ -0,0 +1,177 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package common + +// header X-System-Supported-Docs +type BitMaskTuple struct { + GroupBit int + CpeBit int +} + +// The group based bitmaps will be merged into 1 cpe bitmap +// 1: []BitMaskTuple{ // meta_group_id: defined by RDK +// +// BitMaskTuple{1, 1}, // {"index_of_bit_from_lsb" for a group bitmap, "index_of_bit_from_lsb" for the cpe bitmap +var ( + SupportedDocsBitMaskMap = map[int][]BitMaskTuple{ + 1: { + {1, 1}, + {2, 2}, + {3, 3}, + {4, 4}, + {5, 5}, + {6, 6}, + {7, 29}, // connectedbuilding + {8, 35}, // xmspeedboost + {9, 40}, // webui + }, + 2: { + {1, 7}, + {2, 8}, + {3, 9}, + {4, 43}, // ignitewifi + }, + 3: { + {1, 10}, + }, + 4: { + {1, 11}, + }, + 5: { + {1, 12}, + }, + 6: { + {1, 13}, // mesh + {2, 31}, // clienttosteeringprofile + {3, 36}, // meshsteeringprofiles + {4, 37}, // wifistatsconfig + {5, 38}, // mwoconfigs + {6, 39}, // interference + {7, 34}, // wifimotionsettings + {8, 41}, // channelplan + }, + 7: { + {1, 14}, + }, + 8: { + {1, 15}, + {2, 32}, + {3, 33}, + }, + 9: { + {1, 16}, + {2, 17}, + }, + 10: { + {1, 18}, + {2, 19}, + }, + 11: { + {1, 20}, + {2, 25}, + }, + 12: { + {1, 21}, + {2, 23}, + {3, 42}, // hotspotwantolan + }, + 13: { + {1, 22}, + }, + 14: { + {1, 24}, + }, + 15: { + {1, 26}, + {2, 27}, + }, + 16: { + {1, 28}, + }, + 17: { + {1, 30}, + }, + } +) + +var ( + SubdocBitIndexMap = map[string]int{ + "portforwarding": 1, + "lan": 2, + "wan": 3, + "macbinding": 4, + "hotspot": 5, + "bridge": 6, + "privatessid": 7, + "homessid": 8, + "radio": 9, + "moca": 10, + "xdns": 11, + "advsecurity": 12, + "mesh": 13, + "aker": 14, + "telemetry": 15, + "statusreport": 16, + "trafficreport": 17, + "interfacereport": 18, + "radioreport": 19, + "telcovoip": 20, + "wanmanager": 21, + "voiceservice": 22, + "wanfailover": 23, + "cellularconfig": 24, + "telcovoice": 25, + "gwfailover": 26, + "gwrestore": 27, + "prioritizedmacs": 28, + "connectedbuilding": 29, + "lldqoscontrol": 30, + "clienttosteeringprofile": 31, + "defaultrfc": 32, + "rfc": 33, + "wifimotionsettings": 34, + "xmspeedboost": 35, + "meshsteeringprofiles": 36, + "wifistatsconfig": 37, + "mwoconfigs": 38, + "interference": 39, + "webui": 40, + "channelplan": 41, + "hotspotwantolan": 42, + "ignitewifi": 43, + } +) + +func GetDefaultSupportedSubdocMap() map[string]bool { + m := make(map[string]bool) + for k := range SubdocBitIndexMap { + m[k] = false + } + return m +} + +func BuildSupportedSubdocMapWithDefaults(supportedSubdocIds []string) map[string]bool { + m := make(map[string]bool) + for k := range SubdocBitIndexMap { + m[k] = false + } + for _, s := range supportedSubdocIds { + m[s] = true + } + return m +} diff --git a/common/bitmap_test.go b/common/bitmap_test.go new file mode 100644 index 0000000..9c75139 --- /dev/null +++ b/common/bitmap_test.go @@ -0,0 +1,47 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "slices" + "testing" + + "gotest.tools/assert" +) + +func TestGetDefaultSupportedSubdocMap(t *testing.T) { + m := GetDefaultSupportedSubdocMap() + assert.Equal(t, len(m), len(SubdocBitIndexMap)) +} + +func TestBuildSupportedSubdocMapWithDefaults(t *testing.T) { + supportedSubdocIds := []string{ + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + } + m := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.Equal(t, len(m), len(SubdocBitIndexMap)) + for k := range m { + if m[k] { + assert.Assert(t, slices.Contains(supportedSubdocIds, k)) + } + } +} diff --git a/common/const_var.go b/common/const_var.go index ddba5f8..776cda6 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common const ( @@ -64,6 +64,9 @@ var ( ) const ( + HeaderContentType = "Content-Type" + HeaderApplicationJson = "application/json" + HeaderApplicationMsgpack = "application/msgpack" HeaderEtag = "Etag" HeaderIfNoneMatch = "If-None-Match" HeaderFirmwareVersion = "X-System-Firmware-Version" @@ -73,6 +76,8 @@ const ( HeaderProfileVersion = "X-System-Telemetry-Profile-Version" HeaderPartnerID = "X-System-PartnerID" HeaderAccountID = "X-System-AccountID" + HeaderProductClass = "X-System-Product-Class" + HeaderAccountType = "X-System-Type" HeaderUserAgent = "User-Agent" HeaderSchemaVersion = "X-System-Schema-Version" HeaderMetricsAgent = "X-Metrics-Agent" @@ -105,124 +110,21 @@ const ( HeaderSourceAppName = "X-Source-App-Name" HeaderTraceparent = "Traceparent" HeaderTracestate = "Tracestate" + HeaderContentLength = "Content-Length" + HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" + HeaderUpstreamResponse = "X-Upstream-Response" + HeaderMoneytrace = "X-Moneytrace" + HeaderWebconfigTransactionId = "X-Webconfig-Transaction-Id" + HeaderMoracide = "X-Cl-Experiment" + HeaderCanary = "X-Cl-Canary" ) -// header X-System-Supported-Docs -type BitMaskTuple struct { - GroupBit int - CpeBit int -} - -// The group based bitmaps will be merged into 1 cpe bitmap -// 1: []BitMaskTuple{ // meta_group_id: defined by RDK -// -// BitMaskTuple{1, 1}, // {"index_of_bit_from_lsb" for a group bitmap, "index_of_bit_from_lsb" for the cpe bitmap -var ( - SupportedDocsBitMaskMap = map[int][]BitMaskTuple{ - 1: { - {1, 1}, - {2, 2}, - {3, 3}, - {4, 4}, - {5, 5}, - {6, 6}, - {7, 29}, - }, - 2: { - {1, 7}, - {2, 8}, - {3, 9}, - }, - 3: { - {1, 10}, - }, - 4: { - {1, 11}, - }, - 5: { - {1, 12}, - }, - 6: { - {1, 13}, - }, - 7: { - {1, 14}, - }, - 8: { - {1, 15}, - }, - 9: { - {1, 16}, - {2, 17}, - }, - 10: { - {1, 18}, - {2, 19}, - }, - 11: { - {1, 20}, - {2, 25}, - }, - 12: { - {1, 21}, - {2, 23}, - }, - 13: { - {1, 22}, - }, - 14: { - {1, 24}, - }, - 15: { - {1, 26}, - {2, 27}, - }, - 16: { - {1, 28}, - }, - 17: { - {1, 30}, - }, - } -) - -var ( - SubdocBitIndexMap = map[string]int{ - "portforwarding": 1, - "lan": 2, - "wan": 3, - "macbinding": 4, - "hotspot": 5, - "bridge": 6, - "privatessid": 7, - "homessid": 8, - "radio": 9, - "moca": 10, - "xdns": 11, - "advsecurity": 12, - "mesh": 13, - "aker": 14, - "telemetry": 15, - "statusreport": 16, - "trafficreport": 17, - "interfacereport": 18, - "radioreport": 19, - "telcovoip": 20, - "wanmanager": 21, - "voiceservice": 22, - "wanfailover": 23, - "cellularconfig": 24, - "telcovoice": 25, - "gwfailover": 26, - "gwrestore": 27, - "prioritizedmacs": 28, - "connectedbuilding": 29, - "lldqoscontrol": 30, - } +const ( + SkipDbUpdate = "skip-db-update" ) var ( - SupportedPokeDocs = []string{"primary", "telemetry"} + SupportedPokeDocs = []string{"primary", "telemetry", "root"} SupportedPokeRoutes = []string{"mqtt"} ) diff --git a/common/document.go b/common/document.go index f400945..506dc93 100644 --- a/common/document.go +++ b/common/document.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -28,7 +28,7 @@ import ( ) // deviceRootDocument is use to stored the data from device headers in the GET call -// we stored it to build "new" headers during the upstream call +// We stored it to build "new" headers during the upstream call // *RootDocument is used to stored data read from db and used to build the "old" headers type Document struct { docmap map[string]SubDocument @@ -105,6 +105,9 @@ func (d *Document) GetRootDocument() *RootDocument { } func (d *Document) RootVersion() string { + if d.rootDocument == nil { + return "" + } return d.rootDocument.Version } @@ -117,7 +120,7 @@ func (d *Document) FilterForMqttSend() *Document { for subdocId, subDocument := range d.docmap { if subDocument.State() != nil { state := *subDocument.State() - if state == PendingDownload || state == Failure { + if state > Deployed { newdoc.SetSubDocument(subdocId, &subDocument) } } @@ -148,6 +151,10 @@ func (d *Document) FilterForGet(versionMap map[string]string) *Document { } func (d *Document) Bytes() ([]byte, error) { + if len(d.docmap) == 0 { + return nil, nil + } + // build the http stream mparts := []Multipart{} for subdocId, subdoc := range d.docmap { @@ -187,7 +194,7 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { } header := make(http.Header) - header.Set("Content-type", MultipartContentType) + header.Set(HeaderContentType, MultipartContentType) header.Set("Etag", rootVersion) var traceId string @@ -212,3 +219,24 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { return BuildPayloadAsHttp(http.StatusOK, header, bbytes), nil } + +func (d *Document) FilterByBitmap(alwaysTrueSubdocIds ...string) *Document { + rootdoc := d.GetRootDocument() + if rootdoc == nil { + return d + } + + newdoc := NewDocument(rootdoc) + supportedMap := GetSupportedMap(rootdoc.Bitmap) + for _, sid := range alwaysTrueSubdocIds { + supportedMap[sid] = true + } + + for subdocId, subDocument := range d.docmap { + if supportedMap[subdocId] { + newdoc.SetSubDocument(subdocId, &subDocument) + } + } + + return newdoc +} diff --git a/common/document_test.go b/common/document_test.go index 26caeb7..c7ea662 100644 --- a/common/document_test.go +++ b/common/document_test.go @@ -14,10 +14,11 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( + "strconv" "testing" "time" @@ -25,7 +26,95 @@ import ( ) func TestDocument(t *testing.T) { - // TODO a place holder for now - ts := int(time.Now().UnixNano() / 1000000) - assert.Assert(t, ts > 0) + document := NewDocument(nil) + assert.Equal(t, len(document.RootVersion()), 0) + + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") + document = NewDocument(rootdoc) + + subdocIds := []string{"red", "orange", "yellow", "green"} + mparts := []Multipart{} + versionMap := make(map[string]string) + for _, subdocId := range subdocIds { + bbytes := RandomBytes(10, 20) + version := strconv.Itoa(int(time.Now().Unix())) + mpart := Multipart{ + Bytes: bbytes, + Version: version, + Name: subdocId, + State: Deployed, + } + mparts = append(mparts, mpart) + versionMap[subdocId] = version + } + + document.SetSubDocuments(mparts) + assert.Equal(t, len(document.Items()), len(subdocIds)) + + filteredDocument := document.FilterForGet(versionMap) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 0) + + filteredDocument = document.FilterForGet(nil) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) +} + +// write a test for FilterByBitmap +func TestFilterByBitmap(t *testing.T) { + document := NewDocument(nil) + assert.Equal(t, len(document.RootVersion()), 0) + + bitmap := 32479 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") + document = NewDocument(rootdoc) + + subdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "remotedebugger", + } + mparts := []Multipart{} + versionMap := make(map[string]string) + for _, subdocId := range subdocIds { + bbytes := RandomBytes(10, 20) + version := strconv.Itoa(int(time.Now().Unix())) + mpart := Multipart{ + Bytes: bbytes, + Version: version, + Name: subdocId, + State: Deployed, + } + mparts = append(mparts, mpart) + versionMap[subdocId] = version + } + + document.SetSubDocuments(mparts) + assert.Equal(t, len(document.Items()), len(subdocIds)) + + filteredDocument := document.FilterByBitmap() + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) + + exemptedSubdocIds := []string{} + filteredDocument = document.FilterByBitmap(exemptedSubdocIds...) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) + + filteredDocument = document.FilterByBitmap("remotedebugger") + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 5) } diff --git a/common/error.go b/common/error.go index 0bb87e5..880ea08 100644 --- a/common/error.go +++ b/common/error.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -22,11 +22,20 @@ import ( "fmt" "path/filepath" "runtime" + "slices" + "strings" +) + +const ( + defaultTracebackLevel = 3 ) var ( - NotOK = fmt.Errorf("!ok") - ProfileNotFound = fmt.Errorf("profile not found") + ErrNotOK = fmt.Errorf("!ok") + ErrRootDocumentLocked = fmt.Errorf("root document is locked") + ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") + ErrLowTrust = fmt.Errorf("token trust is lower than threshold") + ErrPending = fmt.Errorf("application_status pending") ) type Http400Error struct { @@ -121,12 +130,24 @@ func UnwrapAll(wrappedErr error) error { return err } -func GetCaller() string { - _, file, line, _ := runtime.Caller(1) - filename := filepath.Base(file) - fulldir := filepath.Dir(file) - dir := filepath.Base(fulldir) - return fmt.Sprintf("%v/%v[%v]", dir, filename, line) +func GetCaller(args ...int) string { + items := []string{} + + tracebackLevel := defaultTracebackLevel + if len(args) > 0 { + tracebackLevel = args[0] + } + + for i := 0; i < tracebackLevel; i++ { + _, file, line, _ := runtime.Caller(i + 1) + filename := filepath.Base(file) + fulldir := filepath.Dir(file) + dir := filepath.Base(fulldir) + items = append(items, fmt.Sprintf("%v/%v[%v]", dir, filename, line)) + } + + slices.Reverse(items) + return strings.Join(items, " ") } type NoCapabilitiesError struct { diff --git a/util/firmware_bitmap.go b/common/firmware_bitmap.go similarity index 87% rename from util/firmware_bitmap.go rename to common/firmware_bitmap.go index ab308fc..4ddef83 100644 --- a/util/firmware_bitmap.go +++ b/common/firmware_bitmap.go @@ -15,21 +15,19 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package util +package common import ( "fmt" "strconv" "strings" - - "github.com/rdkcentral/webconfig/common" ) // bitmap is used to name int variables // bitarray is used to name string variables func SetBitmapByGroup(cpeBitmap *int, groupId int, groupBitmap int) error { - tuples, ok := common.SupportedDocsBitMaskMap[groupId] + tuples, ok := SupportedDocsBitMaskMap[groupId] if !ok { return nil } @@ -47,7 +45,7 @@ func SetBitmapByGroup(cpeBitmap *int, groupId int, groupBitmap int) error { func ParseFirmwareGroupBitarray(s string) (int, int, error) { i, err := strconv.Atoi(s) if err != nil { - return 0, 0, common.NewError(err) + return 0, 0, NewError(err) } groupId := i >> 24 groupBitMask := 1<<24 - 1 @@ -62,12 +60,12 @@ func GetCpeBitmap(rdkSupportedDocsHeaderStr string) (int, error) { for _, sid := range sids { groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) if err != nil { - return 0, common.NewError(err) + return 0, NewError(err) } err = SetBitmapByGroup(&cpeBitmap, groupId, groupBitmap) if err != nil { - return 0, common.NewError(err) + return 0, NewError(err) } } @@ -100,7 +98,7 @@ func PrettyGroupBitarray(i int) string { } func IsSubdocSupported(cpeBitmap int, subdocId string) bool { - index, ok := common.SubdocBitIndexMap[subdocId] + index, ok := SubdocBitIndexMap[subdocId] if !ok { return false } @@ -114,7 +112,7 @@ func IsSubdocSupported(cpeBitmap int, subdocId string) bool { func GetSupportedMap(cpeBitmap int) map[string]bool { supportedMap := map[string]bool{} - for k, index := range common.SubdocBitIndexMap { + for k, index := range SubdocBitIndexMap { shift := index - 1 bitmask := 1 << shift supportedMap[k] = false @@ -129,7 +127,7 @@ func BitarrayToBitmap(src string) (int, error) { s := strings.ReplaceAll(src, " ", "") v, err := strconv.ParseInt(s, 2, 64) if err != nil { - return 0, common.NewError(err) + return 0, NewError(err) } i := int(v) return i, nil @@ -138,7 +136,7 @@ func BitarrayToBitmap(src string) (int, error) { func GetBitmapFromSupportedMap(srcMap map[string]bool) int { var bitmap int - for k, index := range common.SubdocBitIndexMap { + for k, index := range SubdocBitIndexMap { shift := index - 1 bitmask := 1 << shift if srcMap[k] { diff --git a/common/firmware_bitmap_test.go b/common/firmware_bitmap_test.go new file mode 100644 index 0000000..975ed3f --- /dev/null +++ b/common/firmware_bitmap_test.go @@ -0,0 +1,676 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package common + +import ( + "fmt" + "math/rand" + "strings" + "testing" + + "gotest.tools/assert" +) + +// bitmap is used to name int variables +// bitarray is used to name string variables + +func TestPrettyBitarray(t *testing.T) { + i := 8 + bs := PrettyBitarray(i) + expected := "0000 0000 0000 0000 0000 0000 0000 1000" + assert.Equal(t, bs, expected) + + j := 16777231 + bs = PrettyGroupBitarray(j) + expected = "00000001 0000 0000 0000 0000 0000 1111" + assert.Equal(t, bs, expected) +} + +func TestParseRdkGroupBitarray(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + expectedMap := map[int]int{ + 1: 15, + 2: 3, + 3: 1, + 4: 1, + 5: 1, + 6: 1, + 7: 1, + 8: 1, + } + + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + for _, sid := range sids { + groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) + assert.NilError(t, err) + // assert.Equal(t, groupId, 1) + + expectedBitmap, ok := expectedMap[groupId] + assert.Assert(t, ok) + assert.Equal(t, groupBitmap, expectedBitmap) + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + + // bs := PrettyGroupBitarray(cpeBitmap) + + // build expected + expectedEnabled := map[string]bool{ + "portforwarding": true, + "lan": true, + "wan": true, + "macbinding": true, + "hotspot": false, + "bridge": false, + "privatessid": true, + "homessid": true, + "radio": false, + "moca": true, + "xdns": true, + "advsecurity": true, + "mesh": true, + "aker": true, + "telemetry": true, + "statusreport": false, + "trafficreport": false, + "interfacereport": false, + "radioreport": false, + } + + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } +} + +func TestParseCustomizedGroupBitarray(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + // build expected + expectedEnabled := map[string]bool{ + "portforwarding": true, + "lan": true, + "wan": true, + "macbinding": true, + "hotspot": false, + "bridge": false, + "privatessid": true, + "homessid": true, + "radio": false, + "moca": true, + "xdns": true, + "advsecurity": true, + "mesh": true, + "aker": true, + "telemetry": true, + "statusreport": false, + "trafficreport": false, + "interfacereport": false, + "radioreport": false, + "telcovoip": false, + "telcovoice": false, + "wanmanager": false, + "voiceservice": false, + } + + newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" + group1Bitmap, err := BitarrayToBitmap(newGroup1Bitarray) + assert.NilError(t, err) + sids[0] = fmt.Sprintf("%v", group1Bitmap) + expectedEnabled["wan"] = false + expectedEnabled["macbinding"] = false + expectedEnabled["hotspot"] = true + expectedEnabled["bridge"] = true + + newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" + group2Bitmap, err := BitarrayToBitmap(newGroup2Bitarray) + assert.NilError(t, err) + sids[1] = fmt.Sprintf("%v", group2Bitmap) + expectedEnabled["privatessid"] = false + expectedEnabled["radio"] = true + + rdkSupportedDocsHeaderStr = strings.Join(sids, ",") + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocIds := []string{} + for k, v := range expectedEnabled { + if v { + supportedSubdocIds = append(supportedSubdocIds, k) + } + } + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseTelcovoipAndWanmanager(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549377,201326593" + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "privatessid", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "telcovoip", + "wanmanager", + } + + rdkSupportedDocsHeaderStr = strings.Join(sids, ",") + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for _, subdocId := range supportedSubdocIds { + assert.Assert(t, IsSubdocSupported(cpeBitmap, subdocId)) + } + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestBitmapParsing(t *testing.T) { + // clearn the wan bit + newBitmap := 16777231 & ^(1 << 2) + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,33554435,50331649,67108865,83886081,100663297,117440513,134217729", newBitmap) + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "macbinding", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseVoiceService(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809" + + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestManualBitmap(t *testing.T) { + for i := 0; i < 10; i++ { + bitmap := rand.Intn(40000) + parsedSupportedMap := GetSupportedMap(bitmap) + revBitmap := GetBitmapFromSupportedMap(parsedSupportedMap) + assert.Equal(t, bitmap, revBitmap) + } +} + +func TestParseSupportedDocsWithNewGroups(t *testing.T) { + cellularBitGroupId := 14 + xBitValue := (cellularBitGroupId << 24) + 1 + ss := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809,234881025" + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", + "cellularconfig", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,67108865,100663297,117440513,134217729,201326595,234881025" + + // build expected + supportedSubdocIds := []string{ + "aker", + "cellularconfig", + "homessid", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549378,201326595" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telcovoice", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,251658241,268435457" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderRfc(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "defaultrfc", + "rfc", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderHcm(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWebui(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderChannelplan(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663487,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + "channelplan", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderHotspotwantolan(t *testing.T) { + // group_id=12, group-bit=3 → CPE bit 42 (hotspotwantolan) + group12Value := (12 << 24) + (1 << (3 - 1)) // bit 3 set in group 12 + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v", group12Value) + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + assert.Assert(t, IsSubdocSupported(cpeBitmap, "hotspotwantolan")) + assert.Assert(t, !IsSubdocSupported(cpeBitmap, "ignitewifi")) +} + +func TestParseSupportedDocsHeaderIgnitewifi(t *testing.T) { + // group_id=2, group-bit=4 → CPE bit 43 (ignitewifi) + group2Value := (2 << 24) + (1 << (4 - 1)) // bit 4 set in group 2 + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v", group2Value) + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + assert.Assert(t, IsSubdocSupported(cpeBitmap, "ignitewifi")) + assert.Assert(t, !IsSubdocSupported(cpeBitmap, "hotspotwantolan")) +} + +func TestParseSupportedDocsHeaderBothNewSubdocs(t *testing.T) { + // group 12: bits 1, 2, 3 → CPE bits 21 (wanmanager), 23 (wanfailover), 42 (hotspotwantolan) + // group 2: bits 1, 2, 3, 4 → CPE bits 7 (privatessid), 8 (homessid), 9 (radio), 43 (ignitewifi) + group12Value := (12 << 24) + (1 << 0) + (1 << 1) + (1 << 2) + group2Value := (2 << 24) + (1 << 0) + (1 << 1) + (1 << 2) + (1 << 3) + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", group12Value, group2Value) + + supportedSubdocIds := []string{ + "privatessid", + "homessid", + "radio", + "ignitewifi", + "wanmanager", + "wanfailover", + "hotspotwantolan", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} diff --git a/common/kafka_tls.go b/common/kafka_tls.go new file mode 100644 index 0000000..bcc182a --- /dev/null +++ b/common/kafka_tls.go @@ -0,0 +1,135 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package common + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + + "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" +) + +// LoadKafkaTLSConfig loads TLS configuration from HOCON config at the specified prefix. +// Prefix should be like "webconfig.kafka" or "webconfig.kafka.clusters.mesh" or "webconfig.kafka_producer". +// Returns nil if TLS is not enabled. +// Returns error if TLS is enabled but configuration is invalid. +func LoadKafkaTLSConfig(conf *configuration.Config, prefix string) (*tls.Config, error) { + tlsEnabled := conf.GetBoolean(prefix + ".tls_enabled") + if !tlsEnabled { + return nil, nil + } + + tlsConfig := &tls.Config{} + + // Check insecure_skip_verify flag first + insecureSkipVerify := conf.GetBoolean(prefix + ".tls_insecure_skip_verify") + + // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) + certFile := conf.GetString(prefix + ".tls_cert_file") + keyFile := conf.GetString(prefix + ".tls_key_file") + + // When insecure_skip_verify is true and no cert files configured, skip loading certificates + // This allows TLS without client authentication (server-only TLS) + if insecureSkipVerify && (len(certFile) == 0 || len(keyFile) == 0) { + // Insecure mode without client certificates - skip cert loading + } else if len(certFile) > 0 && len(keyFile) > 0 { + // Only validate cert files exist when verification is enabled + if !insecureSkipVerify { + // Validate certificate file exists + if _, err := os.Stat(certFile); os.IsNotExist(err) { + return nil, NewError(fmt.Errorf("TLS certificate file does not exist: %s", certFile)) + } + + // Validate key file exists + if _, err := os.Stat(keyFile); os.IsNotExist(err) { + return nil, NewError(fmt.Errorf("TLS key file does not exist: %s", keyFile)) + } + } + + // Load and parse the certificate and key + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, NewError(fmt.Errorf("failed to load TLS certificate and key from %s and %s: %v", certFile, keyFile, err)) + } + + tlsConfig.Certificates = []tls.Certificate{cert} + log.WithFields(log.Fields{ + "prefix": prefix, + "cert_file": certFile, + "key_file": keyFile, + }).Info("Loaded TLS client certificate for mTLS") + } else if len(certFile) > 0 || len(keyFile) > 0 { + // Partial cert configuration detected - require both cert and key when verification is enabled + if !insecureSkipVerify { + return nil, NewError(fmt.Errorf("TLS enabled with verification but incomplete certificate configuration (cert: %s, key: %s)", certFile, keyFile)) + } + } + + // Load CA certificate if provided (optional when insecure_skip_verify is true) + caCertFile := conf.GetString(prefix + ".tls_ca_cert_file") + // When insecure_skip_verify is true and no CA file configured, skip loading CA cert + // This allows TLS without broker verification (insecure mode) + if insecureSkipVerify && len(caCertFile) == 0 { + // Insecure mode without CA cert - skip CA loading + } else if len(caCertFile) > 0 { + // Only validate CA cert file exists when verification is enabled + if !insecureSkipVerify { + // Validate CA certificate file exists + if _, err := os.Stat(caCertFile); os.IsNotExist(err) { + return nil, NewError(fmt.Errorf("TLS CA certificate file does not exist: %s", caCertFile)) + } + } + + // Load CA certificate + caCert, err := os.ReadFile(caCertFile) + if err != nil { + return nil, NewError(fmt.Errorf("failed to read TLS CA certificate from %s: %v", caCertFile, err)) + } + + // Parse CA certificate + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, NewError(fmt.Errorf("failed to parse TLS CA certificate from %s", caCertFile)) + } + + tlsConfig.RootCAs = caCertPool + log.WithFields(log.Fields{ + "prefix": prefix, + "ca_cert_file": caCertFile, + }).Info("Loaded TLS CA certificate for broker verification") + } + + if insecureSkipVerify { + tlsConfig.InsecureSkipVerify = true + log.WithFields(log.Fields{ + "prefix": prefix, + }).Warn("TLS certificate verification is disabled (insecure_skip_verify=true). This is insecure and should only be used for testing.") + } + + log.WithFields(log.Fields{ + "prefix": prefix, + "has_client_cert": len(tlsConfig.Certificates) > 0, + "has_ca_cert": tlsConfig.RootCAs != nil, + "insecure_skip_verify": insecureSkipVerify, + }).Info("TLS configuration loaded for Kafka connection") + + return tlsConfig, nil +} diff --git a/common/kafka_tls_test.go b/common/kafka_tls_test.go new file mode 100644 index 0000000..688d07d --- /dev/null +++ b/common/kafka_tls_test.go @@ -0,0 +1,375 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package common + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-akka/configuration" + "gotest.tools/assert" +) + +// Helper function to generate a self-signed certificate for testing +func generateTestCertificate(t *testing.T, certFile, keyFile string) { + // Generate private key + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + assert.NilError(t, err) + + // Create certificate template + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Test Org"}, + CommonName: "localhost", + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + + // Create self-signed certificate + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + assert.NilError(t, err) + + // Write certificate to file + certOut, err := os.Create(certFile) + assert.NilError(t, err) + defer certOut.Close() + err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + assert.NilError(t, err) + + // Write private key to file + keyOut, err := os.Create(keyFile) + assert.NilError(t, err) + defer keyOut.Close() + privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey) + err = pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: privateKeyBytes}) + assert.NilError(t, err) +} + +func TestLoadKafkaTLSConfig_Disabled(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls_enabled = false + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig == nil, "TLS config should be nil when TLS is disabled") +} + +func TestLoadKafkaTLSConfig_EnabledWithoutCerts(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls_enabled = true + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil when TLS is enabled") + assert.Assert(t, len(tlsConfig.Certificates) == 0, "Should have no client certificates") + assert.Assert(t, tlsConfig.RootCAs == nil, "Should have no custom CA") + assert.Assert(t, !tlsConfig.InsecureSkipVerify, "Should not skip verification by default") +} + +func TestLoadKafkaTLSConfig_WithClientCertificates(t *testing.T) { + // Create temp directory for test certificates + tempDir := t.TempDir() + certFile := filepath.Join(tempDir, "client.crt") + keyFile := filepath.Join(tempDir, "client.key") + + // Generate test certificate + generateTestCertificate(t, certFile, keyFile) + + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_cert_file = "` + certFile + `" + tls_key_file = "` + keyFile + `" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, len(tlsConfig.Certificates) == 1, "Should have one client certificate") + assert.Assert(t, tlsConfig.RootCAs == nil, "Should have no custom CA") +} + +func TestLoadKafkaTLSConfig_WithCACertificate(t *testing.T) { + // Create temp directory for test certificates + tempDir := t.TempDir() + caCertFile := filepath.Join(tempDir, "ca.crt") + caKeyFile := filepath.Join(tempDir, "ca.key") + + // Generate CA certificate + generateTestCertificate(t, caCertFile, caKeyFile) + + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_ca_cert_file = "` + caCertFile + `" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, len(tlsConfig.Certificates) == 0, "Should have no client certificates") + assert.Assert(t, tlsConfig.RootCAs != nil, "Should have custom CA") +} + +func TestLoadKafkaTLSConfig_WithInsecureSkipVerify(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_insecure_skip_verify = true + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, tlsConfig.InsecureSkipVerify, "Should skip verification when configured") +} + +func TestLoadKafkaTLSConfig_MissingCertFile(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_cert_file = "/nonexistent/client.crt" + tls_key_file = "/nonexistent/client.key" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.Assert(t, err != nil, "Should return error for missing certificate file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "does not exist") +} + +func TestLoadKafkaTLSConfig_MissingCertFileWithInsecureSkipVerify(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_insecure_skip_verify = true + tls_cert_file = "/nonexistent/client.crt" + tls_key_file = "/nonexistent/client.key" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + // When insecure_skip_verify is true, missing cert files should not cause validation errors on file check, + // but will still fail when trying to actually load them + assert.Assert(t, err != nil, "Should return error trying to load non-existent files") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "failed to load TLS certificate") +} + +func TestLoadKafkaTLSConfig_InvalidCertFile(t *testing.T) { + // Create temp directory for test files + tempDir := t.TempDir() + certFile := filepath.Join(tempDir, "invalid.crt") + keyFile := filepath.Join(tempDir, "invalid.key") + + // Create invalid certificate files + err := os.WriteFile(certFile, []byte("invalid certificate content"), 0644) + assert.NilError(t, err) + err = os.WriteFile(keyFile, []byte("invalid key content"), 0644) + assert.NilError(t, err) + + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_cert_file = "` + certFile + `" + tls_key_file = "` + keyFile + `" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.Assert(t, err != nil, "Should return error for invalid certificate file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "failed to load TLS certificate") +} + +func TestLoadKafkaTLSConfig_MissingCACertFile(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_ca_cert_file = "/nonexistent/ca.crt" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.Assert(t, err != nil, "Should return error for missing CA certificate file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "does not exist") +} + +func TestLoadKafkaTLSConfig_MissingCACertFileWithInsecureSkipVerify(t *testing.T) { + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_insecure_skip_verify = true + tls_ca_cert_file = "/nonexistent/ca.crt" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + // When insecure_skip_verify is true, missing CA file will not cause validation error on file check, + // but will still fail when trying to actually read it + assert.Assert(t, err != nil, "Should return error trying to load non-existent CA file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "failed to read TLS CA certificate") +} + +func TestLoadKafkaTLSConfig_InvalidCACertFile(t *testing.T) { + // Create temp directory for test files + tempDir := t.TempDir() + caCertFile := filepath.Join(tempDir, "invalid_ca.crt") + + // Create invalid CA certificate file + err := os.WriteFile(caCertFile, []byte("invalid CA certificate content"), 0644) + assert.NilError(t, err) + + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_ca_cert_file = "` + caCertFile + `" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.Assert(t, err != nil, "Should return error for invalid CA certificate file") + assert.Assert(t, tlsConfig == nil) + assert.ErrorContains(t, err, "failed to parse TLS CA certificate") +} + +func TestLoadKafkaTLSConfig_FullConfiguration(t *testing.T) { + // Create temp directory for test certificates + tempDir := t.TempDir() + certFile := filepath.Join(tempDir, "client.crt") + keyFile := filepath.Join(tempDir, "client.key") + caCertFile := filepath.Join(tempDir, "ca.crt") + caKeyFile := filepath.Join(tempDir, "ca.key") + + // Generate test certificates + generateTestCertificate(t, certFile, keyFile) + generateTestCertificate(t, caCertFile, caKeyFile) + + confStr := ` +webconfig { + kafka { + tls_enabled = true + tls_cert_file = "` + certFile + `" + tls_key_file = "` + keyFile + `" + tls_ca_cert_file = "` + caCertFile + `" + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, len(tlsConfig.Certificates) == 1, "Should have one client certificate") + assert.Assert(t, tlsConfig.RootCAs != nil, "Should have custom CA") + assert.Assert(t, !tlsConfig.InsecureSkipVerify, "Should not skip verification") +} + +func TestLoadKafkaTLSConfig_DifferentPrefixes(t *testing.T) { + // Test with producer prefix + confStr := ` +webconfig { + kafka_producer { + tls_enabled = true + tls_insecure_skip_verify = true + } +} +` + conf := configuration.ParseString(confStr) + tlsConfig, err := LoadKafkaTLSConfig(conf, "webconfig.kafka_producer") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig != nil, "TLS config should not be nil") + assert.Assert(t, tlsConfig.InsecureSkipVerify) + + // Test with cluster prefix + confStr2 := ` +webconfig { + kafka { + clusters { + mesh { + tls_enabled = true + } + } + } +} +` + conf2 := configuration.ParseString(confStr2) + tlsConfig2, err := LoadKafkaTLSConfig(conf2, "webconfig.kafka.clusters.mesh") + + assert.NilError(t, err) + assert.Assert(t, tlsConfig2 != nil, "TLS config should not be nil for cluster") +} diff --git a/common/log_fields.go b/common/log_fields.go index 06289e6..4bbed90 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -14,10 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( + "maps" + "slices" + log "github.com/sirupsen/logrus" ) @@ -25,15 +28,42 @@ var ( unloggedFields = []string{ "moneytrace", "token", + "xpc_trace", + } + coreFields = []string{ + "app_name", + "audit_id", + "body", + "cpe_mac", + } + nonEmptyFields = []string{ + "traceparent", + "tracestate", "out_traceparent", "out_tracestate", + "req_moracide_tag", } ) func FilterLogFields(src log.Fields, excludes ...string) log.Fields { fields := log.Fields{} for k, v := range src { - fields[k] = v + switch ty := v.(type) { + case map[string]string: + fields[k] = maps.Clone(ty) + case map[string]interface{}: + fields[k] = maps.Clone(ty) + case string: + if slices.Contains(nonEmptyFields, k) { + if len(ty) > 0 { + fields[k] = ty + } + } else { + fields[k] = ty + } + default: + fields[k] = ty + } } for _, x := range unloggedFields { @@ -45,6 +75,7 @@ func FilterLogFields(src log.Fields, excludes ...string) log.Fields { delete(fields, x) } } + return fields } @@ -53,3 +84,13 @@ func UpdateLogFields(fields, newfields log.Fields) { fields[k] = v } } + +func CopyCoreLogFields(src log.Fields) log.Fields { + fields := log.Fields{} + for _, k := range coreFields { + if itf, ok := src[k]; ok { + fields[k] = itf + } + } + return fields +} diff --git a/common/log_fields_test.go b/common/log_fields_test.go index e357855..e2298cd 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -49,16 +49,14 @@ func TestFilterLogFields(t *testing.T) { assert.DeepEqual(t, expected, c2) src3 := log.Fields{ - "red": "maroon", - "orange": "auburn", - "yellow": "amber", - "green": "viridian", - "blue": "turquoise", - "indigo": "sapphire", - "violet": "purple", - "out_traceparent": "foo", - "out_tracestate": "cyan", - "token": "bar", + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "token": "bar", } c3 := FilterLogFields(src3) assert.DeepEqual(t, src, c3) @@ -95,3 +93,97 @@ func TestUpdateLogFields(t *testing.T) { assert.DeepEqual(t, src, expected) } + +func TestCopyCoreLogFields(t *testing.T) { + body := map[string]interface{}{ + "device_id": "mac:29cf4fe3914e", + "http_status_code": 304, + "transaction_uuid": "f160f5f2-c899-4652-b066-c9b68328d74f", + "version": "1719689278", + } + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + expected := log.Fields{ + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + copied := CopyCoreLogFields(src) + assert.DeepEqual(t, copied, expected) + + body["violet"] = "purple" + +} + +func TestFilterLogFieldsWithItfMap(t *testing.T) { + weekday := map[string]interface{}{ + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]interface{}) + fw["fri"] = 5 + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]interface{}) + assert.Assert(t, len(sw) == 4) +} + +func TestFilterLogFieldsWithStrMap(t *testing.T) { + weekday := map[string]string{ + "mon": "1", + "tue": "2", + "wed": "3", + "thu": "4", + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]string) + fw["fri"] = "5" + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]string) + assert.Assert(t, len(sw) == 4) +} diff --git a/common/metrics.go b/common/metrics.go index ad34337..18b11f2 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -74,6 +74,7 @@ type AppMetrics struct { indeploymentDecCount *prometheus.CounterVec failureIncCount *prometheus.CounterVec failureDecCount *prometheus.CounterVec + kafkaProducerErrCount *prometheus.CounterVec watchedCpes []string logrusLevel log.Level } @@ -298,6 +299,13 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet }, stateMetricsLabels, ), + kafkaProducerErrCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: appName + "_kafka_producer_err_count", + Help: "A counter for the number of errors by kafka producer.", + }, + []string{"topic", "partition"}, + ), watchedCpes: watchedCpes, logrusLevel: logrusLevel, } @@ -326,6 +334,7 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet appMetrics.indeploymentDecCount, appMetrics.failureIncCount, appMetrics.failureDecCount, + appMetrics.kafkaProducerErrCount, ) return appMetrics } @@ -534,6 +543,14 @@ func (m *AppMetrics) CountKafkaEvents(eventName string, status string, partition m.eventCounter.With(labels).Inc() } +func (m *AppMetrics) ObserveKafkaProducerErr(topic string, partition int32) { + labels := prometheus.Labels{ + "topic": topic, + "partition": strconv.Itoa(int(partition)), + } + m.kafkaProducerErrCount.With(labels).Inc() +} + func (m *AppMetrics) GetStateCounter(labels prometheus.Labels) (*StateCounter, error) { // REMINDER if a label is defined with 2 dimensions, then it must be referred // with 2 dimensions. Aggregation happens at prometheus level diff --git a/util/random.go b/common/random.go similarity index 91% rename from util/random.go rename to common/random.go index 4400e3f..32eebb0 100644 --- a/util/random.go +++ b/common/random.go @@ -15,17 +15,13 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package util +package common import ( + cryptorand "crypto/rand" "math/rand" - "time" ) -func init() { - rand.Seed(time.Now().Unix()) -} - func RandomDouble() float64 { return rand.Float64() } @@ -46,6 +42,7 @@ func RandomBytes(lowerBound, upperBound int) []byte { n = rand.Intn(delta) + lowerBound } bbytes := make([]byte, n) - rand.Read(bbytes) + cryptorand.Read(bbytes) + bbytes[len(bbytes)-1] = 1 return bbytes } diff --git a/util/random_test.go b/common/random_test.go similarity index 98% rename from util/random_test.go rename to common/random_test.go index ef53a06..224158b 100644 --- a/util/random_test.go +++ b/common/random_test.go @@ -15,7 +15,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package util +package common import ( "testing" diff --git a/common/refsubdocument.go b/common/refsubdocument.go new file mode 100644 index 0000000..cad7ffc --- /dev/null +++ b/common/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "bytes" +) + +type RefSubDocument struct { + payload []byte + version *string +} + +func NewRefSubDocument(payload []byte, version *string) *RefSubDocument { + return &RefSubDocument{ + payload: payload, + version: version, + } +} + +func (d *RefSubDocument) Payload() []byte { + return d.payload +} + +func (d *RefSubDocument) SetPayload(payload []byte) { + d.payload = payload +} + +func (d *RefSubDocument) HasPayload() bool { + if d.payload != nil && len(d.payload) > 0 { + return true + } else { + return false + } +} + +func (d *RefSubDocument) Version() *string { + return d.version +} + +func (d *RefSubDocument) SetVersion(version *string) { + d.version = version +} + +func (d *RefSubDocument) Equals(tdoc *RefSubDocument) bool { + if d.HasPayload() && tdoc.HasPayload() { + if !bytes.Equal(d.Payload(), tdoc.Payload()) { + return false + } + } else { + if d.HasPayload() != tdoc.HasPayload() { + return false + } + } + + if d.Version() != nil && tdoc.Version() != nil { + if *d.Version() != *tdoc.Version() { + return false + } + } else { + if d.Version() != tdoc.Version() { + return false + } + } + return true +} diff --git a/common/refsubdocument_test.go b/common/refsubdocument_test.go new file mode 100644 index 0000000..416c123 --- /dev/null +++ b/common/refsubdocument_test.go @@ -0,0 +1,40 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestRefSubDocument(t *testing.T) { + bbytes1 := []byte("hello world") + version1 := "12345" + refsubdoc1 := NewRefSubDocument(bbytes1, &version1) + + bbytes2 := []byte("hello world") + version2 := "12345" + refsubdoc2 := NewRefSubDocument(bbytes2, &version2) + assert.Assert(t, refsubdoc1.Equals(refsubdoc2)) + + bbytes3 := []byte("foo bar") + version3 := "12345" + refsubdoc3 := NewRefSubDocument(bbytes3, &version3) + assert.Assert(t, !refsubdoc1.Equals(refsubdoc3)) +} diff --git a/common/req_header.go b/common/req_header.go new file mode 100644 index 0000000..1a21f23 --- /dev/null +++ b/common/req_header.go @@ -0,0 +1,51 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "net/http" +) + +type ReqHeader struct { + http.Header +} + +func NewReqHeader(header http.Header) *ReqHeader { + return &ReqHeader{ + Header: header, + } +} + +func (h *ReqHeader) Get(k string) (string, error) { + v := h.Header.Get(k) + if !IsPrintable([]byte(v)) { + return "", fmt.Errorf("header %v invalid value %v discarded", k, v) + } + return v, nil +} + +func IsPrintable(bbytes []byte) bool { + for _, char := range bbytes { + // Check if the rune is outside the printable ASCII character range. + if char < 32 || char > 126 { + return false + } + } + return true +} diff --git a/common/req_header_test.go b/common/req_header_test.go new file mode 100644 index 0000000..829720c --- /dev/null +++ b/common/req_header_test.go @@ -0,0 +1,70 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "net/http" + "testing" + + "gotest.tools/assert" +) + +func TestIsPrintable(t *testing.T) { + b1 := []byte("hello world") + assert.Assert(t, IsPrintable(b1)) + b2 := []byte{0x00, 0x00, 0x00, 0x01} + assert.Assert(t, !IsPrintable(b2)) + b3 := append(b1, b2...) + assert.Assert(t, !IsPrintable(b3)) + + b4 := []byte("CGM4140COM_6.8p8s1_PROD_sey") + b5 := []byte{0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8} + b6 := append(b4, b5...) + assert.Assert(t, !IsPrintable(b6)) +} + +func TestReqHeader(t *testing.T) { + s1 := "hello world" + s2 := string([]byte{0x00, 0x00, 0x00, 0x01}) + s3 := s1 + s2 + + header := make(http.Header) + k1 := "maroon" + header.Set(k1, "helloworld") + k2 := "auburn" + header.Set(k2, s2) + k3 := "amber" + header.Set(k3, s3) + reqHeader := NewReqHeader(header) + + v1, err := reqHeader.Get(k1) + assert.NilError(t, err) + assert.Equal(t, v1, "helloworld") + + v2, err := reqHeader.Get(k2) + assert.Assert(t, err != nil) + assert.Equal(t, v2, "") + + v3, err := reqHeader.Get(k3) + assert.Assert(t, err != nil) + assert.Equal(t, v3, "") + + v4, err := reqHeader.Get("viridian") + assert.NilError(t, err) + assert.Equal(t, v4, "") +} diff --git a/common/root_document.go b/common/root_document.go index dc21075..b40296b 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "encoding/json" "fmt" + "time" ) const ( @@ -36,10 +37,13 @@ type RootDocument struct { SchemaVersion string `json:"schema_version"` Version string `json:"version"` QueryParams string `json:"query_params"` + LockedTill int `json:"locked_till"` + ProductClass string `json:"product_class"` + AccountType string `json:"account_type"` } -// (bitmap, firmware_version, model_name, partner_id, schema_version, version), nil -func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVersion, version, query_params string) *RootDocument { +// (bitmap, firmware_version, model_name, partner_id, schema_version, version, query_params, product_class, account_type), nil +func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVersion, version, query_params, productClass, accountType string) *RootDocument { return &RootDocument{ Bitmap: bitmap, FirmwareVersion: firmwareVersion, @@ -48,6 +52,8 @@ func NewRootDocument(bitmap int, firmwareVersion, modelName, partnerId, schemaVe SchemaVersion: schemaVersion, Version: version, QueryParams: query_params, + ProductClass: productClass, + AccountType: accountType, } } @@ -60,6 +66,8 @@ func (d *RootDocument) ColumnMap() map[string]interface{} { "schema_version": d.SchemaVersion, "version": d.Version, "query_params": d.QueryParams, + "product_class": d.ProductClass, + "account_type": d.AccountType, } return dict } @@ -69,6 +77,9 @@ func (d *RootDocument) NonEmptyColumnMap() map[string]interface{} { if d.Bitmap > 0 { dict["bitmap"] = d.Bitmap } + if d.LockedTill > 0 { + dict["locked_till"] = int64(d.LockedTill) + } tempDict := map[string]string{ "firmware_version": d.FirmwareVersion, @@ -77,6 +88,8 @@ func (d *RootDocument) NonEmptyColumnMap() map[string]interface{} { "schema_version": d.SchemaVersion, "version": d.Version, "query_params": d.QueryParams, + "product_class": d.ProductClass, + "account_type": d.AccountType, } for k, v := range tempDict { @@ -104,6 +117,12 @@ func (d *RootDocument) Compare(r *RootDocument) int { if d.SchemaVersion != r.SchemaVersion { return RootDocumentMetaChanged } + if d.ProductClass != r.ProductClass { + return RootDocumentMetaChanged + } + if d.AccountType != r.AccountType { + return RootDocumentMetaChanged + } if d.Version != r.Version { return RootDocumentVersionOnlyChanged } @@ -113,23 +132,29 @@ func (d *RootDocument) Compare(r *RootDocument) int { return RootDocumentEquals } -func (d *RootDocument) IsDifferent(r *RootDocument) bool { +func (d *RootDocument) Equals(r *RootDocument) bool { if d.Bitmap != r.Bitmap { - return true + return false } if d.FirmwareVersion != r.FirmwareVersion { - return true + return false } if d.ModelName != r.ModelName { - return true + return false } if d.PartnerId != r.PartnerId { - return true + return false } if d.SchemaVersion != r.SchemaVersion { - return true + return false + } + if d.ProductClass != r.ProductClass { + return false } - return false + if d.AccountType != r.AccountType { + return false + } + return true } // update in place @@ -155,6 +180,37 @@ func (d *RootDocument) Update(r *RootDocument) { if len(r.QueryParams) > 0 { d.QueryParams = r.QueryParams } + if len(r.ProductClass) > 0 { + d.ProductClass = r.ProductClass + } + if len(r.AccountType) > 0 { + d.AccountType = r.AccountType + } +} + +func (d *RootDocument) UpdateMetadata(r *RootDocument) { + // Version and QueryParams are cloud data, so not changed + if r.Bitmap > 0 { + d.Bitmap = r.Bitmap + } + if len(r.FirmwareVersion) > 0 { + d.FirmwareVersion = r.FirmwareVersion + } + if len(r.ModelName) > 0 { + d.ModelName = r.ModelName + } + if len(r.PartnerId) > 0 { + d.PartnerId = r.PartnerId + } + if len(r.SchemaVersion) > 0 { + d.SchemaVersion = r.SchemaVersion + } + if len(r.ProductClass) > 0 { + d.ProductClass = r.ProductClass + } + if len(r.AccountType) > 0 { + d.AccountType = r.AccountType + } } func (d *RootDocument) String() string { @@ -170,3 +226,7 @@ func (d *RootDocument) Clone() *RootDocument { obj := *d return &obj } + +func (d *RootDocument) Locked() bool { + return d.LockedTill > 0 && int(time.Now().UnixMilli()) < d.LockedTill +} diff --git a/common/root_document_test.go b/common/root_document_test.go index a3c2d88..41ceaff 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "strings" "testing" + "time" "gotest.tools/assert" ) @@ -31,18 +32,18 @@ func TestRootDocumentCompare(t *testing.T) { modelName := "bar" partnerId := "cox" firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") rootdoc2 := rootdoc1.Clone() enum := rootdoc1.Compare(rootdoc2) assert.Equal(t, enum, RootDocumentEquals) firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "") + rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "", "", "") enum = rootdoc1.Compare(rootdoc3) assert.Equal(t, enum, RootDocumentMetaChanged) version4 := "3456" - rootdoc4 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version4, "") + rootdoc4 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version4, "", "", "") enum = rootdoc1.Compare(rootdoc4) assert.Equal(t, enum, RootDocumentVersionOnlyChanged) } @@ -54,7 +55,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName1 := "TG4482" partnerId1 := "" firmwareVersion1 := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc1 := NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "") + rootdoc1 := NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", "", "") bitmap2 := 123 version2 := "bar" @@ -62,7 +63,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName2 := "TG4482" partnerId2 := "cox" firmwareVersion2 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc2 := NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, "") + rootdoc2 := NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, "", "", "") bitmap3 := 123 version3 := "bar" @@ -70,7 +71,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName3 := "TG4482" partnerId3 := "cox" firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc3 := NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, "") + rootdoc3 := NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, "", "", "") rootdoc1.Update(rootdoc2) assert.Equal(t, *rootdoc1, *rootdoc3) @@ -80,20 +81,36 @@ func TestRootDocumentUpdate(t *testing.T) { assert.Assert(t, !strings.Contains(line, "map[")) } -func TestRootDocumentIsDifferent(t *testing.T) { +func TestRootDocumentEquals(t *testing.T) { bitmap := 123 version := "foo" schemaVersion := "33554433-1.3,33554434-1.3" modelName := "bar" partnerId := "cox" firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" - rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") rootdoc2 := rootdoc1.Clone() - isDiff := rootdoc1.IsDifferent(rootdoc2) - assert.Assert(t, !isDiff) + ok := rootdoc1.Equals(rootdoc2) + assert.Assert(t, ok) firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "") - isDiff = rootdoc1.IsDifferent(rootdoc3) - assert.Assert(t, isDiff) + rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "", "", "") + ok = rootdoc1.Equals(rootdoc3) + assert.Assert(t, !ok) +} + +func TestRootDocumentLocked(t *testing.T) { + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") + assert.Assert(t, !rootdoc.Locked()) + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + assert.Assert(t, rootdoc.Locked()) + time.Sleep(time.Duration(1) * time.Second) + assert.Assert(t, !rootdoc.Locked()) } diff --git a/common/server_config.go b/common/server_config.go index b37da7f..2a0296a 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -14,13 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "fmt" - "io/ioutil" "os" + "strings" "github.com/go-akka/configuration" ) @@ -31,6 +31,7 @@ var ( "/app/webconfig/test_webconfig.conf", "../config/sample_webconfig.conf", "/app/webconfig/webconfig.conf", + "/app/webconfig/conf/webconfig.conf", } ) @@ -40,7 +41,7 @@ type ServerConfig struct { } func NewServerConfig(configFile string) (*ServerConfig, error) { - configBytes, err := ioutil.ReadFile(configFile) + configBytes, err := os.ReadFile(configFile) if err != nil { return nil, NewError(err) } @@ -65,6 +66,32 @@ func (c *ServerConfig) KafkaClusterNames() []string { return clustersNode.GetKeys() } +// NOTE that "bad" entries (keys without values, ill-formatted) can still be added +// hence no parsing error +func (c *ServerConfig) AddConfig(args ...string) { + lines := []string{ + string(c.configBytes), + } + lines = append(lines, args...) + ss := strings.Join(lines, "\n") + c.Config = configuration.ParseString(ss) + c.configBytes = []byte(ss) +} + +// copy the config and add extra items +func (c *ServerConfig) Copy(args ...string) *ServerConfig { + lines := []string{ + string(c.configBytes), + } + lines = append(lines, args...) + ss := strings.Join(lines, "\n") + conf := configuration.ParseString(ss) + return &ServerConfig{ + Config: conf, + configBytes: []byte(ss), + } +} + func GetTestConfigFile() (string, error) { testConfigFile := os.Getenv("TEST_CONFIG_FILE") if len(testConfigFile) > 0 { diff --git a/common/subdocument.go b/common/subdocument.go index 836eb37..bd54947 100644 --- a/common/subdocument.go +++ b/common/subdocument.go @@ -102,6 +102,14 @@ func (d *SubDocument) SetVersion(version *string) { d.version = version } +// convenient function +func (d *SubDocument) GetVersion() string { + if d.version != nil { + return *d.version + } + return "" +} + func (d *SubDocument) State() *int { return d.state } @@ -110,6 +118,14 @@ func (d *SubDocument) SetState(state *int) { d.state = state } +// convenient function +func (d *SubDocument) GetState() int { + if d.state != nil { + return *d.state + } + return 0 +} + func (d *SubDocument) UpdatedTime() *int { return d.updatedTime } @@ -142,59 +158,59 @@ func (d *SubDocument) SetExpiry(expiry *int) { d.expiry = expiry } -func (d *SubDocument) Equals(tdoc *SubDocument) error { +func (d *SubDocument) Equals(tdoc *SubDocument) (bool, error) { if d.HasPayload() && tdoc.HasPayload() { if !bytes.Equal(d.Payload(), tdoc.Payload()) { - err := fmt.Errorf("d.Payload() != tdoc.Payload(), len(d.Payload())=%v, len(tdoc.Payload())=%v", len(d.payload), len(tdoc.payload)) - return NewError(err) + err := fmt.Errorf("d.Payload() != tdoc.Payload(), d.Payload()=%v, tdoc.Payload()=%v", d.payload, tdoc.payload) + return false, NewError(err) } } else { if d.HasPayload() != tdoc.HasPayload() { err := fmt.Errorf("d.HasPayload() != tdoc.HasPayload()") - return NewError(err) + return false, NewError(err) } } if d.Version() != nil && tdoc.Version() != nil { if *d.Version() != *tdoc.Version() { err := fmt.Errorf("*d.Version()[%v] != *tdoc.Version()[%v]", *d.Version(), *tdoc.Version()) - return NewError(err) + return false, NewError(err) } } else { if d.Version() != tdoc.Version() { err := fmt.Errorf("d.Version()[%v] != tdoc.Version()[%v]", d.Version(), tdoc.Version()) - return NewError(err) + return false, NewError(err) } } if d.UpdatedTime() != nil && tdoc.UpdatedTime() != nil { if *d.UpdatedTime() != *tdoc.UpdatedTime() { err := fmt.Errorf("*d.UpdatedTime()[%v] != *tdoc.UpdatedTime()[%v]", *d.UpdatedTime(), *tdoc.UpdatedTime()) - return NewError(err) + return false, NewError(err) } } else { if d.UpdatedTime() != tdoc.UpdatedTime() { err := fmt.Errorf("d.UpdatedTime()[%v] != tdoc.UpdatedTime()[%v]", d.UpdatedTime(), tdoc.UpdatedTime()) - return NewError(err) + return false, NewError(err) } } if d.State() != nil && tdoc.State() != nil { if *d.State() != *tdoc.State() { err := fmt.Errorf("*d.State()[%v] != *tdoc.State()[%v]", *d.State(), *tdoc.State()) - return NewError(err) + return false, NewError(err) } } else { if d.State() != tdoc.State() { err := fmt.Errorf("d.State()[%v] != tdoc.State()[%v]", d.State(), tdoc.State()) - return NewError(err) + return false, NewError(err) } } if d.ErrorCode() != nil && tdoc.ErrorCode() != nil { if *d.ErrorCode() != *tdoc.ErrorCode() { err := fmt.Errorf("*d.ErrorCode()[%v] != *tdoc.ErrorCode()[%v]", *d.ErrorCode(), *tdoc.ErrorCode()) - return NewError(err) + return false, NewError(err) } } else { var dErrorCode, tdocErrorCode int @@ -206,14 +222,14 @@ func (d *SubDocument) Equals(tdoc *SubDocument) error { } if dErrorCode != tdocErrorCode { err := fmt.Errorf("d.ErrorCode()[%v] != tdoc.ErrorCode()[%v]", d.ErrorCode(), tdoc.ErrorCode()) - return NewError(err) + return false, NewError(err) } } if d.ErrorDetails() != nil && tdoc.ErrorDetails() != nil { if *d.ErrorDetails() != *tdoc.ErrorDetails() { err := fmt.Errorf("*d.ErrorDetails()[%v] != *tdoc.ErrorDetails()[%v]", *d.ErrorDetails(), *tdoc.ErrorDetails()) - return NewError(err) + return false, NewError(err) } } else { var dErrorDetails, tdocErrorDetails string @@ -225,23 +241,23 @@ func (d *SubDocument) Equals(tdoc *SubDocument) error { } if dErrorDetails != tdocErrorDetails { err := fmt.Errorf("d.ErrorDetails()[%v] != tdoc.ErrorDetails()[%v]", d.ErrorDetails(), tdoc.ErrorDetails()) - return NewError(err) + return false, NewError(err) } } if d.Expiry() != nil && tdoc.Expiry() != nil { if *d.Expiry() != *tdoc.Expiry() { err := fmt.Errorf("*d.Expiry()[%v] != *tdoc.Expiry()[%v]", *d.Expiry(), *tdoc.Expiry()) - return NewError(err) + return false, NewError(err) } } else { if d.Expiry() != tdoc.Expiry() { err := fmt.Errorf("d.Expiry()[%v] != tdoc.Expiry()[%v]", d.Expiry(), tdoc.Expiry()) - return NewError(err) + return false, NewError(err) } } - return nil + return true, nil } func (d *SubDocument) NeedsUpdateForHttp304() bool { diff --git a/common/subdocument_test.go b/common/subdocument_test.go index 3b4bf1a..379ad1b 100644 --- a/common/subdocument_test.go +++ b/common/subdocument_test.go @@ -32,6 +32,12 @@ func TestSubDocumentString(t *testing.T) { errorCode := 103 errorDetails := "cannot parse" - doc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) - assert.Assert(t, doc != nil) + subdoc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) + assert.Assert(t, subdoc != nil) + + subdoc = &SubDocument{} + tgtVersion := subdoc.GetVersion() + assert.Equal(t, tgtVersion, "") + tgtState := subdoc.GetState() + assert.Equal(t, tgtState, 0) } diff --git a/common/writer.go b/common/writer.go index e8a16ee..8ea8504 100644 --- a/common/writer.go +++ b/common/writer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -38,9 +38,9 @@ func WriteMultipartBytes(mparts []Multipart) ([]byte, error) { writer.SetBoundary(Boundary) for _, m := range mparts { header := textproto.MIMEHeader{ - "Content-type": {"application/msgpack"}, - "Namespace": {m.Name}, - "Etag": {m.Version}, + HeaderContentType: {HeaderApplicationMsgpack}, + "Namespace": {m.Name}, + "Etag": {m.Version}, } p, err := writer.CreatePart(header) if err != nil { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 2655e1f..84c6849 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -4,12 +4,23 @@ webconfig { } panic_exit_enabled = false - traceparent_parent_id = "0000000000000001" - tracestate_vendor_id = "webconfig" + tracing { + moracide_tag_prefix = "X-Cl-Experiment" + otel { + enabled = false + endpoint = "127.0.0.1:4318" + operation_name = "http.request" + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop" + } + } // build info - code_git_commit = "2ac7ff4" - build_time = "Thu Feb 14 01:57:26 2019 UTC" + code_git_commit = "oswebconfig-25.2.2.3-249.cea2264" + build_time = "2025-02-24_18:50:55_UTC" token_api_enabled = true server { @@ -23,6 +34,7 @@ webconfig { log { level = "info" file = "/tmp/webconfig.log" + sarama_logger_enabled = false } metrics { @@ -37,10 +49,11 @@ webconfig { read_timeout_in_secs = 142 max_idle_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "https://api.webpa.comcast.net" + host = "http://localhost:12345" async_poke_enabled = false async_poke_concurrent_calls = 100 api_version = "v2" + url_template = "%s/%s/%s" } xconf { @@ -51,7 +64,8 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "http://qa2.xconfds.coast.xcal.tv:8080" + host = "http://localhost:12346" + url_template = "%s/%s" } mqtt { @@ -62,7 +76,8 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "https://hcbroker.staging.us-west-2.plume.comcast.net" + host = "http://localhost:12347" + url_template = "%s/%s" } upstream { @@ -74,8 +89,9 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "http://localhost:9009" - url_template = "/api/v1/device/%v/upstream" + host = "http://localhost:12348" + url_template = "%s/%s" + profile_url_template = "%s/%s/%s" } http_client { @@ -155,6 +171,21 @@ webconfig { user = "dbuser" test_keyspace = "test_webconfig" is_ssl_enabled = true + + // TLS/SSL configuration (optional when is_ssl_enabled = true) + // If tls block is not provided or incomplete, will use insecure TLS + tls { + // Client certificate for mutual TLS (mTLS) - optional + cert_file = "/path/to/client-cert.pem" + key_file = "/path/to/client-key.pem" + + // CA certificate for server verification - optional + ca_cert_file = "/path/to/ca-cert.pem" + + // Skip certificate verification (INSECURE - for testing only) + // When true, allows TLS without certificates or CA validation + insecure_skip_verify = false + } } yugabyte { @@ -174,6 +205,21 @@ webconfig { // TODO change to false for CI/CD is_ssl_enabled = true + + // TLS/SSL configuration (optional when is_ssl_enabled = true) + // If tls block is not provided or incomplete, will use insecure TLS + tls { + // Client certificate for mutual TLS (mTLS) - optional + cert_file = "/path/to/client-cert.pem" + key_file = "/path/to/client-key.pem" + + // CA certificate for server verification - optional + ca_cert_file = "/path/to/ca-cert.pem" + + // Skip certificate verification (INSECURE - for testing only) + // When true, allows TLS without certificates or CA validation + insecure_skip_verify = false + } } } @@ -190,6 +236,22 @@ webconfig { messages_per_second = 10 } + // TLS configuration for secure Kafka connections + tls { + enabled = false + // Path to client certificate file (PEM format) + // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) + // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) + cert_file = "/etc/webconfig/kafka/client-cert.pem" + // Path to client private key file (PEM format) + key_file = "/etc/webconfig/kafka/client-key.pem" + // Path to CA certificate file for broker verification (PEM format) + ca_cert_file = "/etc/webconfig/kafka/ca-cert.pem" + // Skip certificate verification (WARNING: use ONLY in dev/test environments, NEVER in production) + // When true, certificate files become optional (TLS without client authentication) + insecure_skip_verify = false + } + // if we want to use more than 1 cluster clusters { mesh { @@ -204,6 +266,17 @@ webconfig { ratelimit { messages_per_second = 10 } + + // TLS configuration for mesh cluster + tls { + enabled = false + // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) + // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) + cert_file = "/etc/webconfig/kafka/mesh-client-cert.pem" + key_file = "/etc/webconfig/kafka/mesh-client-key.pem" + ca_cert_file = "/etc/webconfig/kafka/mesh-ca-cert.pem" + insecure_skip_verify = false + } } east { enabled = false @@ -217,6 +290,17 @@ webconfig { ratelimit { messages_per_second = 10 } + + // TLS configuration for east cluster + tls { + enabled = false + // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) + // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) + cert_file = "/etc/webconfig/kafka/east-client-cert.pem" + key_file = "/etc/webconfig/kafka/east-client-key.pem" + ca_cert_file = "/etc/webconfig/kafka/east-ca-cert.pem" + insecure_skip_verify = false + } } } } @@ -230,4 +314,46 @@ webconfig { // if valid_partners is empty, then all partners are accepted // if valid partners is not empty, all parters NOT in the list will be stored as "unknown" valid_partners = ["company", "vendor1", "vendor2"] + + supplementary_appending_enabled = true + + // correct subdoc states if versions match but not "deployed" + state_correction_enabled= false + + // forward kafka messages if needed + kafka_producer { + enabled = false + brokers = "localhost:9092" + topic = "webconfig_downstream" + + // TLS configuration for Kafka producer + tls { + enabled = false + // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) + // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) + cert_file = "/etc/webconfig/kafka/producer-client-cert.pem" + key_file = "/etc/webconfig/kafka/producer-client-key.pem" + ca_cert_file = "/etc/webconfig/kafka/producer-ca-cert.pem" + insecure_skip_verify = false + } + } + + // this allows the root document locked if needed + lock_root_document_enabled = false + + // get extra profiles from upstream + upstream_profiles_enabled = false + + // only valid query_params formats are accepted + query_params_validation_enabled = false + + // only devices using tokens with trust higher or equal can GET the full document + min_trust = 0 + + // subdoc ids accepted by the validator even if they are without bitmap definition + valid_subdoc_ids = [] + + filter_output_by_bitmap_enabled = false + + bitmap_filter_exempt_subdoc_ids = [] } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index d9ca1f7..d46eb15 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -14,21 +14,25 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( + "crypto/tls" + "crypto/x509" "errors" "fmt" "os" + "strings" "time" + "github.com/go-akka/configuration" + "github.com/gocql/gocql" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gocql/gocql" + log "github.com/sirupsen/logrus" ) const ( @@ -48,10 +52,14 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool + supplementaryPrecookEnabled bool + supplementaryPrecookStateTTLDays int } /* @@ -137,9 +145,16 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl } if isSslEnabled { + tlsConfig, err := loadCassandraTLSConfig(dbconf, dbdriver) + if err != nil { + return nil, common.NewError(err) + } + sslOpts := &gocql.SslOptions{ + Config: tlsConfig, EnableHostVerification: false, } + cluster.SslOpts = sslOpts } @@ -159,18 +174,140 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") + supplementaryPrecookEnabled := conf.GetBoolean("webconfig.supplementary_precook_enabled") + supplementaryPrecookStateTTLDays := int(conf.GetInt32("webconfig.supplementary_precook_state_ttl_days", 7)) return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, + supplementaryPrecookEnabled: supplementaryPrecookEnabled, + supplementaryPrecookStateTTLDays: supplementaryPrecookStateTTLDays, }, nil } +// loadCassandraTLSConfig loads TLS configuration for Cassandra connection. +// Returns a tls.Config with certificates loaded from the configuration. +// The function expects tls.{} block under the database driver config (cassandra or yugabyte). +func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls.Config, error) { + // Check insecure_skip_verify flag first + insecureSkipVerify := dbconf.GetBoolean("tls.insecure_skip_verify") + + // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) + certFile := dbconf.GetString("tls.cert_file") + keyFile := dbconf.GetString("tls.key_file") + caCertFile := dbconf.GetString("tls.ca_cert_file") + + // Create TLS config for Cassandra connection with compatible cipher suite + // Cassandra 3.11.x requires specific cipher suites that are disabled by default in newer Go crypto + tlsConfig := &tls.Config{ + CipherSuites: []uint16{ + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + }, + InsecureSkipVerify: insecureSkipVerify, + } + + // When insecure_skip_verify is true and no cert files configured, skip loading certificates + // This allows TLS without client authentication (server-only TLS) + if insecureSkipVerify && (len(certFile) == 0 || len(keyFile) == 0) { + // Insecure mode without client certificates - skip cert loading + log.WithFields(log.Fields{ + "driver": dbdriver, + }).Warn("Cassandra TLS enabled in insecure mode without client certificates") + } else if len(certFile) > 0 && len(keyFile) > 0 { + // Only validate cert files exist when verification is enabled + if !insecureSkipVerify { + // Validate certificate file exists + if _, err := os.Stat(certFile); os.IsNotExist(err) { + return nil, fmt.Errorf("Cassandra TLS certificate file does not exist: %s", certFile) + } + + // Validate key file exists + if _, err := os.Stat(keyFile); os.IsNotExist(err) { + return nil, fmt.Errorf("Cassandra TLS key file does not exist: %s", keyFile) + } + } + + // Load and parse the certificate and key + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("failed to load Cassandra TLS certificate and key from %s and %s: %v", certFile, keyFile, err) + } + + tlsConfig.Certificates = []tls.Certificate{cert} + log.WithFields(log.Fields{ + "driver": dbdriver, + "cert_file": certFile, + "key_file": keyFile, + }).Info("Loaded Cassandra TLS client certificate for mTLS") + } else if len(certFile) > 0 || len(keyFile) > 0 { + // Partial cert configuration detected - require both cert and key when verification is enabled + if !insecureSkipVerify { + return nil, fmt.Errorf("Cassandra TLS enabled with verification but incomplete certificate configuration (cert: %s, key: %s)", certFile, keyFile) + } + } + + // Load CA certificate if provided (optional when insecure_skip_verify is true) + // When insecure_skip_verify is true and no CA file configured, skip loading CA cert + // This allows TLS without server verification (insecure mode) + if insecureSkipVerify && len(caCertFile) == 0 { + // Insecure mode without CA cert - skip CA loading + log.WithFields(log.Fields{ + "driver": dbdriver, + }).Warn("Cassandra TLS enabled in insecure mode without CA certificate") + } else if len(caCertFile) > 0 { + // Only validate CA cert file exists when verification is enabled + if !insecureSkipVerify { + // Validate CA certificate file exists + if _, err := os.Stat(caCertFile); os.IsNotExist(err) { + return nil, fmt.Errorf("Cassandra TLS CA certificate file does not exist: %s", caCertFile) + } + } + + // Load CA certificate + caCert, err := os.ReadFile(caCertFile) + if err != nil { + return nil, fmt.Errorf("failed to read Cassandra TLS CA certificate from %s: %v", caCertFile, err) + } + + // Parse CA certificate + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to parse Cassandra TLS CA certificate from %s", caCertFile) + } + + tlsConfig.RootCAs = caCertPool + log.WithFields(log.Fields{ + "driver": dbdriver, + "ca_cert_file": caCertFile, + }).Info("Loaded Cassandra TLS CA certificate for server verification") + } + + if insecureSkipVerify { + log.WithFields(log.Fields{ + "driver": dbdriver, + }).Warn("Cassandra TLS certificate verification is disabled (insecure_skip_verify=true). This is insecure and should only be used for testing.") + } + + log.WithFields(log.Fields{ + "driver": dbdriver, + "has_client_cert": len(tlsConfig.Certificates) > 0, + "has_ca_cert": tlsConfig.RootCAs != nil, + "insecure_skip_verify": insecureSkipVerify, + "cipher_suites": len(tlsConfig.CipherSuites), + }).Info("TLS configuration loaded for Cassandra connection") + + return tlsConfig, nil +} + func (c *CassandraClient) Codec() *security.AesCodec { return c.AesCodec } @@ -216,6 +353,22 @@ func (c *CassandraClient) SetEncryptedSubdocIds(x []string) { c.encryptedSubdocIds = x } +func (c *CassandraClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + +func (c *CassandraClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *CassandraClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) @@ -263,13 +416,39 @@ func GetTestCassandraClient(conf *configuration.Config, testOnly bool) (*Cassand if err != nil { return nil, common.NewError(err) } - err = tdbclient.SetUp() - if err != nil { - return nil, common.NewError(err) + + // Check if SKIP_TABLE_CREATION environment variable is set (case-insensitive) + skipTableCreation := false + if skipEnv, exists := os.LookupEnv("SKIP_TABLE_CREATION"); exists { + skipTableCreation = strings.EqualFold(skipEnv, "true") || strings.EqualFold(skipEnv, "1") || strings.EqualFold(skipEnv, "yes") } - err = tdbclient.TearDown() - if err != nil { - return nil, common.NewError(err) + + if !skipTableCreation { + err = tdbclient.SetUp() + if err != nil { + return nil, common.NewError(err) + } + err = tdbclient.TearDown() + if err != nil { + return nil, common.NewError(err) + } } + return tdbclient, nil } + +func (c *CassandraClient) SupplementaryPrecookEnabled() bool { + return c.supplementaryPrecookEnabled +} + +func (c *CassandraClient) SetSupplementaryPrecookEnabled(enabled bool) { + c.supplementaryPrecookEnabled = enabled +} + +func (c *CassandraClient) SupplementaryPrecookStateTTLDays() int { + return c.supplementaryPrecookStateTTLDays +} + +func (c *CassandraClient) SetSupplementaryPrecookStateTTLDays(days int) { + c.supplementaryPrecookStateTTLDays = days +} diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 57c1734..77c882e 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -38,6 +38,22 @@ func TestCassandraClient(t *testing.T) { tgtSubdocIds := tdbclient.EncryptedSubdocIds() assert.Assert(t, len(tgtSubdocIds) == 4) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/cassandra/delete_columns_test.go b/db/cassandra/delete_columns_test.go new file mode 100644 index 0000000..3c65e77 --- /dev/null +++ b/db/cassandra/delete_columns_test.go @@ -0,0 +1,196 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestDeleteSubDocumentColumnsExpiry(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "telemetry" + + // Create subdocument with expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Deployed + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify expiry is set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Equal(t, *fetchedSubdoc.Expiry(), futureExpiry) + + // Delete the expiry column + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry") + assert.NilError(t, err) + + // Verify expiry is now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} + +func TestDeleteSubDocumentColumnsMultiple(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "mesh" + + // Create subdocument with multiple fields + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.InDeployment + errorCode := 100 + errorDetails := "some error" + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with all fields set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errorCode, &errorDetails) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify all fields are set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Assert(t, fetchedSubdoc.ErrorCode() != nil) + assert.Assert(t, fetchedSubdoc.ErrorDetails() != nil) + + // Delete multiple columns + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry", "error_code", "error_details") + assert.NilError(t, err) + + // Verify deleted columns are now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + // Note: error_code and error_details might have default values, so we just verify no error occurred + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} + +func TestDeleteSubDocumentColumnsEmptyList(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "test" + + // Create a simple subdocument + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcState := common.PendingDownload + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + + fields := log.Fields{} + + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Call with empty column list should be no-op + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId) + assert.NilError(t, err) + + // Verify subdocument is unchanged + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *fetchedSubdoc.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc.State(), srcState) +} + +func TestDeleteSubDocumentColumnsErrorFields(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "telemetry" + + // Create subdocument with error fields and expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Failure + errorCode := 204 + errorDetails := "failed_retrying:Error unsupported namespace" + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with error fields and expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errorCode, &errorDetails) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify all fields are set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Equal(t, *fetchedSubdoc.Expiry(), futureExpiry) + assert.Assert(t, fetchedSubdoc.ErrorCode() != nil) + assert.Equal(t, *fetchedSubdoc.ErrorCode(), errorCode) + assert.Assert(t, fetchedSubdoc.ErrorDetails() != nil) + assert.Equal(t, *fetchedSubdoc.ErrorDetails(), errorDetails) + + // Delete expiry and error fields + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry", "error_code", "error_details") + assert.NilError(t, err) + + // Verify deleted columns are now cleared + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + // Note: Cassandra returns default values (0, "") for error_code and error_details after DELETE + assert.Assert(t, fetchedSubdoc2.ErrorCode() != nil) + assert.Equal(t, *fetchedSubdoc2.ErrorCode(), 0) + assert.Assert(t, fetchedSubdoc2.ErrorDetails() != nil) + assert.Equal(t, *fetchedSubdoc2.ErrorDetails(), "") + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} diff --git a/db/cassandra/document.go b/db/cassandra/document.go index 2eaf58c..ec10cd3 100644 --- a/db/cassandra/document.go +++ b/db/cassandra/document.go @@ -14,22 +14,22 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "fmt" + "strings" "time" + "github.com/gocql/gocql" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gocql/gocql" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) -// NOTE this func (c *CassandraClient) GetSubDocument(cpeMac string, groupId string) (*common.SubDocument, error) { var err error var payload []byte @@ -57,6 +57,20 @@ func (c *CassandraClient) GetSubDocument(cpeMac string, groupId string) (*common } } + // Check if payload contains a reference to a refsubdocument + if refId, ok := db.GetRefId(payload); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + if !c.IsDbNotFound(err) { + return nil, common.NewError(err) + } + // If refsubdocument not found, continue with the reference payload + } else { + // Replace payload with the actual payload from refsubdocument + payload = refsubdocument.Payload() + } + } + if x := int(updatedTime.UnixNano() / 1000000); x > 0 { updatedTimeTsPtr = &x } @@ -196,6 +210,23 @@ func (c *CassandraClient) DeleteSubDocument(cpeMac string, groupId string) error return nil } +func (c *CassandraClient) DeleteSubDocumentColumns(cpeMac string, groupId string, columns ...string) error { + if len(columns) == 0 { + return nil + } + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // Build DELETE statement for specific columns + // In Cassandra: DELETE col1, col2 FROM table WHERE conditions + stmt := fmt.Sprintf("DELETE %v FROM xpc_group_config WHERE cpe_mac=? AND group_id=?", strings.Join(columns, ",")) + if err := c.Query(stmt, cpeMac, groupId).Exec(); err != nil { + return common.NewError(err) + } + return nil +} + func (c *CassandraClient) DeleteDocument(cpeMac string) error { c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() @@ -283,7 +314,11 @@ func (c *CassandraClient) GetDocument(cpeMac string, xargs ...interface{}) (fndo if c.IsEncryptedGroup(groupId) { payload, err = c.DecryptBytes(payload) if err != nil { - return nil, common.NewError(err) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "subdoc" + tfields["subdoc_id"] = groupId + log.WithFields(tfields).Warn(err) + continue } } diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index 24bf0ac..9dd9bf0 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -14,11 +14,10 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( - "crypto/rand" "net/http" "strconv" "testing" @@ -36,9 +35,7 @@ func TestMocaSubDocument(t *testing.T) { subdocId := "moca" // prepare the source data - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -57,16 +54,16 @@ func TestMocaSubDocument(t *testing.T) { fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) assert.NilError(t, err) - assert.Assert(t, srcSubdoc.Equals(fetchedSubdoc)) + ok, err := srcSubdoc.Equals(fetchedSubdoc) + assert.NilError(t, err) + assert.Assert(t, ok) } func TestPrivatessidSubDocument(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() groupId := "privatessid" - slen := util.RandomInt(100) + 16 - srcBytes := make([]byte, slen) - rand.Read(srcBytes) + srcBytes := common.RandomBytes(16, 116) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -82,7 +79,9 @@ func TestPrivatessidSubDocument(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - assert.Assert(t, srcDoc.Equals(fetchedDoc)) + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) } func TestMultiSubDocuments(t *testing.T) { @@ -114,8 +113,9 @@ func TestMultiSubDocuments(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) } doc, err := tdbclient.GetDocument(cpeMac) @@ -125,7 +125,9 @@ func TestMultiSubDocuments(t *testing.T) { for k, v := range srcmap { dv := doc.SubDocument(k) assert.Assert(t, dv != nil) - assert.Assert(t, v.Equals(dv)) + ok, err := v.Equals(dv) + assert.NilError(t, err) + assert.Assert(t, ok) } // ==== delete a document ==== @@ -178,8 +180,9 @@ func TestBlockedSubdocIds(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) } // add version1 and bitmap1 version1 := "indigo violet" @@ -198,7 +201,7 @@ func TestBlockedSubdocIds(t *testing.T) { rHeader.Set(common.HeaderSupportedDocs, rdkSupportedDocsHeaderStr) - document, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) + document, _, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) assert.NilError(t, err) assert.Assert(t, document.Length() == 3) versionMap := document.VersionMap() @@ -223,7 +226,7 @@ func TestExpirySubDocument(t *testing.T) { // prepare some subdocs subdocIds := []string{"privatessid", "lan", "wan"} for _, subdocId := range subdocIds { - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -238,7 +241,7 @@ func TestExpirySubDocument(t *testing.T) { assert.Assert(t, doc.Length() == len(subdocIds)) // add an expiry-type but not-yet-expired subdoc - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) now := time.Now() nowTs := int(now.UnixNano() / 1000000) futureT := now.AddDate(0, 0, 2) @@ -257,7 +260,7 @@ func TestExpirySubDocument(t *testing.T) { assert.Assert(t, doc.Length() == len(subdocIds)+1) // set an expired subdoc - srcBytes = util.RandomBytes(100, 150) + srcBytes = common.RandomBytes(100, 150) past := now.Add(time.Duration(-1) * time.Hour) pastTs := int(past.UnixNano() / 1000000) srcVersion = strconv.Itoa(nowTs) @@ -274,3 +277,107 @@ func TestExpirySubDocument(t *testing.T) { assert.NilError(t, err) assert.Assert(t, doc.Length() == len(subdocIds)+1) } + +func TestGetSubDocumentWithReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "lan" + refId := util.GetMurmur3Hash([]byte(cpeMac + subdocId)) + + // Step 1: Create a reference subdocument with actual payload + actualPayload := common.RandomBytes(100, 200) + actualVersion := util.GetMurmur3Hash(actualPayload) + refSubdoc := common.NewRefSubDocument(actualPayload, &actualVersion) + + err := tdbclient.SetRefSubDocument(refId, refSubdoc) + assert.NilError(t, err) + + // Step 2: Create a subdocument with reference payload (4 zero bytes + refId) + referencePayload := append(make([]byte, 4), []byte(refId)...) + refVersion := util.GetMurmur3Hash(referencePayload) + refState := common.InDeployment + refUpdatedTime := int(time.Now().UnixMilli()) + + subdocWithRef := common.NewSubDocument(referencePayload, &refVersion, &refState, &refUpdatedTime, nil, nil) + fields := log.Fields{} + err = tdbclient.SetSubDocument(cpeMac, subdocId, subdocWithRef, fields) + assert.NilError(t, err) + + // Step 3: Call GetSubDocument and verify it returns the actual payload, not the reference + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is the actual payload from refsubdocument, not the reference + assert.DeepEqual(t, fetchedSubdoc.Payload(), actualPayload) + + // Verify other fields remain unchanged + assert.Equal(t, *fetchedSubdoc.Version(), refVersion) + assert.Equal(t, *fetchedSubdoc.State(), refState) + assert.Equal(t, *fetchedSubdoc.UpdatedTime(), refUpdatedTime) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) +} + +func TestGetSubDocumentWithMissingReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "wan" + refId := util.GetMurmur3Hash([]byte(cpeMac + subdocId + "nonexistent")) + + // Create a subdocument with reference payload pointing to non-existent refsubdocument + referencePayload := append(make([]byte, 4), []byte(refId)...) + refVersion := util.GetMurmur3Hash(referencePayload) + refState := common.InDeployment + refUpdatedTime := int(time.Now().UnixMilli()) + + subdocWithRef := common.NewSubDocument(referencePayload, &refVersion, &refState, &refUpdatedTime, nil, nil) + fields := log.Fields{} + err := tdbclient.SetSubDocument(cpeMac, subdocId, subdocWithRef, fields) + assert.NilError(t, err) + + // Call GetSubDocument - should return the reference payload since refsubdocument doesn't exist + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is the reference payload (since refsubdocument was not found) + assert.DeepEqual(t, fetchedSubdoc.Payload(), referencePayload) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) +} + +func TestGetSubDocumentWithoutReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "mesh" + + // Create a regular subdocument without any reference + regularPayload := common.RandomBytes(100, 200) + regularVersion := util.GetMurmur3Hash(regularPayload) + regularState := common.Deployed + regularUpdatedTime := int(time.Now().UnixMilli()) + + subdoc := common.NewSubDocument(regularPayload, ®ularVersion, ®ularState, ®ularUpdatedTime, nil, nil) + fields := log.Fields{} + err := tdbclient.SetSubDocument(cpeMac, subdocId, subdoc, fields) + assert.NilError(t, err) + + // Call GetSubDocument - should return the regular payload unchanged + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is unchanged + assert.DeepEqual(t, fetchedSubdoc.Payload(), regularPayload) + assert.Equal(t, *fetchedSubdoc.Version(), regularVersion) + assert.Equal(t, *fetchedSubdoc.State(), regularState) + assert.Equal(t, *fetchedSubdoc.UpdatedTime(), regularUpdatedTime) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) +} diff --git a/db/cassandra/refsubdocument.go b/db/cassandra/refsubdocument.go new file mode 100644 index 0000000..1c9ca9f --- /dev/null +++ b/db/cassandra/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/gocql/gocql" +) + +func (c *CassandraClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + var payload []byte + var version string + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "SELECT payload,version FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Scan(&payload, &version); err != nil { + return nil, common.NewError(err) + } + + if len(payload) == 0 { + return nil, common.NewError(gocql.ErrNotFound) + } + + refsubdoc := common.NewRefSubDocument(payload, &version) + return refsubdoc, nil +} + +func (c *CassandraClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) (fnerr error) { + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil && len(refsubdoc.Payload()) > 0 { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + stmt := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + if err := c.Query(stmt, values...).Exec(); err != nil { + return common.NewError(err) + } + return nil +} + +func (c *CassandraClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "DELETE FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Exec(); err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/cassandra/refsubdocument_test.go b/db/cassandra/refsubdocument_test.go new file mode 100644 index 0000000..fca3fad --- /dev/null +++ b/db/cassandra/refsubdocument_test.go @@ -0,0 +1,56 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package cassandra + +import ( + "testing" + + "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/cassandra/root_document.go b/db/cassandra/root_document.go index 6d3d554..3a9b0be 100644 --- a/db/cassandra/root_document.go +++ b/db/cassandra/root_document.go @@ -14,15 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "fmt" + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" - "github.com/prometheus/client_golang/prometheus" ) // shared.go: err := c.Query(stmt, cpeMac).MapScan(dict) @@ -32,11 +33,17 @@ func (c *CassandraClient) GetRootDocument(cpeMac string) (*common.RootDocument, defer func() { <-c.concurrentQueries }() var rd common.RootDocument - stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params FROM root_document WHERE cpe_mac=?" - err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams) + var tobj time.Time + stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till,product_class,account_type FROM root_document WHERE cpe_mac=?" + err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj, &rd.ProductClass, &rd.AccountType) if err != nil { return nil, common.NewError(err) } + if tobj.IsZero() { + rd.LockedTill = 0 + } else { + rd.LockedTill = int(tobj.UnixMilli()) + } return &rd, nil } diff --git a/db/cassandra/root_document_test.go b/db/cassandra/root_document_test.go index 04ceb1f..d9e5d5f 100644 --- a/db/cassandra/root_document_test.go +++ b/db/cassandra/root_document_test.go @@ -14,11 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" @@ -29,7 +30,7 @@ func TestRootDocumentOperations(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() bitmap := 123 version := "foo" - rdoc := common.NewRootDocument(bitmap, "", "", "", "", version, "") + rdoc := common.NewRootDocument(bitmap, "", "", "", "", version, "", "", "") err := tdbclient.SetRootDocument(cpeMac, rdoc) assert.NilError(t, err) @@ -85,7 +86,7 @@ func TestRootDocumentDb(t *testing.T) { // set by a RootDocument version4 := "indigo violet" bitmap4 := 67 - rdoc4 := common.NewRootDocument(bitmap4, "", "", "", "", version4, "") + rdoc4 := common.NewRootDocument(bitmap4, "", "", "", "", version4, "", "", "") err = tdbclient.SetRootDocument(cpeMac, rdoc4) assert.NilError(t, err) fetched, err := tdbclient.GetRootDocument(cpeMac) @@ -133,7 +134,7 @@ func TestRootDocumentUpdate(t *testing.T) { partnerId1 := "" firmwareVersion1 := "TG4482PC2_4.12p7s3_PROD_sey" queryParams1 := "stormReadyWifi=true" - srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, queryParams1) + srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, queryParams1, "", "") err = tdbclient.SetRootDocument(cpeMac, srcRootdoc1) assert.NilError(t, err) @@ -150,7 +151,7 @@ func TestRootDocumentUpdate(t *testing.T) { partnerId2 := "cox" firmwareVersion2 := "TG4482PC2_4.14p7s3_PROD_sey" queryParams2 := "stormReadyWifi=true" - rootdoc2 := common.NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, queryParams2) + rootdoc2 := common.NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, queryParams2, "", "") err = tdbclient.SetRootDocument(cpeMac, rootdoc2) assert.NilError(t, err) @@ -163,9 +164,36 @@ func TestRootDocumentUpdate(t *testing.T) { partnerId3 := "cox" firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" queryParams3 := "stormReadyWifi=true" - rootdoc3 := common.NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, queryParams3) + rootdoc3 := common.NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, queryParams3, "", "") tgtRootdoc3, err := tdbclient.GetRootDocument(cpeMac) assert.NilError(t, err) assert.DeepEqual(t, tgtRootdoc3, rootdoc3) } + +func TestRootDocumentLocked(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + + rootdoc := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "", "", "") + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + + err := tdbclient.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + fetched, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.DeepEqual(t, rootdoc, fetched) + assert.Assert(t, fetched.Locked()) + + time.Sleep(time.Duration(1) * time.Second) + + assert.Assert(t, !fetched.Locked()) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index 4d0ac95..d216277 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -38,12 +38,21 @@ var ( `CREATE TABLE IF NOT EXISTS root_document ( cpe_mac text PRIMARY KEY, bitmap bigint, + account_type text, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + product_class text, + query_params text, route text, schema_version text, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } @@ -62,9 +71,11 @@ var ( "root_document": { "cpe_mac": gocql.TypeText, "bitmap": gocql.TypeBigInt, + "account_type": gocql.TypeText, "firmware_version": gocql.TypeText, "model_name": gocql.TypeText, "partner_id": gocql.TypeText, + "product_class": gocql.TypeText, "route": gocql.TypeText, "schema_version": gocql.TypeText, "version": gocql.TypeText, diff --git a/db/cassandra/state_update_telemetry_test.go b/db/cassandra/state_update_telemetry_test.go new file mode 100644 index 0000000..1593a68 --- /dev/null +++ b/db/cassandra/state_update_telemetry_test.go @@ -0,0 +1,229 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestTelemetryStateUpdateWithTTL(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // setup a telemetry subdoc + groupId := "telemetry" + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixMilli()) + srcState := common.PendingDownload + + srcDoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + fields := make(log.Fields) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcDoc, fields) + assert.NilError(t, err) + + fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) + + // Update to success with TTL set to 7 days + template := `{"application_status": "success", "device_id": "mac:%v", "namespace": "telemetry", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes := []byte(fmt.Sprintf(template, cpeMac, srcVersion)) + var m common.EventMessage + err = json.Unmarshal(bbytes, &m) + assert.NilError(t, err) + + // Enable supplementary precook and set TTL to 7 days on database client + supplementaryPrecookStateTTLDays := 7 + tdbclient.SetSupplementaryPrecookEnabled(true) + tdbclient.SetSupplementaryPrecookStateTTLDays(supplementaryPrecookStateTTLDays) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + + subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + assert.Equal(t, *subdoc.ErrorCode(), 0) + assert.Equal(t, *subdoc.ErrorDetails(), "") + + // Verify that expiry was set + assert.Assert(t, subdoc.Expiry() != nil) + expiryTime := *subdoc.Expiry() + currentTime := int(time.Now().UnixMilli()) + expectedExpiry := int(time.Now().Add(time.Duration(supplementaryPrecookStateTTLDays) * 24 * time.Hour).UnixMilli()) + + // Allow some margin for execution time (10 seconds) + margin := int(10 * time.Second / time.Millisecond) + assert.Assert(t, expiryTime >= currentTime) + assert.Assert(t, expiryTime <= expectedExpiry+margin) +} + +func TestTelemetryStateUpdateWithoutTTL(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // setup a telemetry subdoc + groupId := "telemetry" + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixMilli()) + srcState := common.PendingDownload + + srcDoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + fields := make(log.Fields) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcDoc, fields) + assert.NilError(t, err) + + fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) + + // Update to success with TTL set to 0 (no TTL) + template := `{"application_status": "success", "device_id": "mac:%v", "namespace": "telemetry", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes := []byte(fmt.Sprintf(template, cpeMac, srcVersion)) + var m common.EventMessage + err = json.Unmarshal(bbytes, &m) + assert.NilError(t, err) + + // Enable supplementary precook and set TTL to 0 (no expiry) + supplementaryPrecookStateTTLDays := 0 + tdbclient.SetSupplementaryPrecookEnabled(true) + tdbclient.SetSupplementaryPrecookStateTTLDays(supplementaryPrecookStateTTLDays) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + + subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + assert.Equal(t, *subdoc.ErrorCode(), 0) + assert.Equal(t, *subdoc.ErrorDetails(), "") + + // Verify that expiry was NOT set + assert.Assert(t, subdoc.Expiry() == nil) +} + +func TestNonTelemetryStateUpdateNoTTL(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // setup a non-telemetry subdoc (e.g., privatessid) + groupId := "privatessid" + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixMilli()) + srcState := common.PendingDownload + + srcDoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + fields := make(log.Fields) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcDoc, fields) + assert.NilError(t, err) + + fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) + + // Update to success with TTL configured but for non-telemetry subdoc + template := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes := []byte(fmt.Sprintf(template, cpeMac, srcVersion)) + var m common.EventMessage + err = json.Unmarshal(bbytes, &m) + assert.NilError(t, err) + + // Enable supplementary precook and set TTL to 7 days + // but since it's not telemetry, TTL should NOT be set + supplementaryPrecookStateTTLDays := 7 + tdbclient.SetSupplementaryPrecookEnabled(true) + tdbclient.SetSupplementaryPrecookStateTTLDays(supplementaryPrecookStateTTLDays) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + + subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + assert.Equal(t, *subdoc.ErrorCode(), 0) + assert.Equal(t, *subdoc.ErrorDetails(), "") + + // Verify that expiry was NOT set (since it's not telemetry) + assert.Assert(t, subdoc.Expiry() == nil) +} + +func TestTelemetryStateUpdateFailureNoTTL(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // setup a telemetry subdoc + groupId := "telemetry" + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixMilli()) + srcState := common.PendingDownload + + srcDoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + fields := make(log.Fields) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcDoc, fields) + assert.NilError(t, err) + + fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) + + // Update to failure - TTL should NOT be set even with supplementaryPrecookStateTTLDays configured + template := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "telemetry", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template, cpeMac, srcVersion)) + var m common.EventMessage + err = json.Unmarshal(bbytes, &m) + assert.NilError(t, err) + + // Enable supplementary precook and set TTL to 7 days + // But since state is failure, TTL should NOT be set + supplementaryPrecookStateTTLDays := 7 + tdbclient.SetSupplementaryPrecookEnabled(true) + tdbclient.SetSupplementaryPrecookStateTTLDays(supplementaryPrecookStateTTLDays) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + + subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Failure) + assert.Equal(t, *subdoc.ErrorCode(), 204) + assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") + + // Verify that expiry was NOT set (since state is failure, not success) + assert.Assert(t, subdoc.Expiry() == nil) +} diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 4b57677..5c9e1a0 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -35,7 +35,7 @@ func TestStateUpdate1(t *testing.T) { // setup a doct groupId := "privatessid" - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -48,17 +48,19 @@ func TestStateUpdate1(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -66,13 +68,14 @@ func TestStateUpdate1(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success - template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:11.959437", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -85,7 +88,7 @@ func TestStateUpdate2(t *testing.T) { // setup a doct groupId := "privatessid" - srcBytes := util.RandomBytes(100, 150) + srcBytes := common.RandomBytes(100, 150) srcVersion := util.GetMurmur3Hash(srcBytes) srcUpdatedTime := int(time.Now().UnixNano() / 1000000) srcState := common.PendingDownload @@ -98,17 +101,19 @@ func TestStateUpdate2(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -116,13 +121,14 @@ func TestStateUpdate2(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success by http 304 - template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "2023-05-05 07:42:50.395876"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "%v"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) > 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/database_client.go b/db/database_client.go index b074e59..0548a81 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -14,12 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( - "github.com/rdkcentral/webconfig/common" "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" ) type DatabaseClient interface { @@ -30,6 +30,7 @@ type DatabaseClient interface { GetSubDocument(string, string) (*common.SubDocument, error) SetSubDocument(string, string, *common.SubDocument, ...interface{}) error DeleteSubDocument(string, string) error + DeleteSubDocumentColumns(string, string, ...string) error GetDocument(string, ...interface{}) (*common.Document, error) SetDocument(string, *common.Document) error @@ -59,4 +60,22 @@ type DatabaseClient interface { FactoryReset(string) error FirmwareUpdate(string, int, *common.RootDocument) error AppendProfiles(string, []byte) ([]byte, error) + + // reference subdocument + GetRefSubDocument(string) (*common.RefSubDocument, error) + SetRefSubDocument(string, *common.RefSubDocument) error + DeleteRefSubDocument(string) error + + // enable state correction + StateCorrectionEnabled() bool + SetStateCorrectionEnabled(bool) + + LockRootDocumentEnabled() bool + SetLockRootDocumentEnabled(bool) + + // supplementary precook + SupplementaryPrecookEnabled() bool + SetSupplementaryPrecookEnabled(bool) + SupplementaryPrecookStateTTLDays() int + SetSupplementaryPrecookStateTTLDays(int) } diff --git a/db/service.go b/db/service.go index 3da868c..c4de467 100644 --- a/db/service.go +++ b/db/service.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( @@ -22,53 +22,101 @@ import ( "errors" "fmt" "net/http" + "slices" + "sort" "strings" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) +const ( + referenceIndicatorByteLength = 4 +) + +var ( + referenceIndicatorBytes = make([]byte, referenceIndicatorByteLength) +) + // TODO s.MultipartSupplementaryHandler(w, r) should be handled separately // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, []common.EventMessage, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "request" + + // XPC-21583 Validate all headers + rHeader := common.NewReqHeader(inHeader) // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error - supportedDocs := rHeader.Get(common.HeaderSupportedDocs) + messages := []common.EventMessage{} + supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) + if err != nil { + log.WithFields(tfields).Warn(err) + } + if len(supportedDocs) > 0 { - bitmap, err = util.GetCpeBitmap(supportedDocs) + bitmap, err = common.GetCpeBitmap(supportedDocs) if err != nil { log.WithFields(fields).Warn(common.NewError(err)) } } - schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) - modelName := rHeader.Get(common.HeaderModelName) + schemaVersion, err := rHeader.Get(common.HeaderSchemaVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } + schemaVersion = strings.ToLower(schemaVersion) - partnerId := rHeader.Get(common.HeaderPartnerID) + modelName, err := rHeader.Get(common.HeaderModelName) + if err != nil { + log.WithFields(tfields).Warn(err) + } + + partnerId, err := rHeader.Get(common.HeaderPartnerID) + if err != nil { + log.WithFields(tfields).Warn(err) + } if len(partnerId) == 0 { partnerId = fieldsDict.GetString("partner") } - firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + firmwareVersion, err := rHeader.Get(common.HeaderFirmwareVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } + + productClass, err := rHeader.Get(common.HeaderProductClass) + if err != nil { + log.WithFields(tfields).Warn(err) + } + + accountType, err := rHeader.Get(common.HeaderAccountType) + if err != nil { + log.WithFields(tfields).Warn(err) + } // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers - deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") + deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "", productClass, accountType) // ==== parse mac ==== - mac := rHeader.Get(common.HeaderDeviceId) + mac, err := rHeader.Get(common.HeaderDeviceId) + if err != nil { + log.WithFields(tfields).Warn(err) + } var document *common.Document // get version map - deviceVersionMap, versions, err := parseVersionMap(rHeader) + deviceVersionMap, versions, err := parseVersionMap(rHeader, tfields) if err != nil { var gvmErr common.GroupVersionMismatchError if errors.As(err, &gvmErr) { @@ -78,11 +126,11 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field document, err = c.GetDocument(mac, fields) if err != nil { // TODO what about 404 should be included here - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } deviceVersionMap = RebuildDeviceVersionMap(versions, document.VersionMap()) } else { - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -93,7 +141,7 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field cloudRootDocument, err := c.GetRootDocument(mac) if err != nil { if !c.IsDbNotFound(err) { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // no root doc in db, create a new one // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud @@ -106,22 +154,29 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } + // the returned err is dbNotFound - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + // WARNING, this should be removed + // return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + cloudRootDocument = clonedRootDoc.Clone() } // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== var rootCmpEnum int // mget fakes no meta change so that meta are not updated - if rHeader.Get("User-Agent") == "mget" { + userAgent, err := rHeader.Get("User-Agent") + if err != nil { + log.WithFields(tfields).Warn(err) + } + if userAgent == "mget" { rootCmpEnum = common.RootDocumentVersionOnlyChanged } else { rootCmpEnum = cloudRootDocument.Compare(deviceRootDocument) } - if isDiff := cloudRootDocument.IsDifferent(deviceRootDocument); isDiff { + if isEqual := cloudRootDocument.Equals(deviceRootDocument); !isEqual { // need to update rootDoc meta // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud clonedRootDoc := deviceRootDocument.Clone() @@ -133,7 +188,71 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + } + } + + if c.StateCorrectionEnabled() { + if document == nil { + document, err = c.GetDocument(mac, fields) + if err != nil { + if !c.IsDbNotFound(err) { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + } + } + } + updatedTime := int(time.Now().UnixMilli()) + for subdocId, subdocument := range document.Items() { + cloudVersion := subdocument.GetVersion() + cloudState := subdocument.GetState() + if len(cloudVersion) == 0 { + continue + } + cloudErrorCode := *subdocument.ErrorCode() + cloudErrorDetails := *subdocument.ErrorDetails() + deviceVersion := deviceVersionMap[subdocId] + if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { + labels := prometheus.Labels{ + "model": modelName, + "fwversion": firmwareVersion, + } + // update state + newState := common.Deployed + subdocument.SetState(&newState) + subdocument.SetUpdatedTime(&updatedTime) + if cloudErrorCode > 0 { + var newErrorCode int + subdocument.SetErrorCode(&newErrorCode) + } + if len(cloudErrorDetails) > 0 { + var newErrorDetails string + subdocument.SetErrorDetails(&newErrorDetails) + } + if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + } + applicationStatus := "success" + namespace := subdocId + version := cloudVersion + m := common.EventMessage{ + DeviceId: "mac:" + mac, + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + Version: &version, + } + messages = append(messages, m) + } + } + } + + // eval if the root_document is locked + if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } else { + tfields := common.FilterLogFields(fields) + tfields["logger"] = "rootdoc" + log.WithFields(tfields).Warn("dryrun409") } } @@ -142,7 +261,7 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field // create an empty "document" document := common.NewDocument(cloudRootDocument) // no need to update root doc - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentVersionOnlyChanged, common.RootDocumentMissing: // meta unchanged but subdoc versions change ==> new configs // getDoc, then filter @@ -150,7 +269,7 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) @@ -158,23 +277,23 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field for _, subdocId := range c.BlockedSubdocIds() { filteredDocument.DeleteSubDocument(subdocId) } - return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentMetaChanged: // getDoc, send it upstream if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, messages, nil } // default, should not come here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil } func GetValuesStr(length int) string { @@ -209,16 +328,22 @@ func GetSetColumnsStr(columns []string) string { } // deviceVersionMap := parseVersionMap(rHeader, d) -func parseVersionMap(rHeader http.Header) (map[string]string, []string, error) { +func parseVersionMap(rHeader *common.ReqHeader, tfields log.Fields) (map[string]string, []string, error) { deviceVersionMap := make(map[string]string) - queryStr := rHeader.Get(common.HeaderDocName) + queryStr, err := rHeader.Get(common.HeaderDocName) + if err != nil { + log.WithFields(tfields).Warn(err) + } subdocIds := strings.Split(queryStr, ",") if len(queryStr) == 0 { return deviceVersionMap, nil, nil } - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + ifNoneMatch, err := rHeader.Get(common.HeaderIfNoneMatch) + if err != nil { + log.WithFields(tfields).Warn(err) + } versions := strings.Split(ifNoneMatch, ",") if len(subdocIds) != len(versions) { @@ -247,19 +372,25 @@ func HashRootVersion(itf interface{}) string { // if len(mparts) == 0, then the murmur hash value is 0 buffer := bytes.NewBufferString("") - for _, version := range versionMap { - buffer.WriteString(version) + keys := []string{} + for k := range versionMap { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + buffer.WriteString(versionMap[k]) } return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) error { +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) ([]string, error) { + updatedSubdocIds := []string{} // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return nil + return updatedSubdocIds, nil } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) // set metrics labels metricsAgent := "default" @@ -268,7 +399,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return common.NewError(err) + return updatedSubdocIds, common.NewError(err) } labels["client"] = metricsAgent @@ -277,14 +408,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return nil + return updatedSubdocIds, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return common.NewError(err) + return updatedSubdocIds, common.NewError(err) } newState := common.Deployed @@ -293,20 +424,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { + updatedSubdocIds = append(updatedSubdocIds, groupId) newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } } - return nil + return updatedSubdocIds, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return common.NewError(fmt.Errorf("ill-formatted event")) + return updatedSubdocIds, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -319,13 +451,13 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return nil + return updatedSubdocIds, common.NewError(common.ErrPending) } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return common.NewError(err) + return updatedSubdocIds, common.NewError(err) } var oldState int @@ -335,7 +467,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return common.NewError(err) + return updatedSubdocIds, common.NewError(err) + } + } + + if subdoc.Version() != nil && m.Version != nil { + if *subdoc.Version() != *m.Version { + log.WithFields(fields).Warnf("skip update dbversion=%v, m.version=%v", *subdoc.Version(), *m.Version) + return updatedSubdocIds, nil } } @@ -345,12 +484,18 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } newSubdoc := common.NewSubDocument(nil, nil, &state, &updatedTime, errorCodePtr, errorDetailsPtr) + // Set expiry for telemetry subdoc if supplementary precook is enabled, success state, and TTL is configured + if c.SupplementaryPrecookEnabled() && targetGroupId == "telemetry" && state == common.Deployed && c.SupplementaryPrecookStateTTLDays() > 0 { + expiryTime := int(time.Now().Add(time.Duration(c.SupplementaryPrecookStateTTLDays()) * 24 * time.Hour).UnixMilli()) + newSubdoc.SetExpiry(&expiryTime) + } + // metricsAgent handling if m.MetricsAgent != nil { labels["client"] = *m.MetricsAgent @@ -358,12 +503,12 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return common.NewError(err) + return updatedSubdocIds, common.NewError(err) } - return nil + return updatedSubdocIds, nil } -func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, fields log.Fields) error { +func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, deviceVersionMap map[string]string, fields log.Fields) error { var oldState int if oldSubdoc != nil && oldSubdoc.State() != nil { oldState = *oldSubdoc.State() @@ -375,10 +520,29 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old } labels["client"] = "default" - updatedTime := int(time.Now().UnixNano() / 1000000) + var oldSubdocVersion string + if oldSubdoc != nil && oldSubdoc.Version() != nil { + oldSubdocVersion = *oldSubdoc.Version() + } + + if deviceVersion, ok := deviceVersionMap[subdocId]; ok { + if newSubdoc.Version() != nil { + if deviceVersion == *newSubdoc.Version() && oldSubdoc != nil && deviceVersion == oldSubdocVersion { + return nil + } + } + } + + updatedTime := int(time.Now().UnixMilli()) newSubdoc.SetUpdatedTime(&updatedTime) + newState := common.InDeployment newSubdoc.SetState(&newState) + zeroErrorCode := 0 + emptyErrorDetails := "" + newSubdoc.SetErrorCode(&zeroErrorCode) + newSubdoc.SetErrorDetails(&emptyErrorDetails) + err = c.SetSubDocument(cpeMac, subdocId, newSubdoc, oldState, labels, fields) if err != nil { return common.NewError(err) @@ -386,7 +550,7 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old return nil } -func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, fields log.Fields) error { +func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, toDelete bool, versionMap map[string]string, fields log.Fields) error { newRootVersion := upstreamRespEtag if d.RootVersion() != newRootVersion { err := c.SetRootDocumentVersion(cpeMac, newRootVersion) @@ -395,13 +559,28 @@ func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string } } + oldMap := map[string]struct{}{} + for k := range d.Items() { + oldMap[k] = struct{}{} + } + // need to set "state" to proper values like the download is complete for subdocId, newSubdoc := range newDoc.Items() { oldSubdoc := d.SubDocument(subdocId) - err := UpdateSubDocument(c, cpeMac, subdocId, &newSubdoc, oldSubdoc, fields) + err := UpdateSubDocument(c, cpeMac, subdocId, &newSubdoc, oldSubdoc, versionMap, fields) if err != nil { return common.NewError(err) } + delete(oldMap, subdocId) + } + + if toDelete { + for subdocId := range oldMap { + err := c.DeleteSubDocument(cpeMac, subdocId) + if err != nil { + return common.NewError(err) + } + } } return nil } @@ -450,3 +629,83 @@ func RebuildDeviceVersionMap(versions []string, cloudVersionMap map[string]strin } return m } + +func RefreshRootDocumentVersion(doc *common.Document) { + versionMap := doc.VersionMap() + rootVersion := HashRootVersion(versionMap) + rootDoc := doc.GetRootDocument() + if rootDoc != nil { + rootDoc.Version = rootVersion + } +} + +func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerId string, fields log.Fields) (*common.RootDocument, error) { + fieldsDict := make(util.Dict) + fieldsDict.Update(fields) + + // ==== deviceRootDocument should always be created from request header ==== + var bitmap int + var err error + supportedDocs := rHeader.Get(common.HeaderSupportedDocs) + if len(supportedDocs) > 0 { + bitmap, err = common.GetCpeBitmap(supportedDocs) + if err != nil { + log.WithFields(fields).Warn(common.NewError(err)) + } + } + + schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) + modelName := rHeader.Get(common.HeaderModelName) + firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + productClass := rHeader.Get(common.HeaderProductClass) + accountType := rHeader.Get(common.HeaderAccountType) + + // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers + deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "", productClass, accountType) + + // ==== read the cloudRootDocument from db ==== + cloudRootDocument, err := c.GetRootDocument(mac) + if err != nil { + if !c.IsDbNotFound(err) { + return cloudRootDocument, common.NewError(err) + } + cloudRootDocument = deviceRootDocument.Clone() + } else { + cloudRootDocument.Update(deviceRootDocument) + } + + if err := c.SetRootDocument(mac, cloudRootDocument); err != nil { + return cloudRootDocument, common.NewError(err) + } + return cloudRootDocument, nil +} + +func GetRefId(payload []byte) (string, bool) { + if len(payload) > referenceIndicatorByteLength { + prefixBytes := payload[:referenceIndicatorByteLength] + if slices.Equal(referenceIndicatorBytes, prefixBytes) { + suffixBytes := payload[referenceIndicatorByteLength:] + return string(suffixBytes), true + } + } + return "", false +} + +func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log.Fields) (*common.Document, error) { + newDocument := common.NewDocument(document.GetRootDocument()) + for subdocId, subDocument := range document.Items() { + payload := subDocument.Payload() + if refId, ok := GetRefId(payload); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + if c.IsDbNotFound(err) { + continue + } + return nil, common.NewError(err) + } + subDocument.SetPayload(refsubdocument.Payload()) + } + newDocument.SetSubDocument(subdocId, &subDocument) + } + return newDocument, nil +} diff --git a/db/service_test.go b/db/service_test.go index 1afe700..79d712b 100644 --- a/db/service_test.go +++ b/db/service_test.go @@ -14,13 +14,14 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( "testing" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -47,3 +48,30 @@ func TestHashRootVersion(t *testing.T) { root = HashRootVersion(doc.VersionMap()) assert.Assert(t, root != "0") } + +func TestUpdateRootVersion(t *testing.T) { + doc := common.NewDocument(nil) + tt := int(123) + + bbytes1 := common.RandomBytes(100, 150) + v1 := util.GetMurmur3Hash(bbytes1) + subdoc1 := common.NewSubDocument(bbytes1, &v1, nil, &tt, nil, nil) + doc.SetSubDocument("advsecurity", subdoc1) + + bbytes2 := common.RandomBytes(100, 150) + v2 := util.GetMurmur3Hash(bbytes2) + subdoc2 := common.NewSubDocument(bbytes2, &v2, nil, &tt, nil, nil) + doc.SetSubDocument("mesh", subdoc2) + + rootVersion := HashRootVersion(doc.VersionMap()) + assert.Assert(t, rootVersion != "0") + rootDoc := common.NewRootDocument(123, "fw_ver_123", "model_123", "partner_123", "", rootVersion, "", "", "") + doc.SetRootDocument(rootDoc) + + doc.DeleteSubDocument("mesh") + + RefreshRootDocumentVersion(doc) + + newRootVersion := doc.RootVersion() + assert.Assert(t, rootVersion != newRootVersion) +} diff --git a/db/sqlite/delete_columns_test.go b/db/sqlite/delete_columns_test.go new file mode 100644 index 0000000..c7a6995 --- /dev/null +++ b/db/sqlite/delete_columns_test.go @@ -0,0 +1,188 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestDeleteSubDocumentColumnsExpiry(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "telemetry" + + // Create subdocument with expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Deployed + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify expiry is set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Equal(t, *fetchedSubdoc.Expiry(), futureExpiry) + + // Delete the expiry column + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry") + assert.NilError(t, err) + + // Verify expiry is now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} + +func TestDeleteSubDocumentColumnsMultiple(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "mesh" + + // Create subdocument with expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.InDeployment + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify expiry is set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + + // Delete expiry column + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry") + assert.NilError(t, err) + + // Verify expiry is now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} + +func TestDeleteSubDocumentColumnsEmptyList(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "test" + + // Create a simple subdocument + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcState := common.PendingDownload + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + + fields := log.Fields{} + + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, nil, nil) + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Call with empty column list should be no-op + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId) + assert.NilError(t, err) + + // Verify subdocument is unchanged + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *fetchedSubdoc.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc.State(), srcState) +} + +func TestDeleteSubDocumentColumnsErrorFields(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "telemetry" + + // Create subdocument with error fields and expiry + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Failure + errorCode := 204 + errorDetails := "failed_retrying:Error unsupported namespace" + futureExpiry := int(time.Now().Add(24*time.Hour).UnixNano() / 1000000) + + fields := log.Fields{} + + // Create subdocument with error fields and expiry set + srcSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errorCode, &errorDetails) + srcSubdoc.SetExpiry(&futureExpiry) + + // Write to database + err := tdbclient.SetSubDocument(cpeMac, groupId, srcSubdoc, fields) + assert.NilError(t, err) + + // Verify all fields are set + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc.Expiry() != nil) + assert.Equal(t, *fetchedSubdoc.Expiry(), futureExpiry) + assert.Assert(t, fetchedSubdoc.ErrorCode() != nil) + assert.Equal(t, *fetchedSubdoc.ErrorCode(), errorCode) + assert.Assert(t, fetchedSubdoc.ErrorDetails() != nil) + assert.Equal(t, *fetchedSubdoc.ErrorDetails(), errorDetails) + + // Delete expiry and error fields + err = tdbclient.DeleteSubDocumentColumns(cpeMac, groupId, "expiry", "error_code", "error_details") + assert.NilError(t, err) + + // Verify deleted columns are now nil + fetchedSubdoc2, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc2.Expiry() == nil) + assert.Assert(t, fetchedSubdoc2.ErrorCode() == nil) + assert.Assert(t, fetchedSubdoc2.ErrorDetails() == nil) + + // Verify other fields are unchanged + assert.Equal(t, *fetchedSubdoc2.Version(), srcVersion) + assert.Equal(t, *fetchedSubdoc2.State(), srcState) + assert.Equal(t, len(fetchedSubdoc2.Payload()), len(srcBytes)) +} diff --git a/db/sqlite/document.go b/db/sqlite/document.go index 1831df0..6711aa5 100644 --- a/db/sqlite/document.go +++ b/db/sqlite/document.go @@ -14,38 +14,39 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( "database/sql" "fmt" + "strings" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" - _ "github.com/mattn/go-sqlite3" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + _ "modernc.org/sqlite" ) func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.SubDocument, error) { c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - rows, err := c.Query("SELECT payload,state,updated_time,version,error_code,error_details FROM xpc_group_config WHERE cpe_mac=? AND group_id=?", cpeMac, groupId) + rows, err := c.Query("SELECT payload,state,updated_time,version,error_code,error_details,expiry FROM xpc_group_config WHERE cpe_mac=? AND group_id=?", cpeMac, groupId) if err != nil { return nil, common.NewError(err) } var ns1, ns2 sql.NullString var b1 []byte - var nt1 sql.NullTime + var nt1, nt2 sql.NullInt64 var ni1, ni2 sql.NullInt64 if !rows.Next() { return nil, sql.ErrNoRows } - err = rows.Scan(&b1, &ni1, &nt1, &ns1, &ni2, &ns2) + err = rows.Scan(&b1, &ni1, &nt1, &ns1, &ni2, &ns2, &nt2) defer rows.Close() if err != nil { return nil, common.NewError(err) @@ -53,7 +54,7 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su var s1, s2 *string var i1, i2 *int - var ts *int + var ts, expiry *int if ns1.Valid { s1 = &(ns1.String) } @@ -61,10 +62,13 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su s2 = &(ns2.String) } if nt1.Valid { - t1 := nt1.Time - tt := int(t1.UnixNano() / 1000000) + tt := int(nt1.Int64) ts = &tt } + if nt2.Valid { + tt := int(nt2.Int64) + expiry = &tt + } if ni1.Valid { ii := int(ni1.Int64) i1 = &ii @@ -74,7 +78,24 @@ func (c *SqliteClient) GetSubDocument(cpeMac string, groupId string) (*common.Su i2 = &ii } + // Check if payload contains a reference to a refsubdocument + if refId, ok := db.GetRefId(b1); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + if !c.IsDbNotFound(err) { + return nil, common.NewError(err) + } + // If refsubdocument not found, continue with the reference payload + } else { + // Replace payload with the actual payload from refsubdocument + b1 = refsubdocument.Payload() + } + } + doc := common.NewSubDocument(b1, s1, i1, ts, i2, s2) + if expiry != nil { + doc.SetExpiry(expiry) + } return doc, nil } @@ -101,6 +122,18 @@ func (c *SqliteClient) insertSubDocument(cpeMac string, groupId string, doc *com columns = append(columns, "updated_time") values = append(values, doc.UpdatedTime()) } + if doc.Expiry() != nil { + columns = append(columns, "expiry") + values = append(values, doc.Expiry()) + } + if doc.ErrorCode() != nil { + columns = append(columns, "error_code") + values = append(values, doc.ErrorCode()) + } + if doc.ErrorDetails() != nil { + columns = append(columns, "error_details") + values = append(values, doc.ErrorDetails()) + } qstr := fmt.Sprintf("INSERT INTO xpc_group_config(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) stmt, err := c.Prepare(qstr) if err != nil { @@ -137,6 +170,14 @@ func (c *SqliteClient) updateSubDocument(cpeMac string, groupId string, doc *com columns = append(columns, "updated_time") values = append(values, doc.UpdatedTime()) } + if doc.ErrorCode() != nil { + columns = append(columns, "error_code") + values = append(values, doc.ErrorCode()) + } + if doc.ErrorDetails() != nil { + columns = append(columns, "error_details") + values = append(values, doc.ErrorDetails()) + } values = append(values, cpeMac) values = append(values, groupId) qstr := fmt.Sprintf("UPDATE xpc_group_config SET %v WHERE cpe_mac=? AND group_id=?", db.GetSetColumnsStr(columns)) @@ -219,6 +260,32 @@ func (c *SqliteClient) DeleteSubDocument(cpeMac string, groupId string) error { return nil } +func (c *SqliteClient) DeleteSubDocumentColumns(cpeMac string, groupId string, columns ...string) error { + if len(columns) == 0 { + return nil + } + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // Build UPDATE statement to set columns to NULL + updateExprs := make([]string, len(columns)) + for i, col := range columns { + updateExprs[i] = col + "=NULL" + } + qstr := fmt.Sprintf("UPDATE xpc_group_config SET %v WHERE cpe_mac=? AND group_id=?", strings.Join(updateExprs, ",")) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(cpeMac, groupId) + if err != nil { + return common.NewError(err) + } + return nil +} + func (c *SqliteClient) DeleteDocument(cpeMac string) error { c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() @@ -250,7 +317,7 @@ func (c *SqliteClient) GetDocument(cpeMac string, xargs ...interface{}) (*common for rows.Next() { var ns0, ns1, ns2 sql.NullString var b1 []byte - var nt1 sql.NullTime + var nt1 sql.NullInt64 var ni1, ni2 sql.NullInt64 err = rows.Scan(&ns0, &b1, &ni1, &nt1, &ns1, &ni2, &ns2) @@ -273,8 +340,7 @@ func (c *SqliteClient) GetDocument(cpeMac string, xargs ...interface{}) (*common s2 = &(ns2.String) } if nt1.Valid { - t1 := nt1.Time - tt := int(t1.UnixNano() / 1000000) + tt := int(nt1.Int64) ts = &tt } if ni1.Valid { diff --git a/db/sqlite/document_test.go b/db/sqlite/document_test.go index a633ee3..78d99ed 100644 --- a/db/sqlite/document_test.go +++ b/db/sqlite/document_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -48,8 +48,9 @@ func TestSubDocumentDb(t *testing.T) { // read a SubDocument from db and verify identical targetSubDocument, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = sourceDoc.Equals(targetSubDocument) + ok, err := sourceDoc.Equals(targetSubDocument) assert.NilError(t, err) + assert.Assert(t, ok) // ==== update an existing doc with the same cpeMac and groupId ==== srcVersion2 := "red white blue" @@ -61,8 +62,9 @@ func TestSubDocumentDb(t *testing.T) { assert.NilError(t, err) expectedDoc := common.NewSubDocument(srcBytes, &srcVersion2, &srcState, &srcUpdatedTime, nil, nil) - err = targetSubDocument.Equals(expectedDoc) + ok, err = targetSubDocument.Equals(expectedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // ==== delete a doc ==== err = tdbclient.DeleteSubDocument(cpeMac, groupId) @@ -106,10 +108,12 @@ func TestDbReadDocument(t *testing.T) { assert.NilError(t, err) assert.Equal(t, Document.Length(), 2) - err = pdoc.Equals(Document.SubDocument("privatessid")) + ok, err := pdoc.Equals(Document.SubDocument("privatessid")) assert.NilError(t, err) - err = hdoc.Equals(Document.SubDocument("homessid")) + assert.Assert(t, ok) + ok, err = hdoc.Equals(Document.SubDocument("homessid")) assert.NilError(t, err) + assert.Assert(t, ok) // ==== delete all SubDocuments ==== err = tdbclient.DeleteDocument(cpeMac) @@ -119,3 +123,107 @@ func TestDbReadDocument(t *testing.T) { _, err = tdbclient.GetDocument(cpeMac) assert.Assert(t, tdbclient.IsDbNotFound(err)) } + +func TestGetSubDocumentWithReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "lan" + refId := util.GetMurmur3Hash([]byte(cpeMac + subdocId)) + + // Step 1: Create a reference subdocument with actual payload + actualPayload := common.RandomBytes(100, 200) + actualVersion := util.GetMurmur3Hash(actualPayload) + refSubdoc := common.NewRefSubDocument(actualPayload, &actualVersion) + + err := tdbclient.SetRefSubDocument(refId, refSubdoc) + assert.NilError(t, err) + + // Step 2: Create a subdocument with reference payload (4 zero bytes + refId) + referencePayload := append(make([]byte, 4), []byte(refId)...) + refVersion := util.GetMurmur3Hash(referencePayload) + refState := common.InDeployment + refUpdatedTime := int(time.Now().UnixMilli()) + + subdocWithRef := common.NewSubDocument(referencePayload, &refVersion, &refState, &refUpdatedTime, nil, nil) + fields := log.Fields{} + err = tdbclient.SetSubDocument(cpeMac, subdocId, subdocWithRef, fields) + assert.NilError(t, err) + + // Step 3: Call GetSubDocument and verify it returns the actual payload, not the reference + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is the actual payload from refsubdocument, not the reference + assert.DeepEqual(t, fetchedSubdoc.Payload(), actualPayload) + + // Verify other fields remain unchanged + assert.Equal(t, *fetchedSubdoc.Version(), refVersion) + assert.Equal(t, *fetchedSubdoc.State(), refState) + assert.Equal(t, *fetchedSubdoc.UpdatedTime(), refUpdatedTime) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) +} + +func TestGetSubDocumentWithMissingReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "wan" + refId := util.GetMurmur3Hash([]byte(cpeMac + subdocId + "nonexistent")) + + // Create a subdocument with reference payload pointing to non-existent refsubdocument + referencePayload := append(make([]byte, 4), []byte(refId)...) + refVersion := util.GetMurmur3Hash(referencePayload) + refState := common.InDeployment + refUpdatedTime := int(time.Now().UnixMilli()) + + subdocWithRef := common.NewSubDocument(referencePayload, &refVersion, &refState, &refUpdatedTime, nil, nil) + fields := log.Fields{} + err := tdbclient.SetSubDocument(cpeMac, subdocId, subdocWithRef, fields) + assert.NilError(t, err) + + // Call GetSubDocument - should return the reference payload since refsubdocument doesn't exist + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is the reference payload (since refsubdocument was not found) + assert.DeepEqual(t, fetchedSubdoc.Payload(), referencePayload) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) +} + +func TestGetSubDocumentWithoutReference(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + subdocId := "mesh" + + // Create a regular subdocument without any reference + regularPayload := common.RandomBytes(100, 200) + regularVersion := util.GetMurmur3Hash(regularPayload) + regularState := common.Deployed + regularUpdatedTime := int(time.Now().UnixMilli()) + + subdoc := common.NewSubDocument(regularPayload, ®ularVersion, ®ularState, ®ularUpdatedTime, nil, nil) + fields := log.Fields{} + err := tdbclient.SetSubDocument(cpeMac, subdocId, subdoc, fields) + assert.NilError(t, err) + + // Call GetSubDocument - should return the regular payload unchanged + fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Assert(t, fetchedSubdoc != nil) + + // Verify the payload is unchanged + assert.DeepEqual(t, fetchedSubdoc.Payload(), regularPayload) + assert.Equal(t, *fetchedSubdoc.Version(), regularVersion) + assert.Equal(t, *fetchedSubdoc.State(), regularState) + assert.Equal(t, *fetchedSubdoc.UpdatedTime(), regularUpdatedTime) + + // Cleanup + err = tdbclient.DeleteSubDocument(cpeMac, subdocId) + assert.NilError(t, err) +} diff --git a/db/sqlite/refsubdocument.go b/db/sqlite/refsubdocument.go new file mode 100644 index 0000000..bcea8c4 --- /dev/null +++ b/db/sqlite/refsubdocument.go @@ -0,0 +1,153 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package sqlite + +import ( + "database/sql" + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + _ "modernc.org/sqlite" +) + +func (c *SqliteClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + rows, err := c.Query("SELECT payload,version FROM reference_document WHERE ref_id=?", refId) + if err != nil { + return nil, common.NewError(err) + } + + var ns1 sql.NullString + var b1 []byte + + if !rows.Next() { + return nil, sql.ErrNoRows + } + err = rows.Scan(&b1, &ns1) + defer rows.Close() + if err != nil { + return nil, common.NewError(err) + } + + var s1 *string + if ns1.Valid { + s1 = &(ns1.String) + } + + refsubdoc := common.NewRefSubDocument(b1, s1) + return refsubdoc, nil +} + +func (c *SqliteClient) insertRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + qstr := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) updateRefSubDocument(refId string, doc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{} + values := []interface{}{} + if doc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, doc.Payload()) + } + if doc.Version() != nil { + columns = append(columns, "version") + values = append(values, doc.Version()) + } + values = append(values, refId) + qstr := fmt.Sprintf("UPDATE reference_document SET %v WHERE cpe_mac=? AND ref_id=?", db.GetSetColumnsStr(columns)) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + _, err := c.GetRefSubDocument(refId) + if err != nil { + if c.IsDbNotFound(err) { + err1 := c.insertRefSubDocument(refId, refsubdoc) + if err1 != nil { + return common.NewError(err1) + } + } else { + // unexpected error + return common.NewError(err) + } + } else { + // normal dbNotFound should not happen + err = c.updateRefSubDocument(refId, refsubdoc) + if err != nil { + return common.NewError(err) + } + } + + return nil +} + +func (c *SqliteClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt, err := c.Prepare("DELETE FROM reference_document WHERE ref_id=?") + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(refId) + if err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/sqlite/refsubdocument_test.go b/db/sqlite/refsubdocument_test.go new file mode 100644 index 0000000..8df729c --- /dev/null +++ b/db/sqlite/refsubdocument_test.go @@ -0,0 +1,56 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package sqlite + +import ( + "testing" + + "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + srcBytes := common.RandomBytes(16, 116) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/sqlite/root_document.go b/db/sqlite/root_document.go index d67fbe6..13a13eb 100644 --- a/db/sqlite/root_document.go +++ b/db/sqlite/root_document.go @@ -14,21 +14,21 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( "database/sql" - "github.com/rdkcentral/webconfig/common" "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" ) func (c *SqliteClient) GetRootDocument(cpeMac string) (*common.RootDocument, error) { c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - rows, err := c.Query("SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version FROM root_document WHERE cpe_mac=?", cpeMac) + rows, err := c.Query("SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,product_class,account_type FROM root_document WHERE cpe_mac=?", cpeMac) if err != nil { return nil, common.NewError(err) } @@ -38,8 +38,8 @@ func (c *SqliteClient) GetRootDocument(cpeMac string) (*common.RootDocument, err } var ni sql.NullInt64 - var ns1, ns2, ns3, ns4, ns5 sql.NullString - err = rows.Scan(&ni, &ns1, &ns2, &ns3, &ns4, &ns5) + var ns1, ns2, ns3, ns4, ns5, ns6, ns7 sql.NullString + err = rows.Scan(&ni, &ns1, &ns2, &ns3, &ns4, &ns5, &ns6, &ns7) defer rows.Close() if err != nil { return nil, common.NewError(err) @@ -67,7 +67,15 @@ func (c *SqliteClient) GetRootDocument(cpeMac string) (*common.RootDocument, err version = ns5.String } - return common.NewRootDocument(bitmap, firmware_version, model_name, partner_id, schema_version, version, ""), nil + var product_class, account_type string + if ns6.Valid { + product_class = ns6.String + } + if ns7.Valid { + account_type = ns7.String + } + + return common.NewRootDocument(bitmap, firmware_version, model_name, partner_id, schema_version, version, "", product_class, account_type), nil } func (c *SqliteClient) insertRootDocumentVersion(cpeMac, version string) error { @@ -206,12 +214,12 @@ func (c *SqliteClient) insertRootDocument(cpeMac string, rd *common.RootDocument c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - stmt, err := c.Prepare("INSERT INTO root_document(cpe_mac,bitmap,firmware_version,model_name,partner_id,schema_version,version) VALUES(?,?,?,?,?,?,?)") + stmt, err := c.Prepare("INSERT INTO root_document(cpe_mac,bitmap,firmware_version,model_name,partner_id,schema_version,version,product_class,account_type) VALUES(?,?,?,?,?,?,?,?,?)") if err != nil { return common.NewError(err) } - _, err = stmt.Exec(cpeMac, rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version) + _, err = stmt.Exec(cpeMac, rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, rd.ProductClass, rd.AccountType) if err != nil { return common.NewError(err) } @@ -222,11 +230,11 @@ func (c *SqliteClient) updateRootDocument(cpeMac string, rd *common.RootDocument c.concurrentQueries <- true defer func() { <-c.concurrentQueries }() - stmt, err := c.Prepare("UPDATE root_document SET bitmap=?,firmware_version=?,model_name=?,partner_id=?,schema_version=?,version=? WHERE cpe_mac=?") + stmt, err := c.Prepare("UPDATE root_document SET bitmap=?,firmware_version=?,model_name=?,partner_id=?,schema_version=?,version=?,product_class=?,account_type=? WHERE cpe_mac=?") if err != nil { return common.NewError(err) } - _, err = stmt.Exec(rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, cpeMac) + _, err = stmt.Exec(rd.Bitmap, rd.FirmwareVersion, rd.ModelName, rd.PartnerId, rd.SchemaVersion, rd.Version, rd.ProductClass, rd.AccountType, cpeMac) if err != nil { return common.NewError(err) } diff --git a/db/sqlite/root_document_test.go b/db/sqlite/root_document_test.go index fdc7933..dec2f00 100644 --- a/db/sqlite/root_document_test.go +++ b/db/sqlite/root_document_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -71,7 +71,7 @@ func TestRootDocumentDb(t *testing.T) { // set by a RootDocument version4 := "indigo violet" bitmap4 := 67 - rdoc4 := common.NewRootDocument(bitmap4, "", "", "", "", version4, "") + rdoc4 := common.NewRootDocument(bitmap4, "", "", "", "", version4, "", "", "") err = tdbclient.SetRootDocument(cpeMac, rdoc4) assert.NilError(t, err) fetched, err := tdbclient.GetRootDocument(cpeMac) @@ -147,7 +147,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName1 := "TG4482" partnerId1 := "" firmwareVersion1 := "TG4482PC2_4.12p7s3_PROD_sey" - srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "") + srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", "", "") err = tdbclient.SetRootDocument(cpeMac, srcRootdoc1) assert.NilError(t, err) @@ -163,7 +163,7 @@ func TestRootDocumentUpdate(t *testing.T) { modelName2 := "TG4482" partnerId2 := "cox" firmwareVersion2 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc2 := common.NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, "") + rootdoc2 := common.NewRootDocument(bitmap2, firmwareVersion2, modelName2, partnerId2, schemaVersion2, version2, "", "", "") err = tdbclient.SetRootDocument(cpeMac, rootdoc2) assert.NilError(t, err) @@ -175,9 +175,51 @@ func TestRootDocumentUpdate(t *testing.T) { modelName3 := "TG4482" partnerId3 := "cox" firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" - rootdoc3 := common.NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, "") + rootdoc3 := common.NewRootDocument(bitmap3, firmwareVersion3, modelName3, partnerId3, schemaVersion3, version3, "", "", "") tgtRootdoc3, err := tdbclient.GetRootDocument(cpeMac) assert.NilError(t, err) assert.DeepEqual(t, tgtRootdoc3, rootdoc3) } + +func TestRootDocumentProductClassAccountType(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + // verify starting empty + _, err := tdbclient.GetRootDocument(cpeMac) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // ==== step 1: set rootdoc with product_class and account_type ==== + bitmap1 := 100 + version1 := "v1" + schemaVersion1 := "33554433-1.3" + modelName1 := "TG3482G" + partnerId1 := "comcast" + firmwareVersion1 := "TG3482G_4.10p7s1_PROD_sey" + productClass1 := "rg" + accountType1 := "residential" + srcRootdoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", productClass1, accountType1) + + err = tdbclient.SetRootDocument(cpeMac, srcRootdoc1) + assert.NilError(t, err) + + // read from db and verify product_class and account_type are stored + tgtRootdoc1, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass1, tgtRootdoc1.ProductClass) + assert.Equal(t, accountType1, tgtRootdoc1.AccountType) + + // ==== step 2: update with new product_class and account_type ==== + productClass2 := "xb" + accountType2 := "business" + rootdoc2 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partnerId1, schemaVersion1, version1, "", productClass2, accountType2) + + err = tdbclient.SetRootDocument(cpeMac, rootdoc2) + assert.NilError(t, err) + + // verify the updated values are stored + tgtRootdoc2, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass2, tgtRootdoc2.ProductClass) + assert.Equal(t, accountType2, tgtRootdoc2.AccountType) +} diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 8ee8d6e..e30f7ac 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -14,11 +14,14 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( + "fmt" "regexp" + "strings" + "unicode" ) const ( @@ -39,17 +42,27 @@ var ( state int, error_code int, error_details text, + expiry timestamp, PRIMARY KEY (cpe_mac, group_id) )`, `CREATE TABLE IF NOT EXISTS root_document ( cpe_mac text PRIMARY KEY, bitmap bigint, + account_type text, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + product_class text, + query_params text, route text, schema_version, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } ) @@ -64,3 +77,68 @@ func init() { } } } + +// parseCreateTable extracts the table name and a map of column-name→type from a +// "CREATE TABLE IF NOT EXISTS" DDL string. It is used by SyncSchema to diff +// expected columns against what PRAGMA table_info reports. +func parseCreateTable(stmt string) (string, map[string]string, error) { + reTableName := regexp.MustCompile(`(?i)CREATE TABLE IF NOT EXISTS (\w+)`) + m := reTableName.FindStringSubmatch(stmt) + if len(m) < 2 { + return "", nil, fmt.Errorf("parseCreateTable: cannot find table name in DDL") + } + tableName := m[1] + + colDefs := make(map[string]string) + for _, rawLine := range strings.Split(stmt, "\n") { + line := strings.TrimSpace(rawLine) + line = strings.TrimRight(line, ",") + line = strings.TrimSpace(line) + if line == "" { + continue + } + upper := strings.ToUpper(line) + // skip the CREATE TABLE header, lone parens, and table-level constraints + if strings.HasPrefix(upper, "CREATE") || + line == "(" || line == ")" || + strings.HasPrefix(upper, "PRIMARY") || + strings.HasPrefix(upper, "UNIQUE") || + strings.HasPrefix(upper, "FOREIGN") || + strings.HasPrefix(upper, "CHECK") || + strings.HasPrefix(upper, "CONSTRAINT") { + continue + } + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + colName := parts[0] + if !isSQLiteIdentifier(colName) { + continue + } + var colType string + if len(parts) > 1 { + t := parts[1] + u := strings.ToUpper(t) + if u != "PRIMARY" && u != "NOT" && u != "UNIQUE" && u != "REFERENCES" { + colType = t + } + } + colDefs[colName] = colType + } + return tableName, colDefs, nil +} + +// isSQLiteIdentifier reports whether s is a valid plain SQL identifier +// (ASCII letters, digits, and underscores only). +func isSQLiteIdentifier(s string) bool { + if len(s) == 0 { + return false + } + for _, ch := range s { + if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' { + return false + } + } + return true +} diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index d623d07..01b1e1e 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -22,10 +22,10 @@ import ( "errors" "fmt" + "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" - "github.com/go-akka/configuration" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" ) const ( @@ -43,8 +43,12 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool + supplementaryPrecookEnabled bool + supplementaryPrecookStateTTLDays int } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -58,15 +62,24 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") - db, err := sql.Open("sqlite3", dbfile) + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") + supplementaryPrecookEnabled := conf.GetBoolean("webconfig.supplementary_precook_enabled") + supplementaryPrecookStateTTLDays := int(conf.GetInt32("webconfig.supplementary_precook_state_ttl_days", 7)) + + db, err := sql.Open("sqlite", dbfile) if err != nil { return nil, common.NewError(err) } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, + supplementaryPrecookEnabled: supplementaryPrecookEnabled, + supplementaryPrecookStateTTLDays: supplementaryPrecookStateTTLDays, }, nil } @@ -136,6 +149,22 @@ func (c *SqliteClient) SetBlockedSubdocIds(x []string) { c.blockedSubdocIds = x } +func (c *SqliteClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + +func (c *SqliteClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *SqliteClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil @@ -150,6 +179,10 @@ func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClie if err = tdbclient.SetUp(); err != nil { return nil, common.NewError(err) } + // add any new columns that are missing from an existing DB file + if err = tdbclient.SyncSchema(); err != nil { + return nil, common.NewError(err) + } if err = tdbclient.TearDown(); err != nil { return nil, common.NewError(err) } @@ -159,3 +192,77 @@ func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClie return tdbclient, nil } + +func (c *SqliteClient) SupplementaryPrecookEnabled() bool { + return c.supplementaryPrecookEnabled +} + +func (c *SqliteClient) SetSupplementaryPrecookEnabled(enabled bool) { + c.supplementaryPrecookEnabled = enabled +} + +func (c *SqliteClient) SupplementaryPrecookStateTTLDays() int { + return c.supplementaryPrecookStateTTLDays +} + +func (c *SqliteClient) SetSupplementaryPrecookStateTTLDays(days int) { + c.supplementaryPrecookStateTTLDays = days +} + +// SyncSchema adds any columns that are defined in SqliteCreateTableStatements but +// missing from the existing tables. It is safe to call on both new and existing DB +// files, and is idempotent — it is a no-op when the schema is already current. +func (c *SqliteClient) SyncSchema() error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + for _, stmt := range SqliteCreateTableStatements { + tableName, colDefs, err := parseCreateTable(stmt) + if err != nil { + return common.NewError(err) + } + + existing, err := c.pragmaTableInfo(tableName) + if err != nil { + return common.NewError(err) + } + + for colName, colType := range colDefs { + if _, ok := existing[colName]; ok { + continue + } + var alterSQL string + if colType == "" { + alterSQL = fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s", tableName, colName) + } else { + alterSQL = fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", tableName, colName, colType) + } + if _, err := c.Exec(alterSQL); err != nil { + return common.NewError(fmt.Errorf("SyncSchema %s.%s: %w", tableName, colName, err)) + } + } + } + return nil +} + +// pragmaTableInfo returns a map of column name → type for an existing table. +// Returns an empty map (not an error) if the table does not exist. +func (c *SqliteClient) pragmaTableInfo(tableName string) (map[string]string, error) { + rows, err := c.Query(fmt.Sprintf("PRAGMA table_info(%s)", tableName)) + if err != nil { + return nil, err + } + defer rows.Close() + + cols := make(map[string]string) + for rows.Next() { + var cid, notNull, pk int + var name, colType string + var dfltValue sql.NullString + if err := rows.Scan(&cid, &name, &colType, ¬Null, &dfltValue, &pk); err != nil { + return nil, err + } + cols[name] = colType + } + return cols, rows.Err() +} diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go new file mode 100644 index 0000000..f915f8e --- /dev/null +++ b/db/sqlite/sqlite_client_test.go @@ -0,0 +1,51 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package sqlite + +import ( + "testing" + + "github.com/rdkcentral/webconfig/common" + "gotest.tools/assert" +) + +func TestSqliteClient(t *testing.T) { + configFile := "../../config/sample_webconfig.conf" + sc, err := common.GetTestServerConfig(configFile) + + assert.NilError(t, err) + dbc, err := GetTestSqliteClient(sc.Config, true) + assert.NilError(t, err) + assert.Assert(t, dbc != nil) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) +} diff --git a/db/sqlite/state_metrics_test.go b/db/sqlite/state_metrics_test.go index d8bcb02..bb2cb00 100644 --- a/db/sqlite/state_metrics_test.go +++ b/db/sqlite/state_metrics_test.go @@ -65,8 +65,9 @@ func TestStateMetrics(t *testing.T) { // read a SubDocument from db and verify identical doc1, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = sourceDoc.Equals(doc1) + ok, err := sourceDoc.Equals(doc1) assert.NilError(t, err) + assert.Assert(t, ok) // ==== update an doc with the same cpeMac and a changed state ==== state2 := common.InDeployment diff --git a/db/sqlite/state_update_test.go b/db/sqlite/state_update_test.go new file mode 100644 index 0000000..96a2ac6 --- /dev/null +++ b/db/sqlite/state_update_test.go @@ -0,0 +1,79 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package sqlite + +import ( + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +// TestUpdateSubDocumentResetsErrorFields verifies that transitioning a subdocument +// to InDeployment (state 3) via UpdateSubDocument clears any stale error_code and +// error_details left from a prior Failure (state 4). +func TestUpdateSubDocumentResetsErrorFields(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + groupId := "privatessid" + + // step 1: seed a root document so GetRootDocumentLabels succeeds + rootdoc := &common.RootDocument{} + err := tdbclient.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + // step 2: write a subdoc in Failure state with non-zero error fields + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Failure + errCode := 204 + errDetails := "failed_retrying:Error unsupported namespace" + failureSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errCode, &errDetails) + fields := log.Fields{} + err = tdbclient.SetSubDocument(cpeMac, groupId, failureSubdoc, fields) + assert.NilError(t, err) + + // verify failure state and error fields persisted + fetched, err := tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *fetched.State(), common.Failure) + assert.Equal(t, *fetched.ErrorCode(), 204) + assert.Equal(t, *fetched.ErrorDetails(), "failed_retrying:Error unsupported namespace") + + // step 3: call UpdateSubDocument (simulating upstream fetch, 2→3 transition) + // newSubdoc represents fresh config from upstream — no state/error fields set + newBytes := common.RandomBytes(100, 150) + newVersion := util.GetMurmur3Hash(newBytes) + newSubdoc := common.NewSubDocument(newBytes, &newVersion, nil, nil, nil, nil) + + // empty versionMap so UpdateSubDocument does not skip via early-return path + versionMap := make(map[string]string) + err = db.UpdateSubDocument(tdbclient, cpeMac, groupId, newSubdoc, failureSubdoc, versionMap, fields) + assert.NilError(t, err) + + // step 4: verify state advanced to InDeployment and error fields are reset + fetched, err = tdbclient.GetSubDocument(cpeMac, groupId) + assert.NilError(t, err) + assert.Equal(t, *fetched.State(), common.InDeployment) + assert.Equal(t, *fetched.ErrorCode(), 0) + assert.Equal(t, *fetched.ErrorDetails(), "") +} diff --git a/dbinit.cql b/dbinit.cql index be8d88b..62ec793 100644 --- a/dbinit.cql +++ b/dbinit.cql @@ -20,7 +20,7 @@ CREATE KEYSPACE IF NOT EXISTS webconfig WITH replication = {'class': 'SimpleStra USE webconfig; -CREATE TABLE root_document ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, model_name text, partner_id text, query_params text, schema_version text, version text); +CREATE TABLE root_document (cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, locked_till timestamp, model_name text, partner_id text, route text, query_params text, schema_version text, version text); -CREATE TABLE xpc_group_config ( cpe_mac text, group_id text, error_code int, error_details text, expiry timestamp, params text, payload blob, state int, updated_time timestamp, version text, PRIMARY KEY (cpe_mac, group_id)) WITH CLUSTERING ORDER BY (group_id ASC); +CREATE TABLE xpc_group_config (cpe_mac text, group_id text, error_code int, error_details text, expiry timestamp, params text, payload blob, state int, updated_time timestamp, version text, PRIMARY KEY (cpe_mac, group_id)) WITH CLUSTERING ORDER BY (group_id ASC); diff --git a/go.mod b/go.mod index 62ba922..abe8783 100644 --- a/go.mod +++ b/go.mod @@ -1,62 +1,82 @@ module github.com/rdkcentral/webconfig -go 1.19 +go 1.25.0 require ( + github.com/IBM/sarama v1.48.0 github.com/MicahParks/keyfunc/v2 v2.1.0 - github.com/Shopify/sarama v1.38.1 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 - github.com/gocql/gocql v1.2.1 - github.com/golang-jwt/jwt/v5 v5.0.0 - github.com/google/uuid v1.3.0 - github.com/gorilla/mux v1.8.0 - github.com/mattn/go-sqlite3 v1.14.15 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 - github.com/sirupsen/logrus v1.9.0 - github.com/twmb/murmur3 v1.1.6 + github.com/gocql/gocql v1.7.0 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/google/uuid v1.6.0 + github.com/gorilla/mux v1.8.1 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 + github.com/sirupsen/logrus v1.9.4 + github.com/twmb/murmur3 v1.1.8 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.uber.org/automaxprocs v1.5.1 - go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.1.0 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 + go.uber.org/automaxprocs v1.6.0 + go.uber.org/ratelimit v0.3.1 + golang.org/x/sync v0.20.0 gotest.tools v2.2.0+incompatible + modernc.org/sqlite v1.50.0 ) require ( - github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.3.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.15.14 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect - golang.org/x/net v0.5.0 // indirect - golang.org/x/sys v0.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 // indirect + google.golang.org/grpc v1.81.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + modernc.org/libc v1.72.2 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index ed18197..1adcd3b 100644 --- a/go.sum +++ b/go.sum @@ -1,176 +1,68 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/IBM/sarama v1.48.0 h1:9LJS0VNeg/boXxT/GLAMDKX6uSQ1mr/5F/j4v9gSeBQ= +github.com/IBM/sarama v1.48.0/go.mod h1:UhvwPF8zilmLOSd6O+ENzdycCJYwMww1U9DJOZpoCro= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= -github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= -github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= -github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= -github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM= -github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= +github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gocql/gocql v1.2.1 h1:G/STxUzD6pGvRHzG0Fi7S04SXejMKBbRZb7pwre1edU= -github.com/gocql/gocql v1.2.1/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= +github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -179,99 +71,63 @@ github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVET github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= -github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= -github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= +github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= -github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= @@ -279,314 +135,137 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 h1:mS47AX77OtFfKG4vtp+84kuGSFZHTyxtXIN269vChY0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0/go.mod h1:PJnsC41lAGncJlPUniSwM81gc80GkgWJWr3cu2nKEtU= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= -go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= -go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348 h1:U8orV30l6KpDsi9dxU0CoJZGbjS8EEpw+6ba+XwGPQA= +google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348/go.mod h1:Yzdzr5OOZFgSsEV2D/Xi9NL3bszpXFAg0hFJiRohcD8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 h1:pfIbyB44sWzHiCpRqIen67ZQnVXSfIxWrqUMk1qwODE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.0 h1:W3G9N3KQf3BU+YuCtGKJk0CmxQNbAISICD/9AORxLIw= +google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +modernc.org/cc/v4 v4.28.1 h1:XpLbkYVQ24E8tX5u8+yWGvaxerxkR/S4zqxI8ZoSBuc= +modernc.org/cc/v4 v4.28.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.72.2 h1:HwRjrHwX7hZIFCfRyw6otVlY+BoZEHFQmHQa5B0BzDE= +modernc.org/libc v1.72.2/go.mod h1:43RZAMuEX483KwP1bW+3lTFm3dzwFpl6R8HMEutqy/w= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM= +modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/http/bitmap_filter_test.go b/http/bitmap_filter_test.go new file mode 100644 index 0000000..f8705dc --- /dev/null +++ b/http/bitmap_filter_test.go @@ -0,0 +1,195 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "strconv" + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "gotest.tools/assert" +) + +func TestFilterOutputByBitmap(t *testing.T) { + tsc1 := sc.Copy("webconfig.filter_output_by_bitmap_enabled=true") + server := NewWebconfigServer(tsc1, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + now := time.Now() + + // ==== step 1 use epochNow as version and set a future expiry ==== + // post + subdocId := "remotedebugger" + remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + remotedebuggerBytes := common.RandomBytes(100, 150) + req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + + // prepare the version header + reqHeaderVersion := strconv.Itoa(int(now.Unix())) + req.Header.Set(common.HeaderSubdocumentVersion, reqHeaderVersion) + + // prepare a future expiry header + futureT := now.AddDate(0, 0, 2) + reqHeaderExpiry := strconv.Itoa(int(futureT.UnixNano() / 1000000)) + req.Header.Set(common.HeaderSubdocumentExpiry, reqHeaderExpiry) + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", remotedebuggerUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + + resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) + assert.Equal(t, reqHeaderVersion, resHeaderVersion) + resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) + assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) + + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, remotedebuggerBytes) + + // check the root doc version + rdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, len(rdoc.Version) > 0) + + // ==== step 2 get document ==== + supportedDocs1 := "16777217" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} + +func TestBitmapFilterExemptSubdocIds(t *testing.T) { + tsc1 := sc.Copy( + "webconfig.filter_output_by_bitmap_enabled=true", + `webconfig.bitmap_filter_exempt_subdoc_ids=["remotedebugger"]`, + ) + server := NewWebconfigServer(tsc1, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + now := time.Now() + + // ==== step 1 use epochNow as version and set a future expiry ==== + // post + subdocId := "remotedebugger" + remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + remotedebuggerBytes := common.RandomBytes(100, 150) + req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + + // prepare the version header + reqHeaderVersion := strconv.Itoa(int(now.Unix())) + req.Header.Set(common.HeaderSubdocumentVersion, reqHeaderVersion) + + // prepare a future expiry header + futureT := now.AddDate(0, 0, 2) + reqHeaderExpiry := strconv.Itoa(int(futureT.UnixNano() / 1000000)) + req.Header.Set(common.HeaderSubdocumentExpiry, reqHeaderExpiry) + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", remotedebuggerUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + + resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) + assert.Equal(t, reqHeaderVersion, resHeaderVersion) + resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) + assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) + + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, remotedebuggerBytes) + + // check the root doc version + rdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, len(rdoc.Version) > 0) + + // ==== step 2 get document ==== + supportedDocs1 := "16777217" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + + mpart, ok := mpartMap["remotedebugger"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, remotedebuggerBytes) +} diff --git a/http/document_handler.go b/http/document_handler.go index 2f38afd..e833c8a 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,10 +24,10 @@ import ( "strings" "time" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) // TODO @@ -84,7 +84,7 @@ func (s *WebconfigServer) GetSubDocumentHandler(w http.ResponseWriter, r *http.R return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) writeStateHeaders(w, subdoc) w.WriteHeader(http.StatusOK) w.Write(subdoc.Payload()) @@ -135,7 +135,9 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. } updatedTime := int(time.Now().UnixNano() / 1000000) - subdoc := common.NewSubDocument(bbytes, &version, statePtr, &updatedTime, nil, nil) + zeroErrorCode := 0 + emptyErrorDetails := "" + subdoc := common.NewSubDocument(bbytes, &version, statePtr, &updatedTime, &zeroErrorCode, &emptyErrorDetails) // handle expiry header expiryTmsStr := r.Header.Get(common.HeaderSubdocumentExpiry) @@ -160,6 +162,8 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. metricsAgent = "default" } + rootVersionMap := make(map[string]string) + var newRootVersion string for _, deviceId := range deviceIds { fields["src_caller"] = common.GetCaller() @@ -189,15 +193,22 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. } doc.SetSubDocument(subdocId, subdoc) - newRootVersion := db.HashRootVersion(doc.VersionMap()) + newRootVersion = db.HashRootVersion(doc.VersionMap()) err = s.SetRootDocumentVersion(deviceId, newRootVersion) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) return } + rootVersionMap[deviceId] = newRootVersion + } + d := make(util.Dict) + if len(rootVersionMap) == 1 { + d["root_version"] = newRootVersion + } else { + d["root_version"] = rootVersionMap } - WriteOkResponse(w, nil) + WriteByMarshal(w, http.StatusOK, d) } func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *http.Request) { @@ -236,6 +247,7 @@ func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *htt if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } @@ -292,6 +304,7 @@ func (s *WebconfigServer) DeleteDocumentHandler(w http.ResponseWriter, r *http.R if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 5b87fcf..0be8cf0 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -14,14 +14,14 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "bytes" "encoding/hex" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "strings" @@ -48,20 +48,20 @@ func TestSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -73,19 +73,19 @@ func TestSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -105,22 +105,22 @@ func TestDeleteDocumentHandler(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -135,21 +135,21 @@ func TestDeleteDocumentHandler(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -167,25 +167,25 @@ func TestDeleteDocumentHandler(t *testing.T) { req, err = http.NewRequest("DELETE", url, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -212,12 +212,12 @@ func TestPostWithDeviceId(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -226,10 +226,10 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -244,12 +244,12 @@ func TestPostWithDeviceId(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -257,10 +257,10 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -281,9 +281,9 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId := "gwrestore" gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - gwrestoreBytes := util.RandomBytes(100, 150) + gwrestoreBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header now := time.Now() @@ -292,19 +292,19 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) assert.Equal(t, reqHeaderVersion, resHeaderVersion) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, gwrestoreBytes) @@ -318,9 +318,9 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId = "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -333,13 +333,13 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -348,7 +348,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, remotedebuggerBytes) @@ -362,22 +362,22 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // post subdocId = "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -392,7 +392,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -419,14 +419,14 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { subdocVersions = append(subdocVersions, mpart.Version) // ==== step 5 get document again ==== - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,gwrestore,remotedebugger,lan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) ifnonematch := strings.Join(subdocVersions, ",") req.Header.Set(common.HeaderIfNoneMatch, ifnonematch) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -441,9 +441,9 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId := "gwrestore" gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - gwrestoreBytes := util.RandomBytes(100, 150) + gwrestoreBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare a version header now := time.Now() @@ -452,19 +452,19 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) assert.Equal(t, reqHeaderVersion, resHeaderVersion) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, gwrestoreBytes) @@ -478,9 +478,9 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId = "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -493,13 +493,13 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -508,7 +508,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, remotedebuggerBytes) @@ -522,22 +522,22 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // post subdocId = "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -552,7 +552,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -580,7 +580,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { ifnonematch := strings.Join(subdocVersions, ",") req.Header.Set(common.HeaderIfNoneMatch, ifnonematch) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -595,9 +595,9 @@ func TestBadHeaderExpiryHandler(t *testing.T) { // post subdocId := "remotedebugger" remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - remotedebuggerBytes := util.RandomBytes(100, 150) + remotedebuggerBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // manage version and expiry headers now := time.Now() @@ -609,7 +609,51 @@ func TestBadHeaderExpiryHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) } + +func TestPostSubDocumentResetsErrorFields(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + subdocId := "lan" + + // step 1: write a subdoc in Failure state with non-zero error fields + srcBytes := common.RandomBytes(100, 150) + srcVersion := util.GetMurmur3Hash(srcBytes) + srcUpdatedTime := int(time.Now().UnixNano() / 1000000) + srcState := common.Failure + errCode := 204 + errDetails := "failed_retrying:Error unsupported namespace" + failureSubdoc := common.NewSubDocument(srcBytes, &srcVersion, &srcState, &srcUpdatedTime, &errCode, &errDetails) + err := server.SetSubDocument(cpeMac, subdocId, failureSubdoc) + assert.NilError(t, err) + + // verify failure state persisted correctly + fetched, err := server.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Equal(t, *fetched.State(), common.Failure) + assert.Equal(t, *fetched.ErrorCode(), 204) + assert.Equal(t, *fetched.ErrorDetails(), "failed_retrying:Error unsupported namespace") + + // step 2: POST new config via HTTP handler (4 → 2 transition) + newBytes := common.RandomBytes(100, 150) + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(newBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // step 3: verify state is PendingDownload and error fields are reset to zero + fetched, err = server.GetSubDocument(cpeMac, subdocId) + assert.NilError(t, err) + assert.Equal(t, *fetched.State(), common.PendingDownload) + assert.Equal(t, *fetched.ErrorCode(), 0) + assert.Equal(t, *fetched.ErrorDetails(), "") +} diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go new file mode 100644 index 0000000..7f67da2 --- /dev/null +++ b/http/factory_reset_upstream_test.go @@ -0,0 +1,573 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestFactoryResetWithoutData(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetUpstreamEnabled(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup upstream mock server ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + if k == common.HeaderContentLength { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + w.WriteHeader(http.StatusNotFound) + w.Write(nil) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== no data ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + rootDocument, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, rootDocument.Bitmap, 32479) +} + +func TestFactoryResetWithoutUpstream(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := common.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(false) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // verify Document is now empty + fields := make(log.Fields) + _, err = server.GetDocument(cpeMac, fields) + assert.Assert(t, server.IsDbNotFound(err)) +} + +func TestFactoryResetWithUpstream(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := common.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + // lanVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== setup mock upstream server ==== + fields := make(log.Fields) + mockDoc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + + mockRootDoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + mockDoc.SetRootDocument(mockRootDoc) + + mockDoc.DeleteSubDocument("wan") + assert.NilError(t, err) + + mockBytes, err := mockDoc.Bytes() + assert.NilError(t, err) + + db.RefreshRootDocumentVersion(mockDoc) + refreshedRootVersion := mockDoc.RootVersion() + + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.Header().Set(common.HeaderContentLength, strconv.Itoa(len(mockBytes))) + ifNoneMatch := refreshedRootVersion + w.Header().Set(common.HeaderEtag, ifNoneMatch) + w.WriteHeader(http.StatusOK) + w.Write(mockBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(true) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, !ok) + + // verify Document is now empty + doc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + assert.Equal(t, doc.Length(), 1) +} + +func TestFactoryResetUpstreamAddData(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetUpstreamEnabled(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup upstream mock server ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for k := range r.Header { + if k == common.HeaderContentLength { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + + // add a subdoc from upsream + mparts := []common.Multipart{ + { + Bytes: common.RandomBytes(100, 150), + Version: strconv.Itoa(int(time.Now().Unix())), + Name: "network", + State: common.PendingDownload, + }, + } + respBytes, err := common.WriteMultipartBytes(mparts) + assert.NilError(t, err) + w.WriteHeader(http.StatusOK) + w.Write(respBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== no data ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + rootDocument, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, rootDocument.Bitmap, 32479) +} + +func TestFactoryResetWithUpstreamThenFilteringByBitmap(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := common.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== group 3 webui ==== + subdocId = "webui" + webuiBytes := common.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(webuiBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, webuiBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderProductClass, "XB8") + req.Header.Set(common.HeaderFirmwareVersion, "CGM4981COM_8.0p4s1_PROD_sey") + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderModelName, "CGM4981COM") + req.Header.Set(common.HeaderSupportedDocs, "16777439,33554435,50331649,67108865,83886081,100663423,117440513,134217735,201326594,218103809,251658241,268435457,285212673") + req.Header.Set(common.HeaderSchemaVersion, "16777232-1.3,33554433-1.3,33554434-1.3") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + // lanVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + mpart, ok = mparts["webui"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, webuiBytes) + + // ==== setup mock upstream server ==== + fields := make(log.Fields) + mockDoc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + + mockRootDoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + mockDoc.SetRootDocument(mockRootDoc) + + // mockDoc.DeleteSubDocument("wan") + // assert.NilError(t, err) + + mockBytes, err := mockDoc.Bytes() + assert.NilError(t, err) + + db.RefreshRootDocumentVersion(mockDoc) + refreshedRootVersion := mockDoc.RootVersion() + + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.Header().Set(common.HeaderContentLength, strconv.Itoa(len(mockBytes))) + ifNoneMatch := refreshedRootVersion + w.Header().Set(common.HeaderEtag, ifNoneMatch) + w.WriteHeader(http.StatusOK) + w.Write(mockBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(true) + server.SetFilterOutputByBitmapEnabled(true) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderProductClass, "XB8") + req.Header.Set(common.HeaderFirmwareVersion, "CGM4981COM_8.0p4s1_PROD_sey") + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderModelName, "CGM4981COM") + req.Header.Set(common.HeaderSupportedDocs, "16777439,33554435,50331649,67108865,83886081,100663423,117440513,134217735,201326594,218103809,251658241,268435457,285212673") + req.Header.Set(common.HeaderSchemaVersion, "16777232-1.3,33554433-1.3,33554434-1.3") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + + // verify Document is now empty + doc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + assert.Equal(t, doc.Length(), 3) +} diff --git a/http/http_client.go b/http/http_client.go index 833c333..2761f9c 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,7 +25,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "net/http" neturl "net/url" @@ -33,10 +32,11 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/tracing" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -53,7 +53,7 @@ type ErrorResponse struct { Message string `json:"message"` } -type StatusHandlerFunc func([]byte) ([]byte, http.Header, error, bool) +type StatusHandlerFunc func([]byte) ([]byte, http.Header, bool, error) type HttpClient struct { *http.Client @@ -61,6 +61,7 @@ type HttpClient struct { retryInMsecs int statusHandlerFuncMap map[int]StatusHandlerFunc userAgent string + moracideTagPrefix string } func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config) *HttpClient { @@ -83,31 +84,43 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) userAgent := conf.GetString("webconfig.http_client.user_agent") + moracideTagPrefix := strings.ToLower(conf.GetString("webconfig.tracing.moracide_tag_prefix", tracing.DefaultMoracideTagPrefix)) + + var transport http.RoundTripper = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: time.Duration(connectTimeout) * time.Second, + KeepAlive: time.Duration(keepaliveTimeout) * time.Second, + }).DialContext, + MaxIdleConns: 0, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: tlsConfig, + } + return &HttpClient{ Client: &http.Client{ - Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: time.Duration(connectTimeout) * time.Second, - KeepAlive: time.Duration(keepaliveTimeout) * time.Second, - }).DialContext, - MaxIdleConns: 0, - MaxIdleConnsPerHost: maxIdleConnsPerHost, - IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientConfig: tlsConfig, - }, - Timeout: time.Duration(readTimeout) * time.Second, + Transport: transport, + Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, retryInMsecs: retryInMsecs, statusHandlerFuncMap: map[int]StatusHandlerFunc{}, userAgent: userAgent, + moracideTagPrefix: moracideTagPrefix, } } -func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, error, bool) { - fields := common.FilterLogFields(auditFields) +func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { + fields := common.FilterLogFields(auditFields, "status") + + var respMoracideTagsFound bool + defer func(found *bool) { + if !*found { + log.Debugf("http_client: no moracide tags in response") + } + }(&respMoracideTagsFound) // verify a response is received var req *http.Request @@ -120,11 +133,11 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] case "DELETE": req, err = http.NewRequest(method, url, nil) default: - return nil, nil, common.NewError(fmt.Errorf("method=%v", method)), false + return nil, nil, false, common.NewError(fmt.Errorf("method=%v", method)) } if err != nil { - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if header == nil { @@ -144,6 +157,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] header.Set("X-Webpa-Transaction-Id", transactionId) } + c.addMoracideTags(header, auditFields) req.Header = header.Clone() if len(c.userAgent) > 0 { req.Header.Set(common.HeaderUserAgent, c.userAgent) @@ -204,16 +218,22 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] startTime := time.Now() - // the core http call res, err := c.Client.Do(req) // err should be *url.Error - tdiff := time.Now().Sub(startTime) - duration := tdiff.Nanoseconds() / 1000000 + tdiff := time.Since(startTime) + duration := tdiff.Milliseconds() fields[fmt.Sprintf("%v_duration", loggerName)] = duration delete(fields, bodyKey) + // We want to capture any errs returned by server + // i.e. timeout, 503 etc. wouldn't have a valid resp, but possible that + // the err returned by http.Do actually includes an err returned by the backend + // In which case, resp would be non-nil + if res != nil { + respMoracideTagsFound = c.addMoracideTagsFromResponse(res.Header, auditFields) + } var endMessage string if retry > 0 { endMessage = fmt.Sprintf("%v retry=%v ends", loggerName, retry) @@ -235,38 +255,47 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] Message: ue.Error(), StatusCode: http.StatusGatewayTimeout, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if errors.Is(innerErr, io.EOF) { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusBadGateway, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if _, ok := innerErr.(*net.OpError); ok { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusServiceUnavailable, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } // Unknown err still appear as 500 } - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if res.Body != nil { defer res.Body.Close() } fields[fmt.Sprintf("%v_status", loggerName)] = res.StatusCode - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) if err != nil { + // XPC-23206 catch the timeout/context_cancellation error + // ex: context deadline exceeded (Client.Timeout or context cancellation while reading body) + lowerErrText := strings.ToLower(err.Error()) + if strings.Contains(lowerErrText, "timeout") { + err = common.RemoteHttpError{ + Message: err.Error(), + StatusCode: http.StatusGatewayTimeout, + } + } fields[errorKey] = err.Error() if userAgent != "mget" { log.WithFields(fields).Info(endMessage) } - return nil, nil, common.NewError(err), false + return nil, nil, true, common.NewError(err) } rbody := string(rbytes) @@ -321,11 +350,27 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] switch res.StatusCode { case http.StatusForbidden, http.StatusBadRequest, http.StatusNotFound: - return rbytes, nil, common.NewError(err), false + return rbytes, nil, false, common.NewError(err) } - return rbytes, nil, common.NewError(err), true + return rbytes, nil, true, common.NewError(err) + } else if res.StatusCode > 200 { + var pokeResponse PokeResponse + var message string + if err := json.Unmarshal(rbytes, &pokeResponse); err == nil { + if len(pokeResponse.Parameters) > 0 { + message = pokeResponse.Parameters[0].Message + } + } + if len(message) == 0 { + message = http.StatusText(res.StatusCode) + } + rherr := common.RemoteHttpError{ + Message: message, + StatusCode: res.StatusCode, + } + return rbytes, nil, false, common.NewError(rherr) } - return rbytes, res.Header, nil, false + return rbytes, res.Header, false, nil } func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { @@ -342,7 +387,7 @@ func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Heade if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, err, cont = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } @@ -364,3 +409,37 @@ func (c *HttpClient) StatusHandler(status int) StatusHandlerFunc { } return nil } + +// addMoracideTags - if ctx has a moracide tag as a header, add it to the headers +// Also add traceparent, tracestate headers +func (c *HttpClient) addMoracideTags(header http.Header, fields log.Fields) { + if itf, ok := fields["out_traceparent"]; ok { + if ss, ok := itf.(string); ok { + if len(ss) > 0 { + header.Set(common.HeaderTraceparent, ss) + } + } + } + if itf, ok := fields["out_tracestate"]; ok { + if ss, ok := itf.(string); ok { + if len(ss) > 0 { + header.Set(common.HeaderTracestate, ss) + } + } + } + + moracide := util.FieldsGetString(fields, "req_moracide_tag") + if len(moracide) > 0 { + header.Set(common.HeaderMoracide, moracide) + } +} + +func (c *HttpClient) addMoracideTagsFromResponse(header http.Header, fields log.Fields) bool { + var respMoracideTagsFound bool + moracide := header.Get(common.HeaderMoracide) + if len(moracide) > 0 { + fields["resp_moracide_tag"] = moracide + respMoracideTagsFound = true + } + return respMoracideTagsFound +} diff --git a/http/main_test.go b/http/main_test.go index 7ccd1f2..6da258a 100644 --- a/http/main_test.go +++ b/http/main_test.go @@ -45,9 +45,8 @@ func TestMain(m *testing.M) { panic(err) } + _ = GetTestDatabaseClient(sc) log.SetOutput(io.Discard) - returnCode := m.Run() - os.Exit(returnCode) } diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 07c086b..66ac2a6 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,32 +23,34 @@ import ( "net/http" "time" - "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) const ( - mqttHostDefault = "https://hcbroker.staging.us-west-2.plume.comcast.net" - mqttUrlTemplate = "%s/v2/mqtt/pub/x/to/%s/webconfig" + defaultMqttHost = "http://localhost:12347" + defaultMqttUrlTemplate = "%s/%s" ) type MqttConnector struct { *HttpClient host string serviceName string + urlTemplate string } func NewMqttConnector(conf *configuration.Config, tlsConfig *tls.Config) *MqttConnector { serviceName := "mqtt" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, mqttHostDefault) + host := conf.GetString("webconfig.mqtt.host", defaultMqttHost) + urlTemplate := conf.GetString("webconfig.mqtt.url_template", defaultMqttUrlTemplate) return &MqttConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, + urlTemplate: urlTemplate, } } @@ -60,12 +62,20 @@ func (c *MqttConnector) SetMqttHost(host string) { c.host = host } +func (c *MqttConnector) MqttUrlTemplate() string { + return c.urlTemplate +} + +func (c *MqttConnector) SetMqttUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *MqttConnector) ServiceName() string { return c.serviceName } func (c *MqttConnector) PostMqtt(cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { - url := fmt.Sprintf(mqttUrlTemplate, c.MqttHost(), cpeMac) + url := fmt.Sprintf(c.MqttUrlTemplate(), c.MqttHost(), cpeMac) var traceId, xmTraceId, outTraceparent, outTracestate string if itf, ok := fields["xmoney_trace_id"]; ok { diff --git a/http/mqtt_connector_test.go b/http/mqtt_connector_test.go index 48a2066..b6cdaae 100644 --- a/http/mqtt_connector_test.go +++ b/http/mqtt_connector_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -30,7 +30,7 @@ import ( func TestPayloadBuilder(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcData := util.Dict{ "device_id": "mac:044e5a22c9bf", diff --git a/http/multipart.go b/http/multipart.go index 6c1c663..bd154ba 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,10 +24,10 @@ import ( "strconv" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -38,6 +38,7 @@ var ( "X-System-Schema-Version", "X-System-Supported-Docs", "X-System-Product-Class", + "X-System-Type", "Transaction-Id", } ) @@ -51,19 +52,10 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. // ==== data integrity check ==== params := mux.Vars(r) - mac, ok := params["mac"] - if !ok { - Error(w, http.StatusNotFound, nil) - return - } + mac := params["mac"] mac = strings.ToUpper(mac) - if s.ValidateMacEnabled() { - if !util.ValidateMac(mac) { - err := *common.NewHttp400Error("invalid mac") - Error(w, http.StatusBadRequest, common.NewError(err)) - return - } - } + // Moved validateMac to CpeMiddleware + r.Header.Set(common.HeaderDeviceId, mac) // ==== processing ==== @@ -75,11 +67,18 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. return } fields := xw.Audit() - fields["cpe_mac"] = mac - if qGroupIds, ok := r.URL.Query()["group_id"]; ok { - fields["group_id"] = qGroupIds[0] - r.Header.Set(common.HeaderDocName, qGroupIds[0]) + + // enforce strict query parameters check + err := util.ValidateQueryParams(r, s.ValidSubdocIdMap(), fields) + if err != nil && s.QueryParamsValidationEnabled() { + if errors.Is(err, common.ErrInvalidQueryParams) { + Error(w, http.StatusBadRequest, nil) + log.WithFields(fields).Error(err) + return + } + Error(w, http.StatusInternalServerError, err) + return } // handle empty schema version header @@ -88,13 +87,13 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. } status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) - if err != nil && respBytes == nil { - respBytes = []byte(err.Error()) - } - // REMINDER 404 use standard response - if status == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) + switch status { + case http.StatusNotFound: + Error(w, status, nil) + return + case http.StatusConflict: + w.WriteHeader(status) return } @@ -102,6 +101,11 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. w.Header().Set(k, respHeader.Get(k)) } + if err != nil && respBytes == nil { + Error(w, status, common.NewError(err)) + return + } + w.WriteHeader(status) _, _ = w.Write(respBytes) } @@ -111,13 +115,36 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin fields["is_primary"] = true c := s.DatabaseClient - uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) respHeader := make(http.Header) userAgent := rHeader.Get("User-Agent") - document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, err := db.BuildGetDocument(c, rHeader, route, fields) - if uconn == nil { + // factory reset handling + ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + if ifNoneMatch == "NONE" || ifNoneMatch == "NONE-REBOOT" { + status, respHeader, rbytes, err := BuildFactoryResetResponse(s, rHeader, fields) + if err != nil { + return status, respHeader, rbytes, common.NewError(err) + } + return status, respHeader, rbytes, nil + } + + document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, messages, err := db.BuildGetDocument(c, rHeader, route, fields) + if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { + s.ForwardSuccessKafkaMessages(messages, fields) + } + + // root_document locked + if errors.Is(err, common.ErrRootDocumentLocked) { + return http.StatusConflict, respHeader, nil, common.NewError(err) + } + + if !s.UpstreamEnabled() { + if postUpstream { + rdoc := oldRootDocument.Clone() + rdoc.UpdateMetadata(newRootDocument) + document.SetRootDocument(rdoc) + } if err != nil { if !s.IsDbNotFound(err) { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -135,6 +162,19 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document.DeleteSubDocument(subdocId) } + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + + if s.FilterOutputByBitmapEnabled() { + document = document.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) + } + + if document.Length() == 0 { + return http.StatusNotFound, respHeader, nil, nil + } + respBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -147,7 +187,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -162,25 +202,42 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } } if document == nil { - rootDocument := common.NewRootDocument(0, "", "", "", "", "", "") + rootDocument := common.NewRootDocument(0, "", "", "", "", "", "", "", "") document = common.NewDocument(rootDocument) } + if userAgent == "mget" { + postUpstream = false + } + var respBytes []byte respStatus := http.StatusNotModified if document.Length() > 0 { + if !postUpstream { + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + } + + if s.FilterOutputByBitmapEnabled() { + document = document.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) + } + respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } respStatus = http.StatusOK + } else if len(document.RootVersion()) == 0 { + respStatus = http.StatusNotFound } // mget ==> no upstream if userAgent == "mget" { - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) - return http.StatusOK, respHeader, respBytes, nil + return respStatus, respHeader, respBytes, nil } if !postUpstream { @@ -189,7 +246,12 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - respHeader.Set("Content-type", common.MultipartContentType) + // 304 + if document.Length() == 0 { + respStatus = http.StatusNotModified + } + + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return respStatus, respHeader, respBytes, nil } @@ -197,8 +259,8 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader := rHeader.Clone() + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) @@ -253,19 +315,33 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) + var bitmap int + if newRootDocument.Bitmap != 0 { + bitmap = newRootDocument.Bitmap + } else if oldRootDocument.Bitmap != 0 { + bitmap = oldRootDocument.Bitmap + } // filter by versionMap and filter by blockedIds - finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") + finalRootDocument := common.NewRootDocument(bitmap, "", "", "", "", upstreamRespEtag, "", "", "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) + + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } + } + finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) for _, subdocId := range c.BlockedSubdocIds() { finalFilteredDocument.DeleteSubDocument(subdocId) } - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalFilteredDocument, document, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + if s.FilterOutputByBitmapEnabled() { + finalFilteredDocument = finalFilteredDocument.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) } // 304 @@ -273,6 +349,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusNotModified, upstreamRespHeader, nil, nil } + finalFilteredDocument, err = db.LoadRefSubDocuments(c, finalFilteredDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalFilteredBytes, err := finalFilteredDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalFilteredBytes, common.NewError(err) @@ -280,3 +360,128 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusOK, upstreamRespHeader, finalFilteredBytes, nil } + +func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { + c := s.DatabaseClient + mac := rHeader.Get(common.HeaderDeviceId) + respHeader := make(http.Header) + + fieldsDict := make(util.Dict) + fieldsDict.Update(fields) + partnerId := rHeader.Get(common.HeaderPartnerID) + if len(partnerId) == 0 { + partnerId = fieldsDict.GetString("partner") + } + + rootDocument, err := db.PreprocessRootDocument(c, rHeader, mac, partnerId, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + + document, err := c.GetDocument(mac, fields) + if err != nil { + if s.IsDbNotFound(err) { + return http.StatusNotFound, respHeader, nil, nil + } else { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + } + if document == nil { + document = common.NewDocument(rootDocument) + } else { + document.SetRootDocument(rootDocument) + } + + oldDocBytes, err := document.Bytes() + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + + if !s.UpstreamEnabled() { + err := c.DeleteDocument(mac) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + return http.StatusNotFound, respHeader, nil, nil + } + + // ============================= + // upstream handling + // ============================= + upstreamHeader := rHeader.Clone() + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) + upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) + upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) + + if itf, ok := fields["audit_id"]; ok { + auditId := itf.(string) + if len(auditId) > 0 { + upstreamHeader.Set(common.HeaderAuditid, auditId) + } + } + + if s.TokenManager != nil { + token := rHeader.Get("Authorization") + if len(token) > 0 { + upstreamHeader.Set("Authorization", token) + } else { + token = s.Generate(mac, 86400) + upstreamHeader.Set("Authorization", "Bearer "+token) + } + } + + // call /upstream to handle factory reset + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) + if err != nil { + var rherr common.RemoteHttpError + if errors.As(err, &rherr) { + return rherr.StatusCode, respHeader, nil, common.NewError(err) + } + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + + // ==== parse the upstreamRespBytes and store them ==== + finalMparts, err := util.ParseMultipartAsList(upstreamRespHeader, upstreamRespBytes) + if err != nil { + return http.StatusInternalServerError, respHeader, oldDocBytes, common.NewError(err) + } + upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) + finalRootDocument := rootDocument.Clone() + finalRootDocument.Version = upstreamRespEtag + + finalDocument := common.NewDocument(finalRootDocument) + finalDocument.SetSubDocuments(finalMparts) + for _, subdocId := range c.BlockedSubdocIds() { + finalDocument.DeleteSubDocument(subdocId) + } + + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } + } + + if finalDocument.Length() == 0 { + return http.StatusNotFound, upstreamRespHeader, nil, nil + } + + finalDocument, err = db.LoadRefSubDocuments(c, finalDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } + + // filter by bitmaps and blockedIds + if s.FilterOutputByBitmapEnabled() { + finalDocument = finalDocument.FilterByBitmap(s.BitmapFilterExemptSubdocIds()...) + } + + finalBytes, err := finalDocument.Bytes() + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) + } + + return http.StatusOK, upstreamRespHeader, finalBytes, nil +} diff --git a/http/multipart_test.go b/http/multipart_test.go index e4ebd4c..775c1e7 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -14,20 +14,22 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "bytes" "encoding/hex" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strings" "testing" + "time" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "github.com/vmihailenco/msgpack/v4" @@ -49,20 +51,20 @@ func TestMultipartConfigHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -78,20 +80,20 @@ func TestMultipartConfigHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -102,7 +104,7 @@ func TestMultipartConfigHandler(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -133,25 +135,25 @@ func TestMultipartConfigHandler(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, etag) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) - // ==== cal GET /config with if-none-match partial match ==== + // ==== call GET /config with if-none-match partial match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) ifNoneMatch := fmt.Sprintf("foo,%v,bar", lanMpartVersion) req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -167,6 +169,37 @@ func TestMultipartConfigHandler(t *testing.T) { assert.NilError(t, err) parameters = response.Parameters assert.Equal(t, len(parameters), 1) + + // test root_document lock + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + rootdoc.LockedTill = int(time.Now().UnixMilli()) + 1000 + err = server.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + // get document again without the feature flag enabled + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get document again with the feature flag enabled + server.SetLockRootDocumentEnabled(true) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusConflict) + + time.Sleep(time.Duration(1) * time.Second) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) } func TestCpeMiddleware(t *testing.T) { @@ -193,20 +226,20 @@ func TestCpeMiddleware(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -220,7 +253,7 @@ func TestCpeMiddleware(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router1).Result() - rbytes, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusForbidden) @@ -231,7 +264,32 @@ func TestCpeMiddleware(t *testing.T) { assert.NilError(t, err) req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) res = ExecuteRequest(req, router1).Result() - rbytes, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // change the min trust to 1000 + server1.SetMinTrust(1000) + assert.Equal(t, 1000, server1.MinTrust()) + zeroToken := server1.Generate(cpeMac, 86400, 0) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + + // change the min trust back to 0 + server1.SetMinTrust(0) + assert.Equal(t, 0, server1.MinTrust()) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -245,25 +303,25 @@ func TestVersionFiltering(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -271,25 +329,25 @@ func TestVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -300,7 +358,7 @@ func TestVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -325,7 +383,7 @@ func TestVersionFiltering(t *testing.T) { assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, etag) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -337,7 +395,7 @@ func TestVersionFiltering(t *testing.T) { ifNoneMatch := fmt.Sprintf("foo,%v,bar", lanMpartVersion) req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -360,7 +418,7 @@ func TestVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -389,25 +447,25 @@ func TestUpstreamVersionFiltering(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -415,25 +473,25 @@ func TestUpstreamVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -444,7 +502,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -472,7 +530,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusServiceUnavailable) @@ -481,7 +539,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusServiceUnavailable) @@ -494,7 +552,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { w.Header().Set(k, r.Header.Get(k)) } w.WriteHeader(http.StatusOK) - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { _, err := w.Write(rbytes) assert.NilError(t, err) } @@ -509,7 +567,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -531,7 +589,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderIfNoneMatch, matchedIfNoneMatch) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -542,7 +600,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderIfNoneMatch, mismatchedIfNoneMatch) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -559,30 +617,29 @@ func TestUpstreamVersionFiltering(t *testing.T) { func TestMqttUpstreamVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) - cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -590,25 +647,25 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -622,7 +679,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err := util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -669,7 +726,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { w.Header().Set(k, r.Header.Get(k)) } w.WriteHeader(http.StatusOK) - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { _, err := w.Write(rbytes) assert.NilError(t, err) } @@ -686,7 +743,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -721,7 +778,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -741,25 +798,25 @@ func TestMultipartConfigMismatch(t *testing.T) { // ==== group 1 lan ==== subdocId := "lan" m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -767,26 +824,26 @@ func TestMultipartConfigMismatch(t *testing.T) { // ==== group 2 wan ==== subdocId = "wan" - wanBytes := util.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) assert.NilError(t, err) // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -797,7 +854,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -819,37 +876,719 @@ func TestMultipartConfigMismatch(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header1 := "NONE," + lanMpartVersion req.Header.Set(common.HeaderIfNoneMatch, header1) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header2 := "NONE,123" req.Header.Set(common.HeaderIfNoneMatch, header2) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header3 := etag + ",123" req.Header.Set(common.HeaderIfNoneMatch, header3) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) +} + +func TestStateCorrectionEnabled(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := common.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== group 3 mesh ==== + subdocId = "mesh" + meshBytes := common.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, meshBytes) + + // ==== group 3 moca ==== + subdocId = "moca" + mocaBytes := common.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, mocaBytes) + + // ==== GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 4) + etag := res.Header.Get(common.HeaderEtag) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + mpart, ok = mpartMap["mesh"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, meshBytes) + meshVersion := mpart.Version + + mpart, ok = mpartMap["moca"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, mocaBytes) + mocaVersion := mpart.Version + "x" + + // verify all states are in-deployment + lanSubdocument, err := server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.InDeployment) + + wanSubdocument, err := server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err := server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.InDeployment) + + mocaSubdocument, err := server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.InDeployment) + + // ==== setup special error conditions to test state correction scenario ==== + lanState := common.PendingDownload + lanSubdocument.SetState(&lanState) + err = server.SetSubDocument(cpeMac, "lan", lanSubdocument) + assert.NilError(t, err) + + wanState := common.InDeployment + wanSubdocument.SetState(&wanState) + err = server.SetSubDocument(cpeMac, "wan", wanSubdocument) + assert.NilError(t, err) + + meshState := common.Failure + meshErrorCode := 307 + meshErrorDetails := "NACK:OneWifi," + meshSubdocument.SetState(&meshState) + meshSubdocument.SetErrorCode(&meshErrorCode) + meshSubdocument.SetErrorDetails(&meshErrorDetails) + err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) + assert.NilError(t, err) + + mocaState := common.PendingDownload + mocaSubdocument.SetState(&mocaState) + err = server.SetSubDocument(cpeMac, "moca", mocaSubdocument) + assert.NilError(t, err) + + // ==== call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(false) + + configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan,mesh,moca", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + header1 := fmt.Sprintf("%v,%v,%v,%v,%v", etag, lanVersion, wanVersion, meshVersion, mocaVersion) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify the states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + oldLanUpdatedTime := *lanSubdocument.UpdatedTime() + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + oldWanUpdatedTime := *wanSubdocument.UpdatedTime() + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Failure) + assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) + assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) + oldMeshUpdatedTime := *meshSubdocument.UpdatedTime() + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + oldMocaUpdatedTime := *mocaSubdocument.UpdatedTime() + + // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(true) + defer func() { + server.SetStateCorrectionEnabled(false) + }() + + time.Sleep(time.Duration(100) * time.Millisecond) + + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify all version-matched states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *lanSubdocument.UpdatedTime() > oldLanUpdatedTime) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *wanSubdocument.UpdatedTime() > oldWanUpdatedTime) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + assert.Equal(t, *meshSubdocument.ErrorCode(), 0) + assert.Equal(t, *meshSubdocument.ErrorDetails(), "") + assert.Assert(t, *meshSubdocument.UpdatedTime() > oldMeshUpdatedTime) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + assert.Assert(t, *mocaSubdocument.UpdatedTime() == oldMocaUpdatedTime) +} + +func TestCorruptedEncryptedDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + tdbclient, ok := server.DatabaseClient.(*cassandra.CassandraClient) + if !ok { + t.Skip("Only test in cassandra env") + } + + cpeMac := util.GenerateRandomCpeMac() + encSubdocIds := []string{} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds := tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, !tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 1 setup lan subdoc ==== + // post + subdocId := "lan" + lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + lanBytes := common.RandomBytes(100, 150) + req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + _ = rbytes + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", lanUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 2 setup wan subdoc ==== + // post + subdocId = "wan" + wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + wanBytes := common.RandomBytes(100, 150) + req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", wanUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 3 setup privatessid subdoc ==== + // post + subdocId = "privatessid" + privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + privatessidBytes := common.RandomBytes(100, 150) + req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", privatessidUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 read the document ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 3) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + mpart, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + + // ==== step 5 set privatessid as an encrypted subdoc ==== + encSubdocIds = []string{"privatessid"} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds = tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 6 read the document expect no error but 1 less subdoc ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + _, ok = mpartMap["privatessid"] + assert.Assert(t, !ok) +} + +func TestValidateQueryParams(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetQueryParamsValidationEnabled(true) + assert.Assert(t, server.QueryParamsValidationEnabled()) + defer server.SetQueryParamsValidationEnabled(false) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := common.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123") + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + etag := res.Header.Get(common.HeaderEtag) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanMpartVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanMpartVersion := mpart.Version + matchedIfNoneMatch := fmt.Sprintf("%v,%v,%v", etag, lanMpartVersion, wanMpartVersion) + + // case 9 versions matched 304 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, matchedIfNoneMatch) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) } + +func TestMultipartConfigHandlerNewHeaders(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== POST lan subdoc ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== GET /config with X-System-Product-Class and X-System-Type headers ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partnerId1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + productClass1 := "rg" + accountType1 := "residential" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partnerId1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + req.Header.Set(common.HeaderProductClass, productClass1) + req.Header.Set(common.HeaderAccountType, accountType1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // read from db and verify product_class and account_type are stored + rdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass1, rdoc.ProductClass) + assert.Equal(t, accountType1, rdoc.AccountType) + + // ==== GET /config again with updated X-System-Product-Class and X-System-Type headers ==== + productClass2 := "xb8" + accountType2 := "business" + + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partnerId1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + req.Header.Set(common.HeaderProductClass, productClass2) + req.Header.Set(common.HeaderAccountType, accountType2) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + + // verify updated values are persisted in the root_document table + rdoc, err = server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass2, rdoc.ProductClass) + assert.Equal(t, accountType2, rdoc.AccountType) +} diff --git a/http/poke_handler.go b/http/poke_handler.go index 62c4092..266023d 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,13 +25,14 @@ import ( "sort" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { + // handler params := mux.Vars(r) mac := params["mac"] mac = strings.ToUpper(mac) @@ -62,6 +63,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } fields := xw.Audit() + fields["API"] = util.GetAPIName(r.URL.RawQuery) // extract "metrics_agent" metricsAgent := "default" @@ -107,10 +109,8 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - if rherr.StatusCode == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) - return - } + Error(w, rherr.StatusCode, common.NewError(err)) + return } Error(w, http.StatusInternalServerError, common.NewError(err)) return @@ -156,16 +156,15 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(mac, token, pokeStr, fields) + transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) + if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - status := http.StatusInternalServerError + status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 - } else if rherr.StatusCode > http.StatusInternalServerError { - status = rherr.StatusCode } // parse the core message diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index bbd0884..e9952a4 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -14,13 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "bytes" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strconv" @@ -35,7 +35,9 @@ import ( ) var ( - mockedWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) + mockWebpaPoke202Response = []byte(`{"parameters":[{"message":"Previous request is in progress","name":"Device.X_RDK_WebConfig.ForceSync"}],"statusCode":202}`) ) func TestPokeHandler(t *testing.T) { @@ -47,7 +49,7 @@ func TestPokeHandler(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -62,7 +64,7 @@ func TestPokeHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusNoContent) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -89,7 +91,7 @@ func TestPokeHandlerWithCpe(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() } @@ -116,21 +118,21 @@ func TestBuildMqttSendDocument(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -148,21 +150,21 @@ func TestBuildMqttSendDocument(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -189,16 +191,16 @@ func TestBuildMqttSendDocument(t *testing.T) { req, err = http.NewRequest("POST", mqttUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusAccepted) // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -207,10 +209,10 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -221,24 +223,24 @@ func TestBuildMqttSendDocument(t *testing.T) { fields = make(log.Fields) document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) // ==== step 7 change the subdoc again ==== - lanBytes2 := util.RandomBytes(100, 150) + lanBytes2 := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes2)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes2) @@ -249,22 +251,22 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 8 check the document length === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 1) + assert.Equal(t, document.Length(), 2) // ==== step 9 send a document through mqtt ==== req, err = http.NewRequest("POST", mqttUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusAccepted) // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -274,7 +276,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 10 check the length again === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) } func TestPokeHandlerInvalidMac(t *testing.T) { @@ -286,7 +288,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -301,7 +303,65 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusBadRequest) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} + +func TestPokeHandlerWebpa403(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write(mockWebpaPoke403Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} + +func TestPokeHandlerWebpa202(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write(mockWebpaPoke202Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusAccepted) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() } diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go new file mode 100644 index 0000000..cd8743d --- /dev/null +++ b/http/refsubdocument_handler.go @@ -0,0 +1,125 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package http + +import ( + "errors" + "net/http" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" +) + +func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + refsubdoc, err := s.GetRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + LogError(w, err) + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + if refsubdoc.Version() != nil { + w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) + } + w.WriteHeader(http.StatusOK) + w.Write(refsubdoc.Payload()) +} + +func (s *WebconfigServer) PostRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, bbytes, _, err := s.ValidateRefData(w, r, true) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + // handle version header + version := r.Header.Get(common.HeaderSubdocumentVersion) + if len(version) == 0 { + version = util.GetMurmur3Hash(bbytes) + } + + refsubdoc := common.NewRefSubDocument(bbytes, &version) + + err = s.SetRefSubDocument(refId, refsubdoc) + if err != nil { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + + WriteOkResponse(w, nil) +} + +func (s *WebconfigServer) DeleteRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + err = s.DeleteRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + WriteOkResponse(w, nil) +} diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go new file mode 100644 index 0000000..f139ac6 --- /dev/null +++ b/http/refsubdocument_handler_test.go @@ -0,0 +1,221 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "testing" + + "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "gotest.tools/assert" +) + +func TestRefSubDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + refId := uuid.New().String() + bbytes := common.RandomBytes(100, 150) + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes) + + // delete + req, err = http.NewRequest("DELETE", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get but expect 404 + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} + +func TestSubDocumentWithInvalidRefDoc(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + referenceIndicatorBytes := make([]byte, 4) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 setup refdoc1 and subdoc1 ==== + refId1 := uuid.New().String() + bbytes1 := common.RandomBytes(100, 150) + subdocId1 := "defaultrfc" + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId1) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes1)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes1) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId1) + xbytes := []byte(refId1) + tmpbytes := append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 2 setup refdoc2 and subdoc2 ==== + refId2 := uuid.New().String() + bbytes2 := common.RandomBytes(100, 150) + subdocId2 := "defaulttelemetry" + + // post + url = fmt.Sprintf("/api/v1/reference/%v/document", refId2) + req, err = http.NewRequest("POST", url, bytes.NewReader(bbytes2)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes2) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId2) + xbytes = []byte(refId2) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 3 GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok := mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) + + // ==== step 4 setup 3rd subdoc but link it to an non-existent refdoc3 ==== + refId3 := uuid.New().String() + subdocId3 := "defaultdcm" + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId3) + xbytes = []byte(refId3) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 5 GET /config returns 2 subdocs and no errors ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) +} diff --git a/http/response.go b/http/response.go index 560a467..baebe01 100644 --- a/http/response.go +++ b/http/response.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,6 +23,7 @@ import ( "net/http" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -53,7 +54,8 @@ func WriteByMarshal(w http.ResponseWriter, status int, o interface{}) { LogError(w, common.NewError(err)) return } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(status) w.Write(rbytes) } @@ -106,14 +108,16 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, SetAuditValue(w, "response", resp) rbytes = []byte(fmt.Sprintf(OkResponseTemplate, s, stateText, updatedTime)) } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(http.StatusOK) w.Write(rbytes) } // this is used to return default tr-181 payload while the cpe is not in the db func WriteContentTypeAndResponse(w http.ResponseWriter, rbytes []byte, version string, contentType string) { - w.Header().Set("Content-type", contentType) + w.Header().Set(common.HeaderContentType, contentType) + addMoracideTagsAsResponseHeaders(w) w.Header().Set(common.HeaderEtag, version) w.WriteHeader(http.StatusOK) w.Write(rbytes) @@ -135,11 +139,14 @@ func WriteErrorResponse(w http.ResponseWriter, status int, err error) { } func Error(w http.ResponseWriter, status int, err error) { - // calling WriteHeader() multiple times will cause errors in "content-type" + // calling WriteHeader() multiple times will cause errors in common.HeaderContentType // ==> errors like 'superfluous response.WriteHeader call' in stderr switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: w.WriteHeader(status) + case http.StatusAccepted: + SetAuditValue(w, "response", err) + WriteByMarshal(w, status, err) default: WriteErrorResponse(w, status, err) } @@ -147,14 +154,35 @@ func Error(w http.ResponseWriter, status int, err error) { func WriteResponseBytes(w http.ResponseWriter, rbytes []byte, statusCode int, vargs ...string) { if len(vargs) > 0 { - w.Header().Set("Content-type", vargs[0]) + w.Header().Set(common.HeaderContentType, vargs[0]) } + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(statusCode) w.Write(rbytes) } func WriteFactoryResetResponse(w http.ResponseWriter) { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(http.StatusOK) w.Write([]byte{}) } + +func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { + xw, ok := w.(*XResponseWriter) + if !ok { + return + } + fields := xw.Audit() + if fields == nil { + return + } + + moracide := util.FieldsGetString(fields, "resp_moracide_tag") + if len(moracide) == 0 { + moracide = util.FieldsGetString(fields, "req_moracide_tag") + } + if len(moracide) > 0 { + w.Header().Set(common.HeaderMoracide, moracide) + } +} diff --git a/http/response_writer.go b/http/response_writer.go index 9f0372c..47c382e 100644 --- a/http/response_writer.go +++ b/http/response_writer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 5debc71..f292764 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,7 +22,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "testing" @@ -48,10 +48,10 @@ func TestRootDocumentHandler(t *testing.T) { // ==== step 0 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -79,9 +79,9 @@ func TestRootDocumentHandler(t *testing.T) { rootdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + expectedBitmap1, err := common.GetCpeBitmap(supportedDocs1) assert.NilError(t, err) - expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") + expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "", "", "") assert.DeepEqual(t, rootdoc, expectedRootdoc) // ==== step 2 build lan subdoc ==== @@ -94,19 +94,19 @@ func TestRootDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -121,19 +121,19 @@ func TestRootDocumentHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -147,7 +147,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set(common.HeaderPartnerID, partner1) req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.Equal(t, res.StatusCode, http.StatusOK) assert.NilError(t, err) res.Body.Close() @@ -182,7 +182,7 @@ func TestRootDocumentHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -190,7 +190,7 @@ func TestRootDocumentHandler(t *testing.T) { err = json.Unmarshal(rbytes, &getResp) assert.NilError(t, err) - expectedRootdoc = common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, "") + expectedRootdoc = common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, "", "", "") assert.Equal(t, getResp.Data, *expectedRootdoc) } @@ -207,7 +207,7 @@ func TestPostRootDocumentHandler(t *testing.T) { schemaVersion1 := "33554433-1.3,33554434-1.3" etag := strconv.Itoa(int(time.Now().Unix())) queryParams1 := "stormReadyWifi=true" - srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1) + srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1, "", "") bbytes, err := json.Marshal(srcDoc1) assert.NilError(t, err) @@ -223,7 +223,7 @@ func TestPostRootDocumentHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -233,3 +233,139 @@ func TestPostRootDocumentHandler(t *testing.T) { assert.Equal(t, getResp.Data, *srcDoc1) } + +func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 GET /rootdocument and expect 404 ==== + rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) + req, err := http.NewRequest("GET", rootdocUrl, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 2 GET /config device ==== + // boots up but with out data in db + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + + expectedBitmap1, err := common.GetCpeBitmap(supportedDocs1) + assert.NilError(t, err) + expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "", "", "") + assert.DeepEqual(t, rootdoc, expectedRootdoc) + + // ==== step 2 ==== + supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(common.RandomBytes(10, 15)) + firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(common.RandomBytes(10, 15)) + modelName2 := "CGM4331COM" + string(common.RandomBytes(10, 15)) + partner2 := "comcast" + string(common.RandomBytes(10, 15)) + schemaVersion2 := "33554433-1.3,33554434-1.3" + string(common.RandomBytes(10, 15)) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs2) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion2) + req.Header.Set(common.HeaderModelName, modelName2) + req.Header.Set(common.HeaderPartnerID, partner2) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc2, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rootdoc.Equals(rootdoc2)) +} + +func TestMultipartConfigHandlerStoresProductClassAndAccountType(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // make a GET /config request with X-System-Product-Class and X-System-Type headers + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + productClass1 := "rg" + accountType1 := "residential" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + req.Header.Set(common.HeaderProductClass, productClass1) + req.Header.Set(common.HeaderAccountType, accountType1) + + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + // expect 404 since no subdocs are stored + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db and verify product_class and account_type are stored + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass1, rootdoc.ProductClass) + assert.Equal(t, accountType1, rootdoc.AccountType) + + // ==== make a second request with different product_class and account_type ==== + productClass2 := "xb8" + accountType2 := "business" + req2, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req2.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req2.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req2.Header.Set(common.HeaderModelName, modelName1) + req2.Header.Set(common.HeaderPartnerID, partner1) + req2.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + req2.Header.Set(common.HeaderProductClass, productClass2) + req2.Header.Set(common.HeaderAccountType, accountType2) + + res2 := ExecuteRequest(req2, router).Result() + _, err = io.ReadAll(res2.Body) + assert.NilError(t, err) + + // verify updated values are stored + rootdoc2, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, productClass2, rootdoc2.ProductClass) + assert.Equal(t, accountType2, rootdoc2.AccountType) +} diff --git a/http/router.go b/http/router.go index 7645c51..103c77a 100644 --- a/http/router.go +++ b/http/router.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -95,9 +95,9 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.TestingMiddleware) } else { if s.ServerApiTokenAuthEnabled() { - sub2.Use(s.ApiMiddleware) + sub2.Use(s.SpanMiddleware, s.ApiMiddleware) } else { - sub2.Use(s.NoAuthMiddleware) + sub2.Use(s.SpanMiddleware, s.NoAuthMiddleware) } } sub2.HandleFunc("", s.PokeHandler).Methods("POST") @@ -127,5 +127,19 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { } sub4.HandleFunc("", s.DeleteDocumentHandler).Methods("DELETE") + sub5 := router.Path("/api/v1/reference/{ref}/document").Subrouter() + if testOnly { + sub5.Use(s.TestingMiddleware) + } else { + if s.ServerApiTokenAuthEnabled() { + sub5.Use(s.ApiMiddleware) + } else { + sub5.Use(s.NoAuthMiddleware) + } + } + sub5.HandleFunc("", s.GetRefSubDocumentHandler).Methods("GET") + sub5.HandleFunc("", s.PostRefSubDocumentHandler).Methods("POST") + sub5.HandleFunc("", s.DeleteRefSubDocumentHandler).Methods("DELETE") + return router } diff --git a/http/simple_handler_test.go b/http/simple_handler_test.go index 0e4e093..28109fa 100644 --- a/http/simple_handler_test.go +++ b/http/simple_handler_test.go @@ -18,7 +18,7 @@ package http import ( - "io/ioutil" + "io" "net/http" "testing" @@ -36,7 +36,7 @@ func TestSimpleHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -45,7 +45,7 @@ func TestSimpleHandler(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, len(rbytes), 0) @@ -56,7 +56,7 @@ func TestSimpleHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, len(rbytes), 0) @@ -70,7 +70,7 @@ func TestNotifHandler(t *testing.T) { req, err := http.NewRequest("GET", "/notif", nil) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) res.Body.Close() diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 880fe57..5561b85 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,13 +22,18 @@ import ( "fmt" "net/http" "strings" + "time" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) +const ( + notFoundProfileText = `{"profiles":[]}` +) + func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { // ==== data integrity check ==== params := mux.Vars(r) @@ -49,17 +54,112 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r return } - // append the extra query_params if any - var queryParams string - rootdoc, err := s.GetRootDocument(mac) - if err != nil { - if !s.IsDbNotFound(err) { + // Check if supplementary_precook feature is enabled + if s.SupplementaryPrecookEnabled() { + // Check state from xpc_group_config by cpe_mac and group_id=telemetry + telemetrySubdoc, err := s.GetSubDocument(mac, "telemetry") + if err != nil && !s.IsDbNotFound(err) { Error(w, http.StatusInternalServerError, common.NewError(err)) return } + + if telemetrySubdoc != nil && telemetrySubdoc.State() != nil { + state := *telemetrySubdoc.State() + + // If state=1 (Deployed), check expiry first before returning 304 + if state == common.Deployed { + // Check if expiry has passed + if telemetrySubdoc.Expiry() != nil { + currentTime := int(time.Now().UnixNano() / 1000000) + if *telemetrySubdoc.Expiry() <= currentTime { + // Expiry has passed, continue with normal xconf flow + // Do not return 304 + } else { + // Expiry has not passed, return 304 + w.WriteHeader(http.StatusNotModified) + return + } + } else { + // No expiry set, return 304 + w.WriteHeader(http.StatusNotModified) + return + } + } + + // If state in (2, 3, 4), use the data from the "telemetry" row as response + if state == common.PendingDownload || state == common.InDeployment || state == common.Failure { + if telemetrySubdoc.Payload() != nil && len(telemetrySubdoc.Payload()) > 0 { // Update state to InDeployment (3) to indicate the data is being delivered + newState := common.InDeployment + updatedTime := int(time.Now().UnixNano() / 1000000) + errorCode := 0 + errorDetails := "" + newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) + // Note: Not setting expiry here means it won't be updated in the database + + labels, err := s.DatabaseClient.GetRootDocumentLabels(mac) + if err == nil { + labels["client"] = "default" + _ = s.DatabaseClient.SetSubDocument(mac, "telemetry", newSubdoc, state, labels, fields) + // Clear expiry and error fields when transitioning to InDeployment + columnsToDelete := []string{} + if telemetrySubdoc.Expiry() != nil { + columnsToDelete = append(columnsToDelete, "expiry") + } + if telemetrySubdoc.ErrorCode() != nil { + columnsToDelete = append(columnsToDelete, "error_code") + } + if telemetrySubdoc.ErrorDetails() != nil { + columnsToDelete = append(columnsToDelete, "error_details") + } + if len(columnsToDelete) > 0 { + _ = s.DatabaseClient.DeleteSubDocumentColumns(mac, "telemetry", columnsToDelete...) + } + } + // Build multipart response from stored payload (already in msgpack format) + version := "" + if telemetrySubdoc.Version() != nil { + version = *telemetrySubdoc.Version() + } + mpart := common.Multipart{ + Bytes: telemetrySubdoc.Payload(), + Version: version, + Name: "telemetry", + State: state, + } + mparts := []common.Multipart{mpart} + fields["telemetry_version"] = version + respBytes, err := common.WriteMultipartBytes(mparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + rootVersion := util.GetRandomRootVersion() + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + w.Header().Set(common.HeaderEtag, rootVersion) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + return + } + } + + // If state == 0 or no payload, continue with existing code + } } - if rootdoc != nil { - queryParams = rootdoc.QueryParams + + // append the extra query_params if any + var rootdoc *common.RootDocument + var queryParams string + var err error + if s.SupplementaryAppendingEnabled() || s.UpstreamProfilesEnabled() { + rootdoc, err = s.GetRootDocument(mac) + if err != nil { + if !s.IsDbNotFound(err) { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + } } // partner handling @@ -68,24 +168,84 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r partnerId = "" } + if s.SupplementaryAppendingEnabled() && rootdoc != nil { + queryParams = rootdoc.QueryParams + } + urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - rbytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + xconfNotFound := false if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { if rherr.StatusCode == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) + if s.UpstreamProfilesEnabled() { + xconfNotFound = true + } else if s.DefaultEmptyProfileEnabled() { + xconfNotFound = true + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } else { + Error(w, rherr.StatusCode, rherr) return } + } + if !xconfNotFound { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + } + var profileBytes, extraProfileBytes []byte + if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { + // Get profiles from the second source + extraProfileBytes, _, err = s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) + if err != nil { + exitNow := true + var rherr common.RemoteHttpError + if errors.As(err, &rherr) { + if rherr.StatusCode == http.StatusNotFound { + exitNow = false + extraProfileBytes = nil + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if exitNow { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } } - Error(w, http.StatusInternalServerError, common.NewError(err)) + + if xconfNotFound { + baseProfileBytes = []byte(notFoundProfileText) + } + + // append profiles stored at webconfig + profileBytes, err = util.AppendProfiles(baseProfileBytes, extraProfileBytes) + if err != nil { + Error(w, http.StatusInternalServerError, err) + return + } + } else { + profileBytes = baseProfileBytes + } + + if xconfNotFound && extraProfileBytes == nil && !s.DefaultEmptyProfileEnabled() { + Error(w, http.StatusNotFound, nil) return } - mpart, err := util.TelemetryBytesToMultipart(rbytes) + if len(profileBytes) == 0 && s.DefaultEmptyProfileEnabled() { + profileBytes = []byte(notFoundProfileText) + } + + mpart, err := util.TelemetryBytesToMultipart(profileBytes) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) return @@ -93,6 +253,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r mparts := []common.Multipart{ mpart, } + fields["telemetry_version"] = mpart.Version respBytes, err := common.WriteMultipartBytes(mparts) if err != nil { @@ -102,7 +263,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } rootVersion := util.GetRandomRootVersion() - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, rootVersion) // help with unit tests diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 8e1e0a3..533626f 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -459,7 +459,7 @@ func TestSupplementaryWithExtraQueryParams(t *testing.T) { schemaVersion1 := "33554433-1.3,33554434-1.3" etag := strconv.Itoa(int(time.Now().Unix())) queryParams1 := "stormReadyWifi=true" - srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1) + srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1, "", "") bbytes, err := json.Marshal(srcDoc1) assert.NilError(t, err) @@ -619,3 +619,705 @@ func TestSupplementaryApiBadPartnerHeader(t *testing.T) { assert.NilError(t, err) assert.DeepEqual(t, coreProfile1, srcItf) } + +func TestSupplementaryAppendingFlag(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock server ==== + var ss string + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ss = r.URL.String() + w.Header().Set(common.HeaderReqUrl, r.URL.String()) + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 set up the query params ==== + bitmap1 := 32479 + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + etag := strconv.Itoa(int(time.Now().Unix())) + queryParams1 := "stormReadyWifi=true" + srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1, "", "") + bbytes, err := json.Marshal(srcDoc1) + assert.NilError(t, err) + + rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) + req, err := http.NewRequest("POST", rootdocUrl, bytes.NewReader(bbytes)) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 2 set append flag true ==== + appendEnabled := true + server.SetSupplementaryAppendingEnabled(appendEnabled) + assert.Assert(t, appendEnabled == server.SupplementaryAppendingEnabled()) + + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // TODO verify the resHeader + xReqUrl := res.Header.Get(common.HeaderReqUrl) + assert.Assert(t, strings.Contains(xReqUrl, queryParams1)) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok := mparts["telemetry"] + assert.Assert(t, ok) + + output := common.TR181Output{} + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes := []byte(output.Parameters[0].Value) + + var itf util.Dict + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok := itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok := profilesItf.([]interface{}) + assert.Assert(t, ok) + + profile1Itf := profilesJs[0] + + profile1, ok := profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok := profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok := coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + var srcItf map[string]interface{} + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 3 verify the query params ==== + assert.Assert(t, strings.Contains(ss, "&stormReadyWifi=true")) + + // ==== step 4 set append flag false ==== + appendEnabled = false + server.SetSupplementaryAppendingEnabled(appendEnabled) + assert.Assert(t, appendEnabled == server.SupplementaryAppendingEnabled()) + + req, err = http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 5 verify the query params ==== + assert.Assert(t, !strings.Contains(ss, "&stormReadyWifi=true")) +} + +func TestSupplementaryApiBadRequest(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + validPartners := []string{"comcast", "comcast-dev", "cox", "rogers", "shaw", "sky-de", "sky-italia", "sky-italia-dev", "sky-roi", "sky-roi-dev", "sky-uk", "sky-uk-dev", "videotron"} + server.SetValidPartners(validPartners) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad Request")) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "R NOT") + req.Header.Set(common.HeaderAccountID, "ERROR: ld.so: object '/usr/lib/libwayland-egl.so.0' from LD_PRE") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) +} + +func TestSupplementaryXconfTimeout(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(time.Duration(1) * time.Second) + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + server.XconfConnector.SetXconfHost(mockServer.URL) + server.XconfConnector.HttpClient.Client.Timeout = time.Duration(500) * time.Millisecond + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} + +type errorRoundTripper struct{} + +func (ert *errorRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Return a response with an error-inducing reader + return &http.Response{ + Body: io.NopCloser(&errorReader{}), + }, nil +} + +// errorReader simulates an error when reading +type errorReader struct{} + +func (er *errorReader) Read(p []byte) (int, error) { + return 0, fmt.Errorf("context deadline exceeded (Client.Timeout or context cancellation while reading body)") +} + +func TestSupplementaryXconfReadAllErr(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock client ==== + mockedClient := &http.Client{ + Transport: &errorRoundTripper{}, + } + + server.XconfConnector.HttpClient.Client = mockedClient + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} + +var ( + mockUpstreamProfileResponse1 = []byte(`[ + { + "name": "subname1", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.1.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "977e16c4" + }, + { + "name": "subname2", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.2.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "4f207ebd" + } +]`) +) + +func TestSupplementaryUpstreamProfiles(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + cxbytes, err := util.CompactJson([]byte(mockProfileResponse)) + assert.NilError(t, err) + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cxbytes) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + cubytes, err := util.CompactJson(mockUpstreamProfileResponse1) + assert.NilError(t, err) + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cubytes) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok := mparts["telemetry"] + assert.Assert(t, ok) + + output := common.TR181Output{} + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes := []byte(output.Parameters[0].Value) + + var itf util.Dict + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok := itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok := profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf := profilesJs[0] + + profile1, ok := profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok := profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok := coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + var srcItf map[string]interface{} + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 4 enable the feature flag but no query_param expect 200 with 1 mpart ==== + server.SetUpstreamProfilesEnabled(true) + defer server.SetUpstreamProfilesEnabled(false) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok = profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok = coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + srcItf = make(map[string]interface{}) + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 5 set query_param in the root_document expect 200 with 1 mpart ==== + rdoc := new(common.RootDocument) + rdoc.QueryParams = "key1=val1" + rdoc.ModelName = modelName + rdoc.PartnerId = partnerID + rdoc.FirmwareVersion = firmwareVersion + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + t.Logf("%s\n", rbytes) + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 3) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + profile2Itf := profilesJs[1] + profile2, ok := profile2Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile2["name"].(string), "subname1") + + profile3Itf := profilesJs[2] + profile3, ok := profile3Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile3["name"].(string), "subname2") +} + +func TestSupplementaryUpstreamProfilesNotFoundNotDefaultEmptyProfile(t *testing.T) { + log.SetOutput(io.Discard) + + conf := sc.Config + ss := "webconfig.upstream_profiles_enabled = true" + conf = conf.AddConfig(ss, nil) + tsc := *sc + tsc.Config = conf + + server := NewWebconfigServer(&tsc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + // set up root_document table + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true", "", "") + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + getRdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rdoc.Compare(getRdoc) == 0) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} + +func TestSupplementaryUpstreamProfilesNotFoundDefaultEmptyProfile(t *testing.T) { + log.SetOutput(io.Discard) + + conf := sc.Config + ss := "webconfig.upstream_profiles_enabled=true\nwebconfig.default_empty_profile_enabled=true" + conf = conf.AddConfig(ss, nil) + tsc := *sc + tsc.Config = conf + + server := NewWebconfigServer(&tsc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + // set up root_document table + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true", "", "") + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + getRdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rdoc.Compare(getRdoc) == 0) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusOK) +} + +func TestSupplementaryDefaultEmptyProfile(t *testing.T) { + log.SetOutput(io.Discard) + + conf := sc.Config + ss := "webconfig.default_empty_profile_enabled=true" + conf = conf.AddConfig(ss, nil) + tsc := *sc + tsc.Config = conf + + server := NewWebconfigServer(&tsc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + // set up root_document table + rdoc := common.NewRootDocument(32479, firmwareVersion, modelName, partnerID, "", "12345", "stormReadyWifi=true", "", "") + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + getRdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rdoc.Compare(getRdoc) == 0) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusOK) +} diff --git a/http/supplementary_precook_test.go b/http/supplementary_precook_test.go new file mode 100644 index 0000000..17d0a73 --- /dev/null +++ b/http/supplementary_precook_test.go @@ -0,0 +1,711 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package http + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestSupplementaryPrecookConfigFlags(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + + // Test default values + assert.Equal(t, server.SupplementaryPrecookEnabled(), false) + assert.Equal(t, server.SupplementaryPrecookStateTTLDays(), 7) + + // Test setter for SupplementaryPrecookEnabled + server.SetSupplementaryPrecookEnabled(true) + assert.Equal(t, server.SupplementaryPrecookEnabled(), true) + + server.SetSupplementaryPrecookEnabled(false) + assert.Equal(t, server.SupplementaryPrecookEnabled(), false) + + // Test setter for SupplementaryPrecookStateTTLDays + server.SetSupplementaryPrecookStateTTLDays(14) + assert.Equal(t, server.SupplementaryPrecookStateTTLDays(), 14) + + server.SetSupplementaryPrecookStateTTLDays(0) + assert.Equal(t, server.SupplementaryPrecookStateTTLDays(), 0) +} + +func TestSupplementaryPrecookStateDeployed(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup telemetry subdoc with state=Deployed ==== + telemetryBytes := common.RandomBytes(100, 150) + telemetryVersion := util.GetMurmur3Hash(telemetryBytes) + telemetryState := common.Deployed + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(telemetryBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler returns 304 ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) +} + +func TestSupplementaryPrecookStatePendingDownload(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload from mock profile response ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.PendingDownload + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler returns cached data ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) +} + +func TestSupplementaryPrecookStateInDeployment(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload from mock profile response ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.InDeployment + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler returns cached data ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) +} + +func TestSupplementaryPrecookStateFailure(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload from mock profile response ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.Failure + telemetryUpdatedTime := int(time.Now().UnixMilli()) + errorCode := 500 + errorDetails := "test error" + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, &errorCode, &errorDetails) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler returns cached data ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) +} + +func TestSupplementaryPrecookDisabled(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Ensure supplementary precook feature is disabled + server.SetSupplementaryPrecookEnabled(false) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup telemetry subdoc with state=Deployed ==== + telemetryBytes := common.RandomBytes(100, 150) + telemetryVersion := util.GetMurmur3Hash(telemetryBytes) + telemetryState := common.Deployed + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(telemetryBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify that MultipartSupplementaryHandler uses normal flow (not 304) ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + + // Should not return 304, should fetch from mock server + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) +} + +func TestSupplementaryPrecookStateUpdate(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload with state=PendingDownload ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.PendingDownload + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify initial state is PendingDownload ==== + subdoc, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.PendingDownload) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + + // ==== verify state is updated to InDeployment ==== + subdoc, err = server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.InDeployment) +} + +func TestSupplementaryPrecookStateUpdateFromFailure(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload with state=Failure ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.Failure + telemetryUpdatedTime := int(time.Now().UnixMilli()) + errorCode := 500 + errorDetails := "test error" + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, &errorCode, &errorDetails) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify initial state is Failure ==== + subdoc, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Failure) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + + // ==== verify state is updated to InDeployment ==== + subdoc, err = server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.InDeployment) +} + +func TestSupplementaryPrecookStateNotUpdatedWhenDeployed(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create telemetry payload with state=Deployed ==== + telemetryBytes := common.RandomBytes(100, 150) + telemetryVersion := util.GetMurmur3Hash(telemetryBytes) + telemetryState := common.Deployed + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(telemetryBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify initial state is Deployed ==== + subdoc, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // ==== verify state is still Deployed (not updated) ==== + subdoc, err = server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) +} + +func TestSupplementaryPrecookStateDeployedWithExpiredTTL(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup mock xconf server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup telemetry subdoc with state=Deployed but expired TTL ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.Deployed + telemetryUpdatedTime := int(time.Now().UnixMilli()) + + // Set expiry to a past time (1 hour ago in milliseconds) + expiredTime := int(time.Now().Add(-1 * time.Hour).UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + telemetrySubdoc.SetExpiry(&expiredTime) + + fields := make(log.Fields) + err := server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // ==== verify the subdoc has state=Deployed and expired expiry ==== + subdoc, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdoc.State(), common.Deployed) + assert.Assert(t, subdoc.Expiry() != nil) + assert.Assert(t, *subdoc.Expiry() < int(time.Now().UnixMilli())) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + + // ==== expectation: should NOT return 304, should fetch from xconf ==== + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart from xconf + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) +} + +func TestSupplementaryPrecookExpiryDeletedOnStateTransition(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create root document for labels ==== + queryParams := "model=TG1682G&partner=comcast" + fwVersion := "TG1682_3.14p9s6_PROD_sey" + modelName := "TG1682G" + partnerId := "comcast" + rootDoc := common.NewRootDocument(0, fwVersion, modelName, partnerId, "", "", queryParams, "", "") + err := server.SetRootDocument(cpeMac, rootDoc) + assert.NilError(t, err) + + // ==== create telemetry payload from mock profile response with state=PendingDownload and expiry set ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.PendingDownload + telemetryUpdatedTime := int(time.Now().UnixMilli()) + futureExpiry := int(time.Now().Add(24 * time.Hour).UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, nil, nil) + telemetrySubdoc.SetExpiry(&futureExpiry) + fields := make(log.Fields) + err = server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // Verify expiry is set before the request + subdocBefore, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Assert(t, subdocBefore.Expiry() != nil) + assert.Equal(t, *subdocBefore.Expiry(), futureExpiry) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) + + // ==== verify state was updated to InDeployment ==== + subdocAfter, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdocAfter.State(), common.InDeployment) + + // ==== verify expiry was deleted ==== + assert.Assert(t, subdocAfter.Expiry() == nil) +} + +func TestSupplementaryPrecookErrorFieldsDeletedOnStateTransition(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // Enable supplementary precook feature + server.SetSupplementaryPrecookEnabled(true) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== create root document for labels ==== + queryParams := "model=TG1682G&partner=comcast" + fwVersion := "TG1682_3.14p9s6_PROD_sey" + modelName := "TG1682G" + partnerId := "comcast" + rootDoc := common.NewRootDocument(0, fwVersion, modelName, partnerId, "", "", queryParams, "", "") + err := server.SetRootDocument(cpeMac, rootDoc) + assert.NilError(t, err) + + // ==== create telemetry payload with state=Failure (state=4) and error fields set ==== + mockProfileBytes := []byte(mockProfileResponse) + telemetryVersion := util.GetMurmur3Hash(mockProfileBytes) + telemetryState := common.Failure + telemetryUpdatedTime := int(time.Now().UnixMilli()) + errorCode := 204 + errorDetails := "failed_retrying:Error unsupported namespace" + futureExpiry := int(time.Now().Add(24 * time.Hour).UnixMilli()) + + telemetrySubdoc := common.NewSubDocument(mockProfileBytes, &telemetryVersion, &telemetryState, &telemetryUpdatedTime, &errorCode, &errorDetails) + telemetrySubdoc.SetExpiry(&futureExpiry) + fields := make(log.Fields) + err = server.SetSubDocument(cpeMac, "telemetry", telemetrySubdoc, fields) + assert.NilError(t, err) + + // Verify expiry and error fields are set before the request + subdocBefore, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Assert(t, subdocBefore.Expiry() != nil) + assert.Equal(t, *subdocBefore.Expiry(), futureExpiry) + assert.Assert(t, subdocBefore.ErrorCode() != nil) + assert.Equal(t, *subdocBefore.ErrorCode(), errorCode) + assert.Assert(t, subdocBefore.ErrorDetails() != nil) + assert.Equal(t, *subdocBefore.ErrorDetails(), errorDetails) + + // ==== make request to MultipartSupplementaryHandler ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // Verify response is multipart + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + _, ok := mparts["telemetry"] + assert.Assert(t, ok) + + // ==== verify state was updated to InDeployment ==== + subdocAfter, err := server.GetSubDocument(cpeMac, "telemetry") + assert.NilError(t, err) + assert.Equal(t, *subdocAfter.State(), common.InDeployment) + + // ==== verify expiry and error fields were deleted ==== + assert.Assert(t, subdocAfter.Expiry() == nil) + // Note: Cassandra returns default values (0, "") for error_code and error_details after DELETE + assert.Assert(t, subdocAfter.ErrorCode() != nil) + assert.Equal(t, *subdocAfter.ErrorCode(), 0) + assert.Assert(t, subdocAfter.ErrorDetails() != nil) + assert.Equal(t, *subdocAfter.ErrorDetails(), "") +} diff --git a/http/supported_groups_handler.go b/http/supported_groups_handler.go index 6eac50b..90a1a27 100644 --- a/http/supported_groups_handler.go +++ b/http/supported_groups_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,9 +23,9 @@ import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) // The supported doc header in GET /config is parsed and stored as a bitmap @@ -54,7 +54,7 @@ func (s *WebconfigServer) GetSupportedGroupsHandler(w http.ResponseWriter, r *ht outdata := common.SupportedGroupsData{ Bitmap: rdoc.Bitmap, - Groups: util.GetSupportedMap(rdoc.Bitmap), + Groups: common.GetSupportedMap(rdoc.Bitmap), } WriteOkResponse(w, outdata) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index fa5a641..9256696 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -46,6 +46,7 @@ func TestSupportedGroupsHandler(t *testing.T) { res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -61,43 +62,26 @@ func TestSupportedGroupsHandler(t *testing.T) { rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - "wanfailover": false, - "cellularconfig": false, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -110,7 +94,7 @@ func TestSupportedGroupsHandler(t *testing.T) { var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) // ==== step 2 add lan data ==== subdocId := "lan" @@ -122,16 +106,17 @@ func TestSupportedGroupsHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -165,26 +150,26 @@ func TestSupportedGroupsHandler(t *testing.T) { err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) // ==== step 5 setup supported docs for fw version 2 ===== sids := strings.Split(rdkSupportedDocsHeaderStr, ",") newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" - group1Bitmap, err := util.BitarrayToBitmap(newGroup1Bitarray) + group1Bitmap, err := common.BitarrayToBitmap(newGroup1Bitarray) assert.NilError(t, err) sids[0] = fmt.Sprintf("%v", group1Bitmap) - expectedEnabled["wan"] = false - expectedEnabled["macbinding"] = false - expectedEnabled["hotspot"] = true - expectedEnabled["bridge"] = true + supportedSubdocMap["wan"] = false + supportedSubdocMap["macbinding"] = false + supportedSubdocMap["hotspot"] = true + supportedSubdocMap["bridge"] = true newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" - group2Bitmap, err := util.BitarrayToBitmap(newGroup2Bitarray) + group2Bitmap, err := common.BitarrayToBitmap(newGroup2Bitarray) assert.NilError(t, err) sids[1] = fmt.Sprintf("%v", group2Bitmap) - expectedEnabled["privatessid"] = false - expectedEnabled["radio"] = true + supportedSubdocMap["privatessid"] = false + supportedSubdocMap["radio"] = true rdkSupportedDocsHeaderStr = strings.Join(sids, ",") @@ -213,7 +198,7 @@ func TestSupportedGroupsHandler(t *testing.T) { err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { @@ -230,6 +215,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -245,43 +231,27 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": false, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": true, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telcovoice", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -294,7 +264,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testing.T) { @@ -311,6 +281,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -326,43 +297,33 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwfailover": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "gwfailover", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -375,5 +336,5 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 165aac6..cae0e4a 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,15 +22,16 @@ import ( "fmt" "net/http" + "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" owcommon "github.com/rdkcentral/webconfig/common" - "github.com/go-akka/configuration" log "github.com/sirupsen/logrus" ) const ( - upstreamHostDefault = "http://localhost:1234" - upstreamUrlTemplateDefault = "/api/v1/device/%v/upstream" + defaultUpstreamHost = "http://localhost:12348" + defaultUpstreamUrlTemplate = "%s/%s" + defaultProfileUrlTemplate = "%s/%s/%s" ) type UpstreamConnector struct { @@ -38,20 +39,21 @@ type UpstreamConnector struct { host string serviceName string upstreamUrlTemplate string + profileUrlTemplate string } func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *UpstreamConnector { serviceName := "upstream" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, upstreamHostDefault) - confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) - upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) + host := conf.GetString("webconfig.upstream.host", defaultUpstreamHost) + upstreamUrlTemplate := conf.GetString("webconfig.upstream.url_template", defaultUpstreamUrlTemplate) + profileUrlTemplate := conf.GetString("webconfig.upstream.profile_url_template", defaultProfileUrlTemplate) return &UpstreamConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, + profileUrlTemplate: profileUrlTemplate, } } @@ -68,7 +70,7 @@ func (c *UpstreamConnector) ServiceName() string { } func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { - url := c.UpstreamHost() + fmt.Sprintf(c.upstreamUrlTemplate, mac) + url := fmt.Sprintf(c.upstreamUrlTemplate, c.UpstreamHost(), mac) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) @@ -90,3 +92,27 @@ func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes } return rbytes, header, nil } + +func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { + url := fmt.Sprintf(c.profileUrlTemplate, c.UpstreamHost(), mac, queryParams) + + if itf, ok := fields["audit_id"]; ok { + auditId := itf.(string) + if len(auditId) > 0 { + header.Set(common.HeaderAuditid, auditId) + } + } + + if itf, ok := fields["app_name"]; ok { + appName := itf.(string) + if len(appName) > 0 { + header.Set(common.HeaderSourceAppName, appName) + } + } + + rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) + if err != nil { + return rbytes, header, owcommon.NewError(err) + } + return rbytes, header, nil +} diff --git a/http/upstream_connector_test.go b/http/upstream_connector_test.go index e9dfccb..f96d986 100644 --- a/http/upstream_connector_test.go +++ b/http/upstream_connector_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,6 +22,7 @@ import ( "net/http/httptest" "testing" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "gotest.tools/assert" @@ -31,7 +32,7 @@ func TestUpstreamConnector(t *testing.T) { server := NewWebconfigServer(sc, true) // setup upstream mock server - mockedUpstreamResponse := util.RandomBytes(100, 150) + mockedUpstreamResponse := common.RandomBytes(100, 150) upstreamMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) diff --git a/http/upstream_test.go b/http/upstream_test.go index f587467..8259c09 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strconv" "testing" @@ -41,10 +42,10 @@ func TestUpstream(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidV13Bytes := util.RandomBytes(m, n) - privatessidV14Bytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) + privatessidV14Bytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidV13Bytes, "lan": lanBytes, @@ -64,7 +65,7 @@ func TestUpstream(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -103,7 +104,7 @@ func TestUpstream(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -142,7 +143,7 @@ func TestUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -151,7 +152,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -165,7 +166,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -174,7 +175,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -188,7 +189,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -197,7 +198,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -244,7 +245,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -263,15 +264,16 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -317,7 +319,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -337,9 +339,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() m, n := 50, 100 - lanBytes := util.RandomBytes(m, n) - wanBytes := util.RandomBytes(m, n) - privatessidBytes := util.RandomBytes(m, n) + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidBytes := common.RandomBytes(m, n) srcbytesMap := map[string][]byte{ "privatessid": privatessidBytes, "lan": lanBytes, @@ -359,7 +361,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -370,7 +372,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { respBytes := reqBytes // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, etag) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -410,7 +412,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -419,7 +421,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -433,7 +435,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -442,7 +444,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -456,7 +458,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -465,7 +467,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -506,7 +508,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -525,15 +527,16 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -545,3 +548,1253 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { assert.Equal(t, state, common.Deployed) } } + +func TestUpstreamUpdatedTime(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) + privatessidV14Bytes := common.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + mparts, err := util.ParseMultipartAsList(r.Header, reqBytes) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // modify the payload + newMparts := []common.Multipart{} + for _, mpart := range mparts { + if mpart.Name == "privatessid" { + version := util.GetMurmur3Hash(privatessidV14Bytes) + newMpart := common.Multipart{ + Name: mpart.Name, + Version: version, + Bytes: privatessidV14Bytes, + } + newMparts = append(newMparts, newMpart) + } else { + newMparts = append(newMparts, mpart) + } + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + srcbytesMap["privatessid"] = privatessidV14Bytes + expectedStateMap := map[string]int{ + "privatessid": common.InDeployment, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + + switch subdocId { + case "privatessid": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) != subdocUpdatedTimeMap[subdocId]) + case "lan", "wan": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } + } +} + +func TestUpstreamResponseSkipDbUpdate(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := common.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} + +func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidV13Bytes := common.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := common.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set(common.HeaderContentType, common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + assert.Assert(t, len(updatedSubdocIds) == 0) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} + +func TestUpstreamBackfill(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + wanBytes := common.RandomBytes(m, n) + privatessidBytes := common.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidBytes, + "lan": lanBytes, + "wan": wanBytes, + } + + var mockedRespBytes []byte + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mockedRespBytes == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + for k := range r.Header { + if k == "Content-Length" { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + w.Write(mockedRespBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderIfNoneMatch, "NONE-POST") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + mockedRespBytes = slices.Clone(rbytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 handle a condition when no subdocs in db ==== + err = server.DeleteDocument(cpeMac) + assert.NilError(t, err) + err = server.DeleteRootDocument(cpeMac) + assert.NilError(t, err) + + // ==== step 9 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotModified) + res.Body.Close() + + // ==== step 10 verify all states, versions and payloads of all subdocs are backfilled ==== + document, err := server.GetDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, document.Length(), 3) + for _, sd := range document.Items() { + assert.Assert(t, sd.GetState() > 0) + assert.Assert(t, len(sd.GetVersion()) > 0) + assert.Assert(t, len(sd.Payload()) > 0) + } +} + +func TestUpstreamNoBitmapHeader(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 GET /config to create root document meta ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + + supportedDocs := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion := "CGM4331COM_4.11p7s1_PROD_sey" + modelName := "CGM4331COM" + partner := "comcast" + schemaVersion := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partner) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 2 POST group lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := common.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 3 POST group wan ==== + subdocId = "wan" + wanBytes := common.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 4 GET /config ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partner) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== step 5 GET /config but with header changes with mock ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.WriteHeader(http.StatusOK) + if rbytes, err := io.ReadAll(r.Body); err == nil { + _, err := w.Write(rbytes) + assert.NilError(t, err) + } + })) + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + server.SetUpstreamEnabled(true) + server.SetFilterOutputByBitmapEnabled(true) + + // ==== step 6 GET /config with no supported-docs/bitmap headers ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "0") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) +} diff --git a/http/validator.go b/http/validator.go index 10683ec..bde64c6 100644 --- a/http/validator.go +++ b/http/validator.go @@ -14,16 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -56,8 +56,8 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") @@ -71,3 +71,40 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid } return mac, subdocId, bodyBytes, fields, nil } + +func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request, validateContent bool) (string, []byte, log.Fields, error) { + var fields log.Fields + + // check mac + params := mux.Vars(r) + refId := params["ref"] + + // check for safety, but it should not fail + xw, ok := w.(*XResponseWriter) + if !ok { + err := *common.NewHttp500Error("responsewriter cast error") + return refId, nil, nil, common.NewError(err) + } + fields = xw.Audit() + + if !validateContent { + return refId, nil, fields, nil + } + + // ==== validate content ==== + // check content-type + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { + // TODO (1) if we should validate this header + // (2) if unexpected, return 400 or 415 + err := *common.NewHttp400Error("content-type not msgpack") + return refId, nil, nil, common.NewError(err) + } + + bodyBytes := xw.BodyBytes() + if len(bodyBytes) == 0 { + err := *common.NewHttp400Error("empty body") + return refId, nil, nil, common.NewError(err) + } + return refId, bodyBytes, fields, nil +} diff --git a/http/validator_test.go b/http/validator_test.go index f368804..0b2687d 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,11 +22,12 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -46,19 +47,19 @@ func TestValidatorDisabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -75,7 +76,7 @@ func TestValidatorDisabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -86,19 +87,19 @@ func TestValidatorDisabled(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -113,7 +114,7 @@ func TestValidatorDisabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -139,40 +140,40 @@ func TestValidatorEnabled(t *testing.T) { server.SetValidateMacEnabled(true) url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) // post without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) // get without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -188,7 +189,7 @@ func TestValidatorEnabled(t *testing.T) { req, err = http.NewRequest("GET", rootdocUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -199,7 +200,7 @@ func TestValidatorEnabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -211,29 +212,29 @@ func TestValidatorEnabled(t *testing.T) { // delete with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) // delete without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -248,7 +249,7 @@ func TestValidatorEnabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -270,21 +271,21 @@ func TestValidatorWithLowerCase(t *testing.T) { // post subdocId := "lan" lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) - lanBytes := util.RandomBytes(100, 150) + lanBytes := common.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -299,21 +300,21 @@ func TestValidatorWithLowerCase(t *testing.T) { // post subdocId = "wan" wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) - wanBytes := util.RandomBytes(100, 150) + wanBytes := common.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -331,7 +332,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 932b8bf..0309cea 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -14,30 +14,36 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "context" "crypto/tls" + "encoding/base64" "encoding/json" + "errors" "fmt" - "io/ioutil" + "io" + "maps" "net/http" "os" "strings" "time" + "github.com/IBM/sarama" + "github.com/go-akka/configuration" + "github.com/google/uuid" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" - owcommon "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" + "github.com/rdkcentral/webconfig/tracing" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" + sdktrace "go.opentelemetry.io/otel/sdk/trace" ) // TODO enum, probably no need @@ -48,15 +54,17 @@ const ( ) const ( - MetricsEnabledDefault = true - FactoryResetEnabledDefault = false - serverApiTokenAuthEnabledDefault = false - deviceApiTokenAuthEnabledDefault = true - tokenApiEnabledDefault = false - activeDriverDefault = "cassandra" - defaultJwksEnabled = false - defaultTraceparentParentID = "0000000000000001" - defaultTracestateVendorID = "webconfig" + MetricsEnabledDefault = true + FactoryResetEnabledDefault = false + serverApiTokenAuthEnabledDefault = false + deviceApiTokenAuthEnabledDefault = true + tokenApiEnabledDefault = false + activeDriverDefault = "cassandra" + defaultJwksEnabled = false + defaultTraceparentParentID = "0000000000000001" + defaultTracestateVendorID = "webconfig" + defaultSupplementaryAppendingEnabled = true + authPrefixLength = 60 ) var ( @@ -73,6 +81,7 @@ var ( "privatessid", "homessid", } + codec *security.AesCodec ) type WebconfigServer struct { @@ -85,21 +94,33 @@ type WebconfigServer struct { *XconfConnector *MqttConnector *UpstreamConnector - tlsConfig *tls.Config - notLoggedHeaders []string - metricsEnabled bool - factoryResetEnabled bool - serverApiTokenAuthEnabled bool - deviceApiTokenAuthEnabled bool - tokenApiEnabled bool - kafkaEnabled bool - upstreamEnabled bool - appName string - validateMacEnabled bool - validPartners []string - jwksEnabled bool - traceparentParentID string - tracestateVendorID string + sarama.AsyncProducer + *tracing.XpcTracer + tlsConfig *tls.Config + notLoggedHeaders []string + metricsEnabled bool + factoryResetEnabled bool + serverApiTokenAuthEnabled bool + deviceApiTokenAuthEnabled bool + tokenApiEnabled bool + kafkaEnabled bool + upstreamEnabled bool + appName string + validateMacEnabled bool + validPartners []string + jwksEnabled bool + traceparentParentID string + tracestateVendorID string + supplementaryAppendingEnabled bool + kafkaProducerEnabled bool + kafkaProducerTopic string + upstreamProfilesEnabled bool + queryParamsValidationEnabled bool + minTrust int + validSubdocIdMap map[string]int + filterOutputByBitmapEnabled bool + defaultEmptyProfileEnabled bool + bitmapFilterExemptSubdocIds []string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -150,10 +171,6 @@ func GetTestDatabaseClient(sc *common.ServerConfig) db.DatabaseClient { err = fmt.Errorf("Unsupported database.active_driver %v is configured", activeDriver) panic(err) } - err = tdbclient.SetUp() - if err != nil { - panic(err) - } return tdbclient } @@ -188,12 +205,14 @@ func GetDatabaseClient(sc *common.ServerConfig) db.DatabaseClient { func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer { conf := sc.Config var dbclient db.DatabaseClient + var tokenManager *security.TokenManager // setup up database client if testOnly { dbclient = GetTestDatabaseClient(sc) } else { dbclient = GetDatabaseClient(sc) + tokenManager = security.NewTokenManager(conf) } // setup jwks manager @@ -240,39 +259,100 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validPartners = append(validPartners, strings.ToLower(p)) } - traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) - tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + xpcTracer := tracing.NewXpcTracer(conf) + + supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) + + // kafka producer + var kafkaProducer sarama.AsyncProducer + kafkaProducerEnabled := conf.GetBoolean("webconfig.kafka_producer.enabled") + var kafkaProducerTopic string + if kafkaProducerEnabled { + brokersStr := conf.GetString("webconfig.kafka_producer.brokers") + if len(brokersStr) == 0 { + panic(fmt.Errorf("webconfig.kafka_producer.brokers is empty")) + } + brokers := strings.Split(brokersStr, ",") + kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") + + saramaConfig := sarama.NewConfig() + saramaConfig.Producer.Return.Errors = true + + // Load TLS configuration for producer + tlsConfig, err := common.LoadKafkaTLSConfig(conf, "webconfig.kafka_producer") + if err != nil { + panic(fmt.Errorf("failed to load TLS configuration for Kafka producer: %v", err)) + } + if tlsConfig != nil { + saramaConfig.Net.TLS.Enable = true + saramaConfig.Net.TLS.Config = tlsConfig + } + + kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) + if err != nil { + panic(err) + } + } + + upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") + queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + minTrust := int(conf.GetInt32("webconfig.min_trust")) + validSubdocIds := conf.GetStringList("webconfig.valid_subdoc_ids") + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + for _, x := range validSubdocIds { + validSubdocIdMap[x] = 1 + } - return &WebconfigServer{ + filterOutputByBitmapEnabled := conf.GetBoolean("webconfig.filter_output_by_bitmap_enabled") + defaultEmptyProfileEnabled := conf.GetBoolean("webconfig.default_empty_profile_enabled") + bitmapFilterExemptSubdocIds := conf.GetStringList("webconfig.bitmap_filter_exempt_subdoc_ids") + + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, WriteTimeout: time.Duration(conf.GetInt32("webconfig.server.write_timeout_in_secs", 3)) * time.Second, }, - DatabaseClient: dbclient, - TokenManager: security.NewTokenManager(conf), - JwksManager: jwksManager, - ServerConfig: sc, - WebpaConnector: NewWebpaConnector(conf, tlsConfig), - XconfConnector: NewXconfConnector(conf, tlsConfig), - MqttConnector: NewMqttConnector(conf, tlsConfig), - UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), - tlsConfig: tlsConfig, - notLoggedHeaders: notLoggedHeaders, - metricsEnabled: metricsEnabled, - factoryResetEnabled: factoryResetEnabled, - serverApiTokenAuthEnabled: serverApiTokenAuthEnabled, - deviceApiTokenAuthEnabled: deviceApiTokenAuthEnabled, - tokenApiEnabled: tokenApiEnabled, - kafkaEnabled: kafkaEnabled, - upstreamEnabled: upstreamEnabled, - appName: appName, - validateMacEnabled: validateMacEnabled, - validPartners: validPartners, - jwksEnabled: jwksEnabled, - traceparentParentID: traceparentParentID, - tracestateVendorID: tracestateVendorID, + DatabaseClient: dbclient, + TokenManager: tokenManager, + JwksManager: jwksManager, + ServerConfig: sc, + WebpaConnector: NewWebpaConnector(conf, tlsConfig), + XconfConnector: NewXconfConnector(conf, tlsConfig), + MqttConnector: NewMqttConnector(conf, tlsConfig), + UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), + AsyncProducer: kafkaProducer, + tlsConfig: tlsConfig, + notLoggedHeaders: notLoggedHeaders, + metricsEnabled: metricsEnabled, + factoryResetEnabled: factoryResetEnabled, + serverApiTokenAuthEnabled: serverApiTokenAuthEnabled, + deviceApiTokenAuthEnabled: deviceApiTokenAuthEnabled, + tokenApiEnabled: tokenApiEnabled, + kafkaEnabled: kafkaEnabled, + upstreamEnabled: upstreamEnabled, + appName: appName, + validateMacEnabled: validateMacEnabled, + validPartners: validPartners, + jwksEnabled: jwksEnabled, + supplementaryAppendingEnabled: supplementaryAppendingEnabled, + kafkaProducerEnabled: kafkaProducerEnabled, + kafkaProducerTopic: kafkaProducerTopic, + upstreamProfilesEnabled: upstreamProfilesEnabled, + queryParamsValidationEnabled: queryParamsValidationEnabled, + minTrust: minTrust, + validSubdocIdMap: validSubdocIdMap, + XpcTracer: xpcTracer, + filterOutputByBitmapEnabled: filterOutputByBitmapEnabled, + defaultEmptyProfileEnabled: defaultEmptyProfileEnabled, + bitmapFilterExemptSubdocIds: bitmapFilterExemptSubdocIds, } + + return ws +} + +func (s *WebconfigServer) Stop() { + s.StopXpcTracer() } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -296,7 +376,7 @@ func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { if r.Method == "POST" { if r.Body != nil { - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { xw.SetBodyBytes(rbytes) } } @@ -323,32 +403,56 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() - if len(token) > 0 { - params := mux.Vars(r) - mac, ok := params["mac"] - if !ok || len(mac) != 12 { - Error(xw, http.StatusForbidden, nil) + fields := xw.Audit() + + params := mux.Vars(r) + mac, ok := params["mac"] + if !ok { + Error(xw, http.StatusForbidden, nil) + return + } + mac = strings.ToUpper(mac) + if s.ValidateMacEnabled() { + if !util.ValidateMac(mac) { + err := *common.NewHttp400Error("invalid mac") + Error(w, http.StatusBadRequest, common.NewError(err)) return } + } - if ok, partnerId, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + authorization := r.Header.Get("Authorization") + var tokenErr error + if len(token) > 0 { + if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true + fields["src_partner"] = partnerId + fields["trust"] = trust + if err := s.ValidatePartner(partnerId); err != nil { - fields := xw.Audit() - fields["src_partner"] = partnerId + // isValid = false partnerId = "unknown" + tokenErr = common.NewError(err) + } + if trust < s.MinTrust() { + isValid = false + tokenErr = common.NewError(common.ErrLowTrust) } xw.SetPartnerId(partnerId) } else { - xw.LogDebug(r, "token", fmt.Sprintf("CpeMiddleware() VerifyCpeToken()=false, err=%v", err)) + tokenErr = common.NewError(err) } } else { - xw.LogDebug(r, "token", "CpeMiddleware() error no token") + tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) + } + + if tokenErr != nil { + fields["error"] = tokenErr } if isValid { next.ServeHTTP(xw, r) } else { + s.LogToken(xw, authorization, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -414,7 +518,7 @@ func (s *WebconfigServer) TestingCpeMiddleware(next http.Handler) http.Handler { return } - if ok, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, _, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true } } @@ -556,6 +660,86 @@ func (s *WebconfigServer) SetTracestateVendorID(x string) { s.tracestateVendorID = x } +func (s *WebconfigServer) SupplementaryAppendingEnabled() bool { + return s.supplementaryAppendingEnabled +} + +func (s *WebconfigServer) SetSupplementaryAppendingEnabled(enabled bool) { + s.supplementaryAppendingEnabled = enabled +} + +func (s *WebconfigServer) KafkaProducerEnabled() bool { + return s.kafkaProducerEnabled +} + +func (s *WebconfigServer) SetKafkaProducerEnabled(enabled bool) { + s.kafkaProducerEnabled = enabled +} + +func (s *WebconfigServer) KafkaProducerTopic() string { + return s.kafkaProducerTopic +} + +func (s *WebconfigServer) SetKafkaProducerTopic(x string) { + s.kafkaProducerTopic = x +} + +func (s *WebconfigServer) UpstreamProfilesEnabled() bool { + return s.upstreamProfilesEnabled +} + +func (s *WebconfigServer) SetUpstreamProfilesEnabled(enabled bool) { + s.upstreamProfilesEnabled = enabled +} + +func (s *WebconfigServer) QueryParamsValidationEnabled() bool { + return s.queryParamsValidationEnabled +} + +func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { + s.queryParamsValidationEnabled = enabled +} + +func (s *WebconfigServer) MinTrust() int { + return s.minTrust +} + +func (s *WebconfigServer) SetMinTrust(trust int) { + s.minTrust = trust +} + +func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { + return s.validSubdocIdMap +} + +func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { + s.validSubdocIdMap = x +} + +func (s *WebconfigServer) FilterOutputByBitmapEnabled() bool { + return s.filterOutputByBitmapEnabled +} + +func (s *WebconfigServer) SetFilterOutputByBitmapEnabled(enabled bool) { + s.filterOutputByBitmapEnabled = enabled +} + +func (s *WebconfigServer) DefaultEmptyProfileEnabled() bool { + return s.defaultEmptyProfileEnabled +} + +func (s *WebconfigServer) SetDefaultEmptyProfileEnabled(enabled bool) { + s.defaultEmptyProfileEnabled = enabled +} + +func (s *WebconfigServer) BitmapFilterExemptSubdocIds() []string { + return s.bitmapFilterExemptSubdocIds +} + +func (s *WebconfigServer) SetBitmapFilterExemptSubdocIds(x []string) { + s.bitmapFilterExemptSubdocIds = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { @@ -568,12 +752,12 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return nil } } - return fmt.Errorf("invalid partner") + return fmt.Errorf("invalid partner %s", partner) } -func (c *WebconfigServer) Poke(cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) - transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) + transactionId, err := c.Patch(rHeader, cpeMac, token, []byte(body), fields) if err != nil { return "", common.NewError(err) } @@ -601,7 +785,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques token = elements[1] } - var xmTraceId, traceId, outTraceparent, outTracestate string + var xmTraceId string // extract moneytrace from the header tracePart := strings.Split(r.Header.Get("X-Moneytrace"), ";")[0] @@ -611,37 +795,36 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } } - // extract traceparent from the header - traceparent := r.Header.Get(owcommon.HeaderTraceparent) - if len(traceparent) == 55 { - traceId = traceparent[3:35] - outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] - } - - // extrac tracestate from the header - tracestate := r.Header.Get(common.HeaderTracestate) - if len(tracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) - } - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { auditId = util.GetAuditId() } + + // traceparent handling for E2E tracing + xpcTrace := tracing.NewXpcTrace(s.XpcTracer, r) + traceId := xpcTrace.TraceID + if len(traceId) == 0 { + traceId = xmTraceId + } + headerMap := util.HeaderToMap(header) fields := log.Fields{ - "path": r.URL.String(), - "method": r.Method, - "audit_id": auditId, - "remote_ip": remoteIp, - "host_name": host, - "header": headerMap, - "logger": "request", - "trace_id": traceId, - "app_name": s.AppName(), - "out_traceparent": outTraceparent, - "out_tracestate": outTracestate, + "path": r.URL.String(), + "method": r.Method, + "audit_id": auditId, + "remote_ip": remoteIp, + "host_name": host, + "header": headerMap, + "logger": "request", + "trace_id": traceId, + "app_name": s.AppName(), + "traceparent": xpcTrace.ReqTraceparent, + "tracestate": xpcTrace.ReqTracestate, + "out_traceparent": xpcTrace.OutTraceparent, + "out_tracestate": xpcTrace.OutTracestate, + "req_moracide_tag": xpcTrace.ReqMoracideTag, + "xpc_trace": xpcTrace, } userAgent := r.UserAgent() @@ -668,7 +851,6 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques case "cpe": mac := params["gid"] mac = strings.ToUpper(mac) - fields["cpemac"] = mac fields["cpe_mac"] = mac case "configset": csid := params["gid"] @@ -677,7 +859,6 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } if mac, ok := params["mac"]; ok { mac = strings.ToUpper(mac) - fields["cpemac"] = mac fields["cpe_mac"] = mac } @@ -685,10 +866,10 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if r.Method == "POST" { if r.Body != nil { - bbytes, err := ioutil.ReadAll(r.Body) + bbytes, err := io.ReadAll(r.Body) if err != nil { fields["error"] = err - log.WithFields(fields).Error("request starts") + log.WithFields(fields).Error("Request started") return xwriter } xwriter.SetBodyBytes(bbytes) @@ -697,9 +878,10 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if userAgent != "mget" { tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("request starts") + log.WithFields(tfields).Info("Request started") } + xwriter.LogDebug(r, "tracing", fmt.Sprintf("Trace final out_traceparent %s out_traceState %s", xpcTrace.OutTraceparent, xpcTrace.OutTracestate)) return xwriter } @@ -722,9 +904,22 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["response"] = mpdict } } else { - res_itf, res_text := GetResponseLogObjs(rbytes) - fields["response"] = res_itf - fields["response_text"] = res_text + var logged bool + if itf, ok := fields["telemetry_version"]; ok { + if version, ok := itf.(string); ok { + if len(version) > 0 { + logged = true + fields["response"] = map[string]string{ + "telemetry": version, + } + } + } + } + if !logged { + res_itf, res_text := GetResponseLogObjs(rbytes) + fields["response"] = res_itf + fields["response_text"] = res_text + } } var doc_map util.Dict @@ -755,6 +950,8 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { err1 := common.NewError(err) fields["response"] = ObfuscatedMap fields["response_text"] = err1.Error() + } else { + fields["response"] = itf } } else { fields["response"] = ObfuscatedMap @@ -767,13 +964,15 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["duration"] = duration fields["logger"] = "request" + s.XpcTracer.SetSpan(fields, s.XpcTracer.MoracideTagPrefix()) + var userAgent string if itf, ok := fields["user_agent"]; ok { userAgent = itf.(string) } if userAgent != "mget" { tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("request ends") + log.WithFields(tfields).Info("Request finished") } } @@ -833,3 +1032,168 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { } return itf, "" } + +func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(kbytes), + Value: sarama.ByteEncoder(bbytes), + } + s.Input() <- outMessage + + tfields["logger"] = "kafkaproducer" + tfields["output_topic"] = outMessage.Topic + tfields["output_key"] = string(kbytes) + tfields["output_body"] = m + log.WithFields(tfields).Info("send") +} + +func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["output_topic"] = s.KafkaProducerTopic() + + for _, m := range messages { + if len(m.DeviceId) != 16 { + log.WithFields(tfields).Warn("invalid device_id " + m.DeviceId) + continue + } + mac := m.DeviceId[4:] + transactionUuid := s.AppName() + "_____" + uuid.New().String() + m.TransactionUuid = &transactionUuid + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(strings.ToLower(mac)), + Value: sarama.ByteEncoder(bbytes), + } + s.Input() <- outMessage + + tfields["output_key"] = mac + tfields["output_body"] = m + log.WithFields(tfields).Info("send") + } +} + +func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token string, tokenErr error) { + fields := xw.Audit() + fields["logger"] = "token" + tfields := common.FilterLogFields(fields) + var headerMap map[string]string + var isObfuscated bool + if itf, ok := tfields["header"]; ok { + headerMap = itf.(map[string]string) + if len(headerMap) > 0 { + ss := authorization + if len(ss) > authPrefixLength { + ss = authorization[:authPrefixLength] + "****" + isObfuscated = true + } + headerMap["Authorization"] = ss + } + } + + if isObfuscated { + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken + } + } + + log.WithFields(tfields).Debug(tokenErr) +} + +func (s *WebconfigServer) HandleKafkaProducerResults() { + if s.AsyncProducer == nil { + return + } + + for { + select { + case success := <-s.Successes(): + if success == nil { + continue + } + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = success.Topic + fields["output_partition"] = success.Partition + fields["output_offset"] = success.Offset + log.WithFields(fields).Debug("sent") + case pErr := <-s.Errors(): + if pErr == nil || pErr.Msg == nil { + continue + } + if m := s.Metrics(); m != nil { + m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) + } + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = pErr.Msg.Topic + fields["output_partition"] = pErr.Msg.Partition + kbytes, err := pErr.Msg.Key.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + fields["output_key"] = string(kbytes) + } + + vbytes, err := pErr.Msg.Value.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + var itf interface{} + err1 := json.Unmarshal(vbytes, &itf) + if err1 != nil { + log.WithFields(fields).Error(common.NewError(err1)) + fields["output_body_text"] = base64.StdEncoding.EncodeToString(vbytes) + } else { + fields["output_body"] = itf + } + } + + log.WithFields(fields).Error(pErr.Err) + } + } +} + +func (s *WebconfigServer) StopXpcTracer() { + sdkTraceProvider, ok := s.XpcTracer.OtelTracerProvider().(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} + +func (s *WebconfigServer) SpanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.XpcTracer.OtelEnabled { + ctx, otelSpan := tracing.NewOtelSpan(s.XpcTracer, r) + r = r.WithContext(ctx) + defer tracing.EndOtelSpan(s.XpcTracer, otelSpan) + } + next.ServeHTTP(w, r) + }) +} diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 1699106..f7d484e 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -97,4 +97,45 @@ func TestWebconfigServerSetterGetter(t *testing.T) { validPartners = []string{"name3", "name4", "name5"} server.SetValidPartners(validPartners) assert.DeepEqual(t, server.ValidPartners(), validPartners) + + // get profiles from upstream + enabled = true + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + enabled = false + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + + // enforce strict query parameters validation + enabled = true + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + enabled = false + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + + //configure trust level + trust := 1000 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + trust = 500 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + + x := true + server.SetFilterOutputByBitmapEnabled(x) + assert.Assert(t, server.FilterOutputByBitmapEnabled()) + x = false + server.SetFilterOutputByBitmapEnabled(x) + assert.Assert(t, !server.FilterOutputByBitmapEnabled()) + + validSubdocIdMap := map[string]int{ + "red": 1, + "orange": 2, + "yellow": 3, + "green": 4, + } + server.SetValidSubdocIdMap(validSubdocIdMap) + assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) + } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 021f049..95b18fa 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,21 +25,21 @@ import ( "net/http" "time" - "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) const ( - defaultWebpaHost = "https://api.webpa.comcast.net:8090" + defaultWebpaHost = "http://localhost:12345" defaultApiVersion = "v2" webpaServiceName = "webpa" asyncWebpaServiceName = "asyncwebpa" - webpaUrlTemplate = "%s/api/%s/device/mac:%s/config" - webpaError404 = `{"code": 521, "message": "Device not found in webpa"}` - webpaError520 = `{"code": 520, "message": "Error unsupported namespace"}` + defaultWebpaUrlTemplate = "%s/%s/%s" + webpaError404 = `{"code": 521, "message": "Device not found in webpa"}` + webpaError520 = `{"code": 520, "message": "Error unsupported namespace"}` // a new error code to indicate it is webpa 520 // but it is caused by some temporary conditions, @@ -65,6 +65,7 @@ type WebpaConnector struct { syncClient *HttpClient asyncClient *HttpClient host string + urlTemplate string queue chan struct{} retries int retryInMsecs int @@ -72,7 +73,7 @@ type WebpaConnector struct { apiVersion string } -func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func syncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -82,16 +83,16 @@ func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } -func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func asyncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -101,43 +102,34 @@ func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), true + return rbytes, nil, true, common.NewError(rerr) } func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *WebpaConnector { - confKey := fmt.Sprintf("webconfig.%v.host", webpaServiceName) - host := conf.GetString(confKey, defaultWebpaHost) - - confKey = fmt.Sprintf("webconfig.%v.async_poke_enabled", webpaServiceName) - asyncPokeEnabled := conf.GetBoolean(confKey, false) - - confKey = fmt.Sprintf("webconfig.%v.async_poke_concurrent_calls", webpaServiceName) - concurrentCalls := int(conf.GetInt32(confKey, 0)) + host := conf.GetString("webconfig.webpa.host", defaultWebpaHost) + asyncPokeEnabled := conf.GetBoolean("webconfig.webpa.async_poke_enabled", false) + concurrentCalls := int(conf.GetInt32("webconfig.webpa.async_poke_concurrent_calls", 0)) var queue chan struct{} if concurrentCalls > 0 { queue = make(chan struct{}, concurrentCalls) } - confKey = fmt.Sprintf("webconfig.%v.retries", webpaServiceName) - retries := int(conf.GetInt32(confKey, defaultRetries)) - - confKey = fmt.Sprintf("webconfig.%v.retry_in_msecs", webpaServiceName) - retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) + retries := int(conf.GetInt32("webconfig.webpa.retries", defaultRetries)) + retryInMsecs := int(conf.GetInt32("webconfig.webpa.retry_in_msecs", defaultRetriesInMsecs)) + apiVersion := conf.GetString("webconfig.webpa.api_version", defaultApiVersion) + urlTemplate := conf.GetString("webconfig.webpa.url_template", defaultWebpaUrlTemplate) syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig) syncClient.SetStatusHandler(520, syncHandle520) asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig) asyncClient.SetStatusHandler(520, asyncHandle520) - confKey = fmt.Sprintf("webconfig.%v.api_version", webpaServiceName) - apiVersion := conf.GetString(confKey, defaultApiVersion) - connector := WebpaConnector{ syncClient: syncClient, asyncClient: asyncClient, @@ -147,6 +139,7 @@ func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *Webpa retryInMsecs: retryInMsecs, asyncPokeEnabled: asyncPokeEnabled, apiVersion: apiVersion, + urlTemplate: urlTemplate, } return &connector @@ -160,6 +153,14 @@ func (c *WebpaConnector) SetWebpaHost(host string) { c.host = host } +func (c *WebpaConnector) WebpaUrlTemplate() string { + return c.urlTemplate +} + +func (c *WebpaConnector) SetWebpaUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *WebpaConnector) ApiVersion() string { return c.apiVersion } @@ -185,10 +186,10 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { c.asyncPokeEnabled = enabled } -func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { - url := fmt.Sprintf(webpaUrlTemplate, c.WebpaHost(), c.ApiVersion(), cpeMac) +func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { + url := fmt.Sprintf(c.WebpaUrlTemplate(), c.WebpaHost(), c.ApiVersion(), cpeMac) - var traceId, xmTraceId, outTraceparent, outTracestate string + var traceId, xmTraceId string if itf, ok := fields["trace_id"]; ok { traceId = itf.(string) } @@ -203,25 +204,14 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field if len(traceId) == 0 { traceId = xmTraceId } - if itf, ok := fields["out_traceparent"]; ok { - outTraceparent = itf.(string) - } - if itf, ok := fields["out_tracestate"]; ok { - outTracestate = itf.(string) - } t := time.Now().UnixNano() / 1000 transactionId := fmt.Sprintf("%s_____%015x", xmTraceId, t) - xmoney := fmt.Sprintf("trace-id=%s;parent-id=0;span-id=0;span-name=%s", xmTraceId, webpaServiceName) - header := make(http.Header) - header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + header := rHeader.Clone() header.Set("X-Webpa-Transaction-Id", transactionId) - header.Set("X-Moneytrace", xmoney) - header.Set(common.HeaderTraceparent, outTraceparent) - header.Set(common.HeaderTracestate, outTracestate) method := "PATCH" - _, _, err, cont := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -251,18 +241,26 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field } func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - var cont bool - + tfields := common.FilterLogFields(fields, "status") + tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) copy(cbytes, bbytes) if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont = c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { + msg := fmt.Sprintf("finished success after 1 retry") + if i > 1 { + fmt.Sprintf("finished success after %v retries", i) + } + log.WithFields(tfields).Info(msg) break } + if i == c.retries { + log.WithFields(tfields).Infof("finished failure after %v retries", i) + } } <-c.queue } @@ -279,7 +277,7 @@ func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header htt if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, err, cont = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError diff --git a/http/xconf_connector.go b/http/xconf_connector.go index d40550a..0c46ec0 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,31 +22,33 @@ import ( "fmt" "net/http" - owcommon "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" + owcommon "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) const ( - xconfHostDefault = "http://qa2.xconfds.coast.xcal.tv:8080" - xconfUrlTemplate = "%s/loguploader/getTelemetryProfiles?%s" + defaultXconfHost = "http://localhost:12346" + defaultXconfUrlTemplate = "%s/%s" ) type XconfConnector struct { *HttpClient host string serviceName string + urlTemplate string } func NewXconfConnector(conf *configuration.Config, tlsConfig *tls.Config) *XconfConnector { serviceName := "xconf" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, xconfHostDefault) + host := conf.GetString("webconfig.xconf.host", defaultXconfHost) + urlTemplate := conf.GetString("webconfig.xconf.url_template", defaultXconfUrlTemplate) return &XconfConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, + urlTemplate: urlTemplate, } } @@ -58,12 +60,20 @@ func (c *XconfConnector) SetXconfHost(host string) { c.host = host } +func (c *XconfConnector) XconfUrlTemplate() string { + return c.urlTemplate +} + +func (c *XconfConnector) SetXconfUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *XconfConnector) ServiceName() string { return c.serviceName } func (c *XconfConnector) GetProfiles(urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { - url := fmt.Sprintf(xconfUrlTemplate, c.XconfHost(), urlSuffix) + url := fmt.Sprintf(c.XconfUrlTemplate(), c.XconfHost(), urlSuffix) rbytes, resHeader, err := c.DoWithRetries("GET", url, nil, nil, fields, c.ServiceName()) if err != nil { return rbytes, resHeader, owcommon.NewError(err) diff --git a/kafka/consumer.go b/kafka/consumer.go index b97268c..5420464 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -14,17 +14,19 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package kafka import ( "encoding/base64" "encoding/json" + "errors" "fmt" + "net/http" "strings" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" wchttp "github.com/rdkcentral/webconfig/http" @@ -75,31 +77,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, []string, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, common.NewError(err) + return nil, nil, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, common.NewError(err) + return nil, nil, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, nil + return &m, nil, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - err = db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, common.NewError(err) + return &m, updatedSubdocIds, common.NewError(err) } - return &m, nil + return &m, updatedSubdocIds, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -170,7 +172,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram // NOTE: // Do not move the code below to a goroutine. // The `ConsumeClaim` itself is called within a goroutine, see: - // https://github.com/Shopify/sarama/blob/master/consumer_group.go#L27-L29 + // https://github.com/IBM/sarama/blob/master/consumer_group.go#L27-L29 rl := ratelimit.New(c.ratelimitMessagesPerSecond, ratelimit.WithoutSlack) // per second, no slack. for { @@ -203,19 +205,20 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage + var updatedSubdocIds []string eventName, rptHeaderValue := getEventName(message) switch eventName { case "mqtt-get": m, err = c.handleGetMessage(message.Value, fields) - logMessage = "request ends" + logMessage = "Request Finished" case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, err = c.handleNotification(bbytes, fields) + m, updatedSubdocIds, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, err = c.handleNotification(message.Value, fields) + m, updatedSubdocIds, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -225,15 +228,19 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram fields["event_name"] = eventName fields["rpt"] = rptHeaderValue + forwardMessage := false if err != nil { if c.IsDbNotFound(err) { log.WithFields(fields).Trace("db not found") + } else if errors.Is(err, common.ErrPending) { + log.WithFields(fields).Trace("pending") } else { fields["error"] = err.Error() fields["kafka_message"] = base64.StdEncoding.EncodeToString(message.Value) log.WithFields(fields).Error("errors") } } else { + forwardMessage = true log.WithFields(fields).Info(logMessage) } @@ -253,6 +260,26 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } metrics.CountKafkaEvents(eventName, status, message.Partition) } + + if c.KafkaProducerEnabled() && m != nil && forwardMessage { + c.ForwardKafkaMessage(message.Key, m, fields) + if len(m.Reports) == 0 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && len(updatedSubdocIds) > 0 { + // build a root/success message + applicationStatus := "success" + for _, subdocId := range updatedSubdocIds { + em := &common.EventMessage{ + Namespace: &subdocId, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) + } + } + } + } case <-session.Context().Done(): return nil } diff --git a/kafka/consumer_group_test.go b/kafka/consumer_group_test.go index 5db165f..d720eb2 100644 --- a/kafka/consumer_group_test.go +++ b/kafka/consumer_group_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "gotest.tools/assert" ) diff --git a/kafka/kafka_consumer_group.go b/kafka/kafka_consumer_group.go index 298a3a2..9cc24e9 100644 --- a/kafka/kafka_consumer_group.go +++ b/kafka/kafka_consumer_group.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package kafka import ( @@ -22,11 +22,11 @@ import ( "strings" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" + "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" wchttp "github.com/rdkcentral/webconfig/http" - "github.com/go-akka/configuration" ) type KafkaConsumerGroup struct { @@ -103,6 +103,16 @@ func NewKafkaConsumerGroup(conf *configuration.Config, s *wchttp.WebconfigServer } } + // Load TLS configuration + tlsConfig, err := common.LoadKafkaTLSConfig(conf, prefix) + if err != nil { + return nil, common.NewError(fmt.Errorf("failed to load TLS configuration for %s: %v", prefix, err)) + } + if tlsConfig != nil { + sconfig.Net.TLS.Enable = true + sconfig.Net.TLS.Config = tlsConfig + } + consumer := NewConsumer(s, ratelimitMessagesPerSecond, m, clusterName, offsetEnum, topicPartitionsMap) client, err := sarama.NewConsumerGroup(brokers, group, sconfig) diff --git a/main.go b/main.go index 4c32552..a6e5f80 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package main import ( @@ -24,12 +24,13 @@ import ( "os" "os/signal" "syscall" + "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rdkcentral/webconfig/common" wchttp "github.com/rdkcentral/webconfig/http" "github.com/rdkcentral/webconfig/kafka" - "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" _ "go.uber.org/automaxprocs" "golang.org/x/sync/errgroup" @@ -60,6 +61,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.Stop() // setup logging logFile := server.GetString("webconfig.log.file") @@ -79,6 +81,7 @@ func main() { TimestampFormat: common.LoggingTimeFormat, FieldMap: log.FieldMap{ log.FieldKeyTime: "timestamp", + log.FieldKeyMsg: "message", }, }) @@ -92,7 +95,9 @@ func main() { log.SetLevel(logLevel) // setup sarama logger - sarama.Logger = log.StandardLogger() + if server.GetBoolean("webconfig.log.sarama_logger_enabled") { + sarama.Logger = log.StandardLogger() + } // setup router router := server.GetRouter(false) @@ -102,6 +107,9 @@ func main() { router.Handle("/metrics", promhttp.Handler()) metrics = common.NewMetrics(sc.Config) server.SetMetrics(metrics) + if server.KafkaProducerEnabled() { + go server.HandleKafkaProducerResults() + } handler := metrics.WebMetrics(router) server.Handler = handler } else { @@ -122,6 +130,11 @@ func main() { func() error { <-gCtx.Done() fmt.Printf("HTTP server shutdown NOW !!\n") + if server.KafkaProducerEnabled() { + if err := server.AsyncProducer.Close(); err != nil { + fmt.Fprintf(os.Stderr, "%v AsyncProducer.Close() err=%v\n", time.Now().Format(common.LoggingTimeFormat), err) + } + } return server.Shutdown(context.Background()) }, ) diff --git a/security/main_test.go b/security/main_test.go index b1223c8..d822e54 100644 --- a/security/main_test.go +++ b/security/main_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -39,7 +39,9 @@ func TestMain(m *testing.M) { NewTestCodec(sc.Config) - tokenManager = NewTokenManager(sc.Config) + if sc.Config.GetBoolean("webconfig.jwt.enabled", false) || os.Getenv("TOKEN_TEST") == "1" { + tokenManager = NewTokenManager(sc.Config) + } log.SetOutput(io.Discard) returnCode := m.Run() diff --git a/security/token.go b/security/token.go index 52592e0..47270a4 100644 --- a/security/token.go +++ b/security/token.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -22,15 +22,15 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io/ioutil" + "os" "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -42,13 +42,13 @@ type ThemisClaims struct { Mac string `json:"mac"` PartnerId string `json:"partner-id"` Serial string `json:"serial"` - Trust string `json:"trust"` + Trust int `json:"trust"` Uuid string `json:"uuid"` Capabilities []string `json:"capabilities"` jwt.RegisteredClaims } -type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, error) +type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, int, error) type TokenManager struct { encodeKey *rsa.PrivateKey @@ -61,11 +61,6 @@ type TokenManager struct { } func NewTokenManager(conf *configuration.Config) *TokenManager { - jwtEnabled := conf.GetBoolean("webconfig.jwt.enabled", false) - if !jwtEnabled { - return nil - } - panicExitEnabled := conf.GetBoolean("webconfig.panic_exit_enabled", false) // prepare args for TokenManager @@ -110,7 +105,7 @@ func NewTokenManager(conf *configuration.Config) *TokenManager { } func loadDecodeKey(keyfile string) (*rsa.PublicKey, error) { - kbytes, err := ioutil.ReadFile(keyfile) + kbytes, err := os.ReadFile(keyfile) if err != nil { return nil, common.NewError(err) } @@ -122,7 +117,7 @@ func loadDecodeKey(keyfile string) (*rsa.PublicKey, error) { } func loadEncodeKey(keyfile string) (*rsa.PrivateKey, error) { - kbytes, err := ioutil.ReadFile(keyfile) + kbytes, err := os.ReadFile(keyfile) if err != nil { return nil, common.NewError(err) } @@ -134,19 +129,25 @@ func loadEncodeKey(keyfile string) (*rsa.PrivateKey, error) { } // TODO this is not an officially supported function. -func (m *TokenManager) Generate(mac string, ttl int64, vargs ...string) string { +func (m *TokenManager) Generate(mac string, ttl int64, itfs ...interface{}) string { // %% NOTE mac should be lowercase to be consistent with reference doc // static themis fields copied from examples in the webconfig confluence kid := "webconfig_key" serial := "ABCNDGE" - trust := "1000" + trust := 1000 capUuid := "1234567891234" capabilities := []string{"x1:issuer:test:.*:all"} - partner := "comcast" - if len(vargs) > 0 { - partner = vargs[0] + + for _, itf := range itfs { + switch ty := itf.(type) { + case string: + partner = ty + case int: + trust = ty + } } + utcnow := time.Now() claims := ThemisClaims{ @@ -213,30 +214,30 @@ func ParseKidFromTokenHeader(tokenString string) (string, error) { rawKid, ok := headers["kid"] if !ok { - return kid, common.NewError(common.NotOK) + return kid, common.NewError(common.ErrNotOK) } kid, ok = rawKid.(string) if !ok { - return kid, common.NewError(common.NotOK) + return kid, common.NewError(common.ErrNotOK) } return kid, nil } func (m *TokenManager) VerifyApiToken(token string) (bool, error) { - ok, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) + ok, _, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) if err != nil { return ok, common.NewError(err) } return ok, err } -func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, error) { - ok, partner, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) +func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, int, error) { + ok, partner, trust, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) if err != nil { - return ok, "", common.NewError(err) + return ok, "", trust, common.NewError(err) } - return ok, partner, nil + return ok, partner, trust, nil } func (m *TokenManager) SetVerifyFunc(fn VerifyFunc) { @@ -293,9 +294,10 @@ func (m *TokenManager) ParseCpeToken(tokenStr string) (map[string]string, error) return data, nil } -func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, error) { +func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, int, error) { tokenString := vargs[0] var kid string + var trust int parser := &jwt.Parser{} @@ -305,11 +307,11 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi // check kid rawkid, ok := token.Header["kid"] if !ok { - return false, "", common.NewError(fmt.Errorf("missing kid in token")) + return false, "", trust, common.NewError(fmt.Errorf("missing kid in token")) } kid, ok = rawkid.(string) if !ok { - return false, "", common.NewError(fmt.Errorf("error in reading kid from header")) + return false, "", trust, common.NewError(fmt.Errorf("error in reading kid from header")) } ok = false @@ -320,7 +322,7 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !ok { - return false, "", common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) + return false, "", trust, common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) } // check capabilities, if requiredCapabilities is nonempty @@ -343,21 +345,21 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !isCapable { - return false, "", common.NewError(fmt.Errorf("token without proper capabilities")) + return false, "", trust, common.NewError(fmt.Errorf("token without proper capabilities")) } } } else { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } decodeKey, ok := decodeKeys[kid] if !ok { - return false, "", common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) + return false, "", trust, common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) } claims := jwt.MapClaims{} if _, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) { return decodeKey, nil }); err != nil { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } if len(vargs) > 1 { @@ -369,12 +371,12 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi if strings.ToLower(mac) == strings.ToLower(macstr) { isMatched = true } else { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) } } } if !isMatched { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) } } @@ -384,5 +386,9 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi partner = itf.(string) } - return true, partner, nil + if itf, ok := claims["trust"]; ok { + trust = util.ToInt(itf) + } + + return true, partner, trust, nil } diff --git a/security/token_test.go b/security/token_test.go index 5f6bab5..6f73aa6 100644 --- a/security/token_test.go +++ b/security/token_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -55,11 +55,7 @@ func TestLoadingKeyFiles(t *testing.T) { } func TestTokenValidation(t *testing.T) { - sc, err := common.GetTestServerConfig() - if err != nil { - panic(err) - } - if !sc.GetBoolean("webconfig.jwt.enabled") { + if tokenManager == nil { t.Skip("webconfig.jwt.enabled = false") } @@ -67,16 +63,26 @@ func TestTokenValidation(t *testing.T) { token := tokenManager.Generate(strings.ToLower(cpeMac), 86400) // default comcast - ok, parsedPartner, err := tokenManager.VerifyCpeToken(token, cpeMac) + ok, parsedPartner, trust, err := tokenManager.VerifyCpeToken(token, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, "comcast") + assert.Equal(t, trust, 1000) // create a partner token partner1 := "cox" token1 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1) - ok, parsedPartner, err = tokenManager.VerifyCpeToken(token1, cpeMac) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token1, cpeMac) + assert.NilError(t, err) + assert.Assert(t, ok) + assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 1000) + + // create a partner token with non-default trust + token2 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1, 500) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token2, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 500) } diff --git a/tracing/ctx.go b/tracing/ctx.go new file mode 100644 index 0000000..b106712 --- /dev/null +++ b/tracing/ctx.go @@ -0,0 +1,32 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package tracing + +import "context" + +type contextKey string + +// SetContext - setting context for logging +func SetContext(ctx context.Context, ctxName string, ctxValue interface{}) context.Context { + return context.WithValue(ctx, contextKey(ctxName), ctxValue) +} + +// GetContext - getting context value from context +func GetContext(ctx context.Context, key string) interface{} { + return ctx.Value(contextKey(key)) +} diff --git a/tracing/otel.go b/tracing/otel.go new file mode 100644 index 0000000..6617224 --- /dev/null +++ b/tracing/otel.go @@ -0,0 +1,259 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package tracing + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + log "github.com/sirupsen/logrus" +) + +type providerConstructor func(*XpcTracer) (oteltrace.TracerProvider, error) + +var ( + providerBuilders = map[string]providerConstructor{ + "http": otelHttpTraceProvider, + "stdout": otelStdoutTraceProvider, + "noop": otelNoopTraceProvider, + } +) + +// defaultOtelTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultOtelTracerProvider = "noop" + +// initOtel - initialize OpenTelemetry constructs +// tracing instrumentation code. +func otelInit(xpcTracer *XpcTracer, conf *configuration.Config) { + xpcTracer.OtelEnabled = conf.GetBoolean("webconfig.tracing.otel.enabled") + if !xpcTracer.OtelEnabled { + return + } + xpcTracer.otelProvider = strings.ToLower(conf.GetString("webconfig.tracing.otel.provider", defaultOtelTracerProvider)) + if xpcTracer.otelProvider == "" { + xpcTracer.otelProvider = defaultOtelTracerProvider + } + if xpcTracer.otelProvider == defaultOtelTracerProvider { + log.Debug("otel disabled, noop provider") + return + } + + log.Debug("otel enabled") + xpcTracer.otelEndpoint = conf.GetString("webconfig.tracing.otel.endpoint") + xpcTracer.otelOpName = conf.GetString("webconfig.tracing.otel.operation_name") + + if providerBuilder := providerBuilders[xpcTracer.otelProvider]; providerBuilder == nil { + log.Errorf("no builder func for otel provider %s", xpcTracer.otelProvider) + return + } else { + var err error + if xpcTracer.otelTracerProvider, err = providerBuilder(xpcTracer); err != nil { + log.Errorf("building otel provider for %s failed with %v", xpcTracer.otelProvider, err) + return + } + } + otel.SetTracerProvider(xpcTracer.otelTracerProvider) + + // Set up propagator. + xpcTracer.otelPropagator = propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) + otel.SetTextMapPropagator(xpcTracer.otelPropagator) + + xpcTracer.otelTracer = otel.Tracer(xpcTracer.appName) +} + +func otelNoopTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func otelStdoutTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), + ), + ), + ) + return tp, nil +} + +func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { + // Send traces over HTTP + if xpcTracer.otelEndpoint == "" { + return nil, fmt.Errorf("building http otel provider failure, no endpoint specified") + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(xpcTracer.otelEndpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("building http otel provider failed with %v", err) + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), + ), + ), + ), nil +} + +func NewOtelSpan(xpcTracer *XpcTracer, r *http.Request) (context.Context, oteltrace.Span) { + ctx := r.Context() + var otelSpan oteltrace.Span + if !xpcTracer.OtelEnabled { + return ctx, otelSpan + } + + pathTemplate := "placeholder" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + var err error + pathTemplate, err = mux.CurrentRoute(r).GetPathTemplate() + if err != nil { + log.Debugf("unable to get path template: %v", err) + } + } + resourceName := r.Method + " " + pathTemplate + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) + + /* + required span attribute: HTTPMethodKey = attribute.Key("http.method") + required span attribute: HTTPRouteKey = attribute.Key("http.route") + required span attribute: HTTPStatusCodeKey = attribute.Key("http.status_code") + required span attribute: HTTPURLKey = attribute.Key("http.url") + custom Comcast attribute: X-Cl-Experiment: true/false + additional: env, operation.name, http.url_details.path + */ + ctx, otelSpan = xpcTracer.otelTracer.Start(ctx, xpcTracer.OtelOpName(), + oteltrace.WithSpanKind(oteltrace.SpanKindServer), + oteltrace.WithAttributes( + attribute.String("env", xpcTracer.appEnv), + attribute.String("http.method", r.Method), + attribute.String("http.route", pathTemplate), + attribute.String("http.url", r.URL.String()), + attribute.String("http.url_details.path", r.URL.Path), + attribute.String("operation.name", xpcTracer.OtelOpName()), + ), + ) + if xpcTracer.region != "" { + rgnAttr := attribute.String("region", xpcTracer.region) + otelSpan.SetAttributes(rgnAttr) + } + + log.Debugf("span started %s", resourceName) + log.Debugf("added span attribute key = env, value = %s", xpcTracer.appEnv) + log.Debugf("added span attribute key = http.method, value = %s", r.Method) + log.Debugf("added span attribute key = http.route, value = %s", pathTemplate) + log.Debugf("added span attribute key = http.url, value = %s", r.URL.String()) + log.Debugf("added span attribute key = http.url_details.path, value = %s", r.URL.Path) + log.Debugf("added span attribute key = operation.name, value = %s", xpcTracer.OtelOpName()) + + carrier := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, carrier) + for key, val := range carrier { + ctx = SetContext(ctx, "otel_"+key, val) + log.Debugf("OtelSpanCreation: otel %s = %s", key, val) + } + + ctx = SetContext(ctx, "otel_span", otelSpan) + return ctx, otelSpan +} + +func otelSetStatusCode(span oteltrace.Span, statusCode int) { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + log.Debugf("added span attribute key = http.status_code, value = %d", statusCode) + + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debugf("added span attribute key=http.response.error, value=%s", statusText) + } +} + +func EndOtelSpan(xpcTracer *XpcTracer, span oteltrace.Span) { + if !xpcTracer.OtelEnabled { + return + } + span.End() +} + +func otelExtractParamsFromSpan(ctx context.Context, xpcTrace *XpcTrace) { + if tmp := GetContext(ctx, "otel_span"); tmp != nil { + if otelSpan, ok := tmp.(oteltrace.Span); ok { + if otelSpan == nil { + return + } + xpcTrace.otelSpan = otelSpan + spanCtx := otelSpan.SpanContext() + xpcTrace.TraceID = spanCtx.TraceID().String() + } + } + if xpcTrace.otelSpan == nil { + return + } + // if otel span is found, use the extracted traceparent and tracestate from the otel span + // We store the extracted values in ctx when we created the otel span + if tmp := GetContext(ctx, "otel_traceparent"); tmp != nil { + xpcTrace.otelTraceparent = tmp.(string) + log.Debugf("Tracing: otel traceparent = %s", xpcTrace.otelTraceparent) + xpcTrace.OutTraceparent = xpcTrace.otelTraceparent + } + if tmp := GetContext(ctx, "otel_tracestate"); tmp != nil { + xpcTrace.otelTracestate = tmp.(string) + log.Debugf("Tracing: otel tracestate = %s", xpcTrace.otelTracestate) + xpcTrace.OutTracestate = xpcTrace.otelTracestate + } +} diff --git a/tracing/span.go b/tracing/span.go new file mode 100644 index 0000000..123925f --- /dev/null +++ b/tracing/span.go @@ -0,0 +1,131 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package tracing + +import ( + "net/http" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + + "go.opentelemetry.io/otel/attribute" + oteltrace "go.opentelemetry.io/otel/trace" +) + +// XpcTrace is a carrier/baggage struct to extract data from spans, request headers for usage later +// Store the trace in ctx for easy retrieval. +// Ideal place to store it is ofc, xw +// But because of legacy reasons, xw is not always available in the API flow +type XpcTrace struct { + ReqMoracideTag string + // The response moracide tags are stored in xw.audit + + // traceparent, tracestate can be set as req headers, may be extracted from otel spans + // These need to be propagated to any http calls we make + // Order of priority; use the value extracted from otel span; + // if no otel span as well, use the value in req headers (Note: this will create islands as both + // the app and its children will have the same tracestate + // Otherwise, nothing will be passed to the child http calls, creating islands + // If any source is found, then it will be propagated to all child http calls + // TODO; also add this to Kafka headers, SNS message attributes + otelTraceparent string + otelTracestate string + ReqTraceparent string + ReqTracestate string + OutTraceparent string + OutTracestate string + + // At the end of API flow, add the status code to OtelSpan; add the Moracide tags to the spans + otelSpan oteltrace.Span + + // These are not useful as of now, just set them for the sake of completion and future + AuditID string + MoneyTrace string + ReqUserAgent string + OutUserAgent string + + TraceID string // use the value in outTraceparent, otherwise MoneyTrace +} + +// NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs +func NewXpcTrace(xpcTracer *XpcTracer, r *http.Request) *XpcTrace { + var xpcTrace XpcTrace + extractParamsFromReq(r, &xpcTrace, xpcTracer.AppName()) + + if xpcTracer.OtelEnabled { + otelExtractParamsFromSpan(r.Context(), &xpcTrace) + } + + return &xpcTrace +} + +func SetSpanStatusCode(fields log.Fields) { + var xpcTrace *XpcTrace + if tmp, ok := fields["xpc_trace"]; ok { + xpcTrace = tmp.(*XpcTrace) + } + if xpcTrace == nil { + // Something went wrong, cannot instrument this span + log.Error("instrumentation error, no trace info") + return + } + if xpcTrace.otelSpan != nil { + if tmp, ok := fields["status"]; ok { + statusCode := tmp.(int) + otelSetStatusCode(xpcTrace.otelSpan, statusCode) + } + } +} + +func SetSpanMoracideTags(fields log.Fields, moracideTagPrefix string) { + var xpcTrace *XpcTrace + if tmp, ok := fields["xpc_trace"]; ok { + xpcTrace = tmp.(*XpcTrace) + } + if xpcTrace == nil { + // Something went wrong, cannot instrument this span + log.Error("instrumentation error, cannot set moracide tags, no trace info") + return + } + + moracide := util.FieldsGetString(fields, "resp_moracide_tag") + if len(moracide) == 0 { + moracide = util.FieldsGetString(fields, "req_moracide_tag") + } + + if xpcTrace.otelSpan != nil && len(moracide) > 0 { + xpcTrace.otelSpan.SetAttributes(attribute.String(common.HeaderMoracide, moracide)) + } +} + +func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace, serviceName string) { + xpcTrace.ReqTraceparent = r.Header.Get(common.HeaderTraceparent) + xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) + xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent + xpcTrace.OutTracestate = xpcTrace.ReqTracestate + xpcTrace.ReqUserAgent = r.Header.Get(UserAgentHeader) + xpcTrace.ReqMoracideTag = r.Header.Get(common.HeaderMoracide) + if ss := r.Header.Get(common.HeaderCanary); ss == "true" { + if len(xpcTrace.ReqMoracideTag) > 0 { + xpcTrace.ReqMoracideTag += "," + serviceName + } else { + xpcTrace.ReqMoracideTag = serviceName + } + } +} diff --git a/tracing/tracer.go b/tracing/tracer.go new file mode 100644 index 0000000..a65455a --- /dev/null +++ b/tracing/tracer.go @@ -0,0 +1,141 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package tracing + +import ( + "os" + "strings" + + "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" + + otelpropagation "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" +) + +const ( + AuditIDHeader = "X-Auditid" + UserAgentHeader = "User-Agent" + DefaultMoracideTagPrefix = "X-Cl-Experiment" +) + +type SpanSetterFunc func(log.Fields, string) + +// XpcTracer is a wrapper around tracer setup +type XpcTracer struct { + OtelEnabled bool + moracideTagPrefix string // Special request header for moracide expts e.g. canary deployments + + // internal vars used by Otel + appEnv string // set this to dev for red, staging for yellow and prod for green + appName string + appVersion string + appSHA string // unused + region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute + siteColor string // red/yellow/green, unused, use it as a span attribute + + // internal otel vars + otelEndpoint string + otelOpName string + otelProvider string + otelTracerProvider oteltrace.TracerProvider + otelPropagator otelpropagation.TextMapPropagator + otelTracer oteltrace.Tracer + + SetSpan SpanSetterFunc +} + +func NewXpcTracer(conf *configuration.Config) *XpcTracer { + xpcTracer := new(XpcTracer) + initAppData(xpcTracer, conf) + otelInit(xpcTracer, conf) + xpcTracer.moracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", DefaultMoracideTagPrefix) + + if xpcTracer.OtelEnabled { + xpcTracer.SetSpan = OtelSetSpan + } else { + xpcTracer.SetSpan = NoopSetSpan + } + + return xpcTracer +} + +func (t *XpcTracer) MoracideTagPrefix() string { + return t.moracideTagPrefix +} + +// otelOpName should return "http.request" by default +func (t *XpcTracer) OtelOpName() string { + if len(t.otelOpName) == 0 { + return "http.request" + } + return t.otelOpName +} + +func (t *XpcTracer) OtelTracerProvider() oteltrace.TracerProvider { + return t.otelTracerProvider +} + +func (t *XpcTracer) AppName() string { + return t.appName +} + +func (t *XpcTracer) AppVersion() string { + return t.appVersion +} + +func (t *XpcTracer) AppEnv() string { + return t.appEnv +} + +func (t *XpcTracer) Region() string { + return t.region +} + +func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { + codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") + xpcTracer.appName = codeGitCommit[0] + if len(codeGitCommit) > 1 { + xpcTracer.appVersion = codeGitCommit[1] + } + if len(codeGitCommit) > 2 { + xpcTracer.appSHA = codeGitCommit[2] + } + + // Env vars + xpcTracer.appEnv = "dev" + siteColor := os.Getenv("site_color") + if strings.EqualFold(siteColor, "yellow") { + xpcTracer.appEnv = "staging" + } else if strings.EqualFold(siteColor, "green") { + xpcTracer.appEnv = "prod" + } + xpcTracer.region = os.Getenv("site_region") + if xpcTracer.region == "" { + xpcTracer.region = os.Getenv("site_region_name") + } + log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) +} + +func OtelSetSpan(fields log.Fields, tag string) { + SetSpanStatusCode(fields) + SetSpanMoracideTags(fields, tag) +} + +func NoopSetSpan(fields log.Fields, tag string) { +} diff --git a/util/dict.go b/util/dict.go index d5242eb..fb8d570 100644 --- a/util/dict.go +++ b/util/dict.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -216,5 +216,16 @@ func HeaderToMap(header http.Header) map[string]string { m[k] = v[0] } return m +} + +func FieldsGetString(fields log.Fields, keyName string, args ...string) string { + var ret string + if len(args) == 1 { + ret = args[0] + } + if itf, ok := fields[keyName]; ok { + ret = itf.(string) + } + return ret } diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go deleted file mode 100644 index 6cae866..0000000 --- a/util/firmware_bitmap_test.go +++ /dev/null @@ -1,621 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 -*/ -package util - -import ( - "fmt" - "strings" - "testing" - - "gotest.tools/assert" -) - -// bitmap is used to name int variables -// bitarray is used to name string variables - -func TestPrettyBitarray(t *testing.T) { - i := 8 - bs := PrettyBitarray(i) - expected := "0000 0000 0000 0000 0000 0000 0000 1000" - assert.Equal(t, bs, expected) - - j := 16777231 - bs = PrettyGroupBitarray(j) - expected = "00000001 0000 0000 0000 0000 0000 1111" - assert.Equal(t, bs, expected) -} - -func TestParseRdkGroupBitarray(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" - expectedMap := map[int]int{ - 1: 15, - 2: 3, - 3: 1, - 4: 1, - 5: 1, - 6: 1, - 7: 1, - 8: 1, - } - - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - for _, sid := range sids { - groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) - assert.NilError(t, err) - // assert.Equal(t, groupId, 1) - - expectedBitmap, ok := expectedMap[groupId] - assert.Assert(t, ok) - assert.Equal(t, groupBitmap, expectedBitmap) - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - - // bs := PrettyGroupBitarray(cpeBitmap) - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - } - - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } -} - -func TestParseCustomizedGroupBitarray(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - } - - newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" - group1Bitmap, err := BitarrayToBitmap(newGroup1Bitarray) - assert.NilError(t, err) - sids[0] = fmt.Sprintf("%v", group1Bitmap) - expectedEnabled["wan"] = false - expectedEnabled["macbinding"] = false - expectedEnabled["hotspot"] = true - expectedEnabled["bridge"] = true - - newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" - group2Bitmap, err := BitarrayToBitmap(newGroup2Bitarray) - assert.NilError(t, err) - sids[1] = fmt.Sprintf("%v", group2Bitmap) - expectedEnabled["privatessid"] = false - expectedEnabled["radio"] = true - - rdkSupportedDocsHeaderStr = strings.Join(sids, ",") - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestParseTelcovoipAndWanmanager(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549377,201326593" - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": false, - "radio": false, - "moca": false, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": true, - "telcovoice": false, - "wanmanager": true, - "voiceservice": false, - } - - rdkSupportedDocsHeaderStr = strings.Join(sids, ",") - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestBitmapParsing(t *testing.T) { - // clearn the wan bit - newBitmap := 16777231 & ^(1 << 2) - rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,33554435,50331649,67108865,83886081,100663297,117440513,134217729", newBitmap) - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": false, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestParseVoiceService(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809" - // sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": true, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": true, - } - - // rdkSupportedDocsHeaderStr = strings.Join(sids, ",") - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestManualBitmap(t *testing.T) { - for i := 0; i < 10; i++ { - bitmap := RandomInt(40000) - parsedSupportedMap := GetSupportedMap(bitmap) - revBitmap := GetBitmapFromSupportedMap(parsedSupportedMap) - assert.Equal(t, bitmap, revBitmap) - } -} - -func TestParseSupportedDocsWithNewGroups(t *testing.T) { - cellularBitGroupId := 14 - xBitValue := (cellularBitGroupId << 24) + 1 - ss := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809,234881025" - rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": true, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": true, - "cellularconfig": true, - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,67108865,100663297,117440513,134217729,201326595,234881025" - - // build expected - expectedEnabled := map[string]bool{ - "advsecurity": false, - "aker": true, - "bridge": false, - "cellularconfig": true, - "homessid": true, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549378,201326595" - - // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": false, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": true, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" - - // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,251658241,268435457" - - // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": false, - "lldqoscontrol": false, - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} - -func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) -} diff --git a/util/list.go b/util/list.go index f58b2b6..41e7641 100644 --- a/util/list.go +++ b/util/list.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -52,7 +52,8 @@ func Contains(collection interface{}, element interface{}) bool { } // TODO keep it for backward compatibility in "webconfig" for now -// plan to remove it later +// +// plan to remove it later func ContainsInt(data []int, x int) bool { for _, d := range data { if d == x { diff --git a/util/mock.go b/util/mock.go index 1c5e564..6f5229d 100644 --- a/util/mock.go +++ b/util/mock.go @@ -14,11 +14,10 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( - "crypto/rand" "strconv" "strings" @@ -29,12 +28,10 @@ func GetMockMultiparts(queryStr string) []common.Multipart { groupIds := strings.Split(queryStr, ",") mparts := []common.Multipart{} for _, g := range groupIds { - slen := RandomInt(100) + 16 - bbytes := make([]byte, slen) - rand.Read(bbytes) + bbytes := common.RandomBytes(16, 116) mpart := common.Multipart{ Bytes: bbytes, - Version: strconv.Itoa(RandomInt(100000000)), + Version: strconv.Itoa(common.RandomInt(100000000)), Name: g, } mparts = append(mparts, mpart) diff --git a/util/multipart.go b/util/multipart.go index 70c4683..ee91abf 100644 --- a/util/multipart.go +++ b/util/multipart.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -49,7 +49,7 @@ func ParseMultipartAsList(header http.Header, bbytes []byte) ([]common.Multipart return mparts, nil } - mediaType, params, err := mime.ParseMediaType(header.Get("Content-Type")) + mediaType, params, err := mime.ParseMediaType(header.Get(common.HeaderContentType)) if err != nil { return nil, common.NewError(err) } diff --git a/util/murmur3_test.go b/util/murmur3_test.go index c0203f5..e173f96 100644 --- a/util/murmur3_test.go +++ b/util/murmur3_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( "testing" + "github.com/rdkcentral/webconfig/common" tmur "github.com/twmb/murmur3" "gotest.tools/assert" ) @@ -46,7 +47,7 @@ func TestSomeMurmur3(t *testing.T) { s1 := GetMurmur3Hash(nil) assert.Equal(t, s1, "0") - bbytes := RandomBytes(10, 20) + bbytes := common.RandomBytes(10, 20) s2 := GetMurmur3Hash(bbytes) assert.Assert(t, len(s2) > 0) } diff --git a/util/parser_test.go b/util/parser_test.go index 1835dae..23fbc65 100644 --- a/util/parser_test.go +++ b/util/parser_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -22,13 +22,14 @@ import ( "net/http" "testing" + "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) func TestParseWebconfigResponseMessage(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("Content-length", "120") srcData := Dict{ @@ -55,7 +56,7 @@ func TestParseWebconfigResponseMessageMultipleHeaders(t *testing.T) { srcHeader.Add("X-Color", "color:red") srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") srcHeader.Add("X-Color", "color:orange") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("X-Color", "color:yellow:green") srcHeader.Add("Content-length", "120") srcHeader.Add("X-Color", "color:blue:indigo:violet") diff --git a/util/print.go b/util/print.go index e8d55bd..afe7de4 100644 --- a/util/print.go +++ b/util/print.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -40,6 +40,14 @@ func PrettyJson(input interface{}) string { if bbytes, err := json.MarshalIndent(input, "", " "); err == nil { pretty = string(bbytes) } + case http.Header: + m := make(map[string]string) + for k := range ty { + m[k] = ty.Get(k) + } + if bbytes, err := json.MarshalIndent(m, "", " "); err == nil { + pretty = string(bbytes) + } } return pretty diff --git a/util/string.go b/util/string.go index a582544..6c6cc0c 100644 --- a/util/string.go +++ b/util/string.go @@ -14,18 +14,26 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "fmt" "net/http" "net/url" "strconv" "strings" - "github.com/rdkcentral/webconfig/common" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" +) + +const ( + Comma = ',' + RightSquareBracket = ']' + RightCurlBracket = '}' + baseProfileStr = `{"profiles":[` ) var ( @@ -53,18 +61,6 @@ func GenerateRandomCpeMac() string { return strings.ToUpper(u[len(u)-12:]) } -func ValidateMac(mac string) bool { - if len(mac) != 12 { - return false - } - for _, r := range mac { - if r < 48 || r > 70 || (r > 57 && r < 65) { - return false - } - } - return true -} - func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId string) string { // build the query parameters in a fixed order params := []string{} @@ -103,6 +99,10 @@ func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId str if len(queryParams) > 0 && len(ret) > 0 { ret += "&" + queryParams } + + ret = url.QueryEscape(ret) + ret = strings.ReplaceAll(ret, "%3D", "=") + ret = strings.ReplaceAll(ret, "%26", "&") return ret } @@ -117,55 +117,6 @@ func GetMacDiff(wanMac, mac string) int { return wanMacVal - macVal } -func ValidatePokeQuery(values url.Values) (string, error) { - // handle ?doc=xxx - if docQueryParamStrs, ok := values["doc"]; ok { - if len(docQueryParamStrs) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(docQueryParamStrs[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeDocs, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // handle ?route=xxx - if qparams, ok := values["route"]; ok { - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(qparams[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeRoutes, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // return default - return "primary", nil -} - func GetEstbMacAddress(mac string) string { // if the mac cannot be parsed, then return back the input i, err := strconv.ParseInt(mac, 16, 64) @@ -180,3 +131,48 @@ func IsValidUTF8(bbytes []byte) bool { str2 := strings.ToValidUTF8(str1, "#") return str1 == str2 } + +func AppendProfiles(pbytes, sbytes []byte) ([]byte, error) { + var builder strings.Builder + if len(pbytes) < 3 { + builder.WriteString(baseProfileStr) + } else { + s := strings.TrimSpace(string(pbytes)) + builder.WriteString(s[:len(s)-2]) + } + if len(sbytes) > 2 { + if len(pbytes) > 20 { + builder.WriteRune(',') + } + builder.Write(sbytes[1 : len(sbytes)-1]) + } + builder.WriteRune(RightSquareBracket) + builder.WriteRune(RightCurlBracket) + return []byte(builder.String()), nil +} + +func CompactJson(sbytes []byte) ([]byte, error) { + var itf interface{} + if err := json.Unmarshal(sbytes, &itf); err != nil { + return nil, common.NewError(err) + } + jsbytes, err := json.Marshal(itf) + if err != nil { + return nil, common.NewError(err) + } + return jsbytes, nil +} + +func GetAPIName(queryStr string) string { + n := "poke" + if strings.Contains(queryStr, "slow") { + n = "slowpoke" + } else if strings.Contains(queryStr, "root") && strings.Contains(queryStr, "telemetry") { + n = "poke_both" + } else if strings.Contains(queryStr, "telemetry") { + n = "poke_telemetry" + } else if strings.Contains(queryStr, "mqtt") { + n = "poke_mqtt" + } + return n +} diff --git a/util/string_test.go b/util/string_test.go index cf6a894..e25b200 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -14,12 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "net/http" - "net/url" "testing" "github.com/rdkcentral/webconfig/common" @@ -33,28 +33,6 @@ func TestString(t *testing.T) { assert.Equal(t, c, expected) } -func TestValidateMac(t *testing.T) { - mac := "001122334455" - assert.Assert(t, ValidateMac(mac)) - - mac = "4444ABCDEF01" - assert.Assert(t, ValidateMac(mac)) - - mac = "00112233445Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "001122334455Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "0H1122334455" - assert.Assert(t, !ValidateMac(mac)) - - for i := 0; i < 10; i++ { - mac := GenerateRandomCpeMac() - assert.Assert(t, ValidateMac(mac)) - } -} - func TestGetAuditId(t *testing.T) { auditId := GetAuditId() assert.Equal(t, len(auditId), 32) @@ -80,68 +58,11 @@ func TestTelemetryQuery(t *testing.T) { assert.Equal(t, qstr, expected) } -func TestValidatePokeQuery(t *testing.T) { - values := url.Values{} - - values["doc"] = []string{ - "primary,telemetry", - "hello,world", - } - _, err := ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,hello,world", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,telemetry", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary", - } - s, err := ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "telemetry", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "telemetry") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "primary", - } - values["route"] = []string{ - "mqtt", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "mqtt") -} - func TestIsValidUTF8(t *testing.T) { b1 := []byte(`{"foo":"bar","hello":123,"world":true}`) assert.Assert(t, IsValidUTF8(b1)) - b2 := RandomBytes(100, 150) + b2 := common.RandomBytes(100, 150) assert.Assert(t, !IsValidUTF8(b2)) } @@ -159,3 +80,17 @@ func TestTelemetryQueryWithWanMac(t *testing.T) { expected := "env=PROD&partnerId=comcast&version=2.0&model=TG1682G&accountId=1234567890&firmwareVersion=TG1682_3.14p9s6_PROD_sey&estbMacAddress=567890ABCDEF" assert.Equal(t, qstr, expected) } + +func TestAppendProfiles(t *testing.T) { + mockedBaseProfilesResponse := `{"profiles":[{"name":"XfinityWIFI_SYNC","value":{"Description":"XfinityWIFI_SYNC to capture XWIFI info every 12 hours","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://stbrtl.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"name":"cpe_passpoint_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingServiceEnable","type":"dataModel"},{"name":"cpe_passpoint_inter_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingService.Parameters","type":"dataModel"},{"name":"cpe_passpoint_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Parameters","type":"dataModel"},{"name":"cpe_passpoint_rdk_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Enable","type":"dataModel"},{"name":"open5_bss_active","reference":"Device.WiFi.SSID.6.Enable","type":"dataModel"},{"name":"secure5_bss_active","reference":"Device.WiFi.SSID.10.Enable","type":"dataModel"},{"name":"secure5_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"wifi_enabled","reference":"Device.DeviceInfo.X_COMCAST_COM_xfinitywifiEnable","type":"dataModel"},{"name":"primary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.PrimaryRemoteEndpoint","type":"dataModel"},{"name":"secondary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.SecondaryRemoteEndpoint","type":"dataModel"},{"name":"open5_advertisement_str","reference":"Device.WiFi.AccessPoint.6.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"secure5_advertisement_str","reference":"Device.WiFi.AccessPoint.10.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"device_vlan_2","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.Interface.2.VLANID","type":"dataModel"},{"name":"open5_ssid_str","reference":"Device.WiFi.SSID.6.SSID","type":"dataModel"},{"name":"secure5_ssid_str","reference":"Device.WiFi.SSID.10.SSID","type":"dataModel"},{"name":"secure5_securitymode_str","reference":"Device.WiFi.AccessPoint.10.Security.ModeEnabled","type":"dataModel"},{"name":"secure5_pri_port","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerPort","type":"dataModel"},{"name":"dscp_marker","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.DSCPMarkPolicy","type":"dataModel"},{"name":"WIFI_CH_2_split","reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"name":"WIFI_CW_2_split","reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"name":"open5_bssid_str","reference":"Device.WiFi.SSID.6.BSSID","type":"dataModel"},{"name":"secure5_bssid_str","reference":"Device.WiFi.SSID.10.BSSID","type":"dataModel"},{"name":"open5_status_str","reference":"Device.WiFi.SSID.6.Status","type":"dataModel"},{"name":"secure5_status_str","reference":"Device.WiFi.SSID.10.Status","type":"dataModel"},{"name":"open5_beaconpower_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"open5_beaconrate_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"secure5_beaconpower_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"secure5_beaconrate_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"radio5_beaconinterval","reference":"Device.WiFi.Radio.2.X_COMCAST-COM_BeaconInterval","type":"dataModel"},{"name":"secure5_encryption","reference":"Device.WiFi.AccessPoint.10.Security.X_CISCO_COM_EncryptionMethod","type":"dataModel"},{"name":"secure5_sec_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerIPAddr","type":"dataModel"},{"name":"secure5_sec_port","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerPort","type":"dataModel"},{"name":"UPTIME_split","reference":"Device.DeviceInfo.UpTime","type":"dataModel"},{"name":"open5_radius_server_ip","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"open5_pri_port","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerPort","type":"dataModel"},{"name":"open5_isolation_enable","reference":"Device.WiFi.AccessPoint.6.IsolationEnable","type":"dataModel"},{"name":"secure5_isolation_enable","reference":"Device.WiFi.AccessPoint.10.IsolationEnable","type":"dataModel"},{"name":"secure24_bss_active","reference":"Device.WiFi.SSID.9.Enable","type":"dataModel"},{"name":"open24_bss_active","reference":"Device.WiFi.SSID.5.Enable","type":"dataModel"}],"Protocol":"HTTP","ReportingAdjustments":{"FirstReportingInterval":300},"ReportingInterval":86400,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.6"},"versionHash":"d9c6f386"},{"name":"WIFI_MOTION_Telemetry","value":{"Description":"CSCWFM_Telemetry","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"}],"URL":"https://stbrtl-oi.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"name":"Profile_Name","reference":"Profile.Name","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMrbussub_fail","name":"SYS_ERROR_WFMrbussub_fail","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_CSIpipe_restart","name":"SYS_SH_WFMpipe_restart","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_ctrlifcreate_fail","name":"SYS_ERROR_WFMifcreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSC_RXMrbusinit_fail","name":"SYS_ERROR_WFM_rbusinit_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionacquire_fail","name":"SYS_ERROR_WFMSessionAcq_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMeventscreate_fail","name":"SYS_ERROR_WFMeventscreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionenable_fail","name":"SYS_ERROR_WFMsessionenable_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXM_restart","name":"SYS_SH_WFMRXM_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_borg_restart","name":"SYS_SH_WFMborg_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_mqtt_restart","name":"SYS_SH_WFMmqtt_restart","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_sounding_state","name":"WFMsoundingstate_split","type":"event","use":"absolute"},{"name":"WFMEnable_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.CognitiveMotionDetection.Enable","type":"dataModel"},{"name":"WFMStatus_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_XHFW.WiFiMotionStatus","type":"dataModel"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentCommFailCnt_split","search":"wfmCogAgentCommFailCnt:","type":"grep","use":"count"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentConnected_split","search":"wfmCogAgentConnected:","type":"grep","use":"absolute"}],"Protocol":"HTTP","ReportingInterval":900,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"bf86fd16"}]}` + mockedExtraProfilesResponse := `[{"name":"james_test_profile_001","value":{"ActivationTimeout":600,"Description":"Telemetry 2.0 HSD Gateway WiFi Radio","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://rdkrtldev.stb.r53.xcal.tv/"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Description","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"}],"Protocol":"HTTP","ReportingInterval":60,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"ed0de6ef"}]` + + appendedBytes, err := AppendProfiles([]byte(mockedBaseProfilesResponse), []byte(mockedExtraProfilesResponse)) + assert.NilError(t, err) + + var itf interface{} + err = json.Unmarshal(appendedBytes, &itf) + assert.NilError(t, err) + _, err = json.MarshalIndent(itf, "", " ") + assert.NilError(t, err) +} diff --git a/util/validator.go b/util/validator.go new file mode 100644 index 0000000..f8a89f2 --- /dev/null +++ b/util/validator.go @@ -0,0 +1,111 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package util + +import ( + "fmt" + "net/http" + "net/url" + "slices" + "strings" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" +) + +func ValidateMac(mac string) bool { + if len(mac) != 12 { + return false + } + allZeroes := true + for _, r := range mac { + if r < 48 || r > 70 || (r > 57 && r < 65) { + return false + } + if r != 48 { + allZeroes = false + } + } + if allZeroes { + return false + } + return true +} + +func ValidatePokeQuery(values url.Values) (string, error) { + // handle ?doc=xxx + docStr := values.Get("doc") + if len(docStr) > 0 { + docNames := strings.Split(docStr, ",") + for _, n := range docNames { + if !slices.Contains(common.SupportedPokeDocs, n) { + err := fmt.Errorf("invalid query parameter: %v", n) + return "", common.NewError(err) + } + } + return docStr, nil + } + + // handle ?route=xxx + routeStr := values.Get("route") + if len(routeStr) > 0 { + if !slices.Contains(common.SupportedPokeRoutes, routeStr) { + err := fmt.Errorf("invalid query parameter: %v", routeStr) + return "", common.NewError(err) + + } + return routeStr, nil + } + + return "root", nil +} + +func ValidateQueryParams(r *http.Request, validSubdocIdMap map[string]int, fields log.Fields) error { + groupIdValues, ok := r.URL.Query()["group_id"] + if !ok || len(groupIdValues) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + fields["group_id"] = groupIdValues[0] + r.Header.Set(common.HeaderDocName, groupIdValues[0]) + + subdocIds := strings.Split(groupIdValues[0], ",") + if len(subdocIds) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + if len(subdocIds) > 0 && subdocIds[0] != "root" { + return common.NewError(common.ErrInvalidQueryParams) + } + + for _, subdocId := range subdocIds[1:] { + if _, ok := validSubdocIdMap[subdocId]; !ok { + return common.NewError(common.ErrInvalidQueryParams) + } + } + + ifNoneMatch := r.Header.Get(common.HeaderIfNoneMatch) + if len(ifNoneMatch) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + versions := strings.Split(ifNoneMatch, ",") + if len(versions) != len(subdocIds) { + return common.NewError(common.ErrInvalidQueryParams) + } + return nil +} diff --git a/util/validator_test.go b/util/validator_test.go new file mode 100644 index 0000000..83ac7f2 --- /dev/null +++ b/util/validator_test.go @@ -0,0 +1,186 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package util + +import ( + "errors" + "fmt" + "maps" + "net/http" + "net/url" + "testing" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestValidateMac(t *testing.T) { + mac := "001122334455" + assert.Assert(t, ValidateMac(mac)) + + mac = "4444ABCDEF01" + assert.Assert(t, ValidateMac(mac)) + + mac = "00112233445Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "001122334455Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "0H1122334455" + assert.Assert(t, !ValidateMac(mac)) + + for i := 0; i < 10; i++ { + mac := GenerateRandomCpeMac() + assert.Assert(t, ValidateMac(mac)) + } +} + +func TestValidatePokeQuery(t *testing.T) { + values := url.Values{} + + values["doc"] = []string{ + "hello,world", + "primary,telemetry", + } + _, err := ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,hello,world", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,telemetry", + } + _, err = ValidatePokeQuery(values) + assert.NilError(t, err) + + values["doc"] = []string{ + "primary", + } + s, err := ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "telemetry", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "telemetry") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "root") + + values["doc"] = []string{ + "primary", + } + values["route"] = []string{ + "mqtt", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "mqtt") +} + +func TestValidateQueryParams(t *testing.T) { + cpeMac := GenerateRandomCpeMac() + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + validSubdocIdMap["red"] = 1 + validSubdocIdMap["orange"] = 1 + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + fields := make(log.Fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,lan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.NilError(t, err) + + // case 9 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,red,orange", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456,678") + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.NilError(t, err) +}