From 8fed9d18d1e6ec3acc3fed3abc774837682985c6 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Mon, 1 Sep 2025 14:03:29 +0200 Subject: [PATCH 1/4] Split felix server into watcher/handler This patch splits the felix server in two pieces: - a felix watcher placed under `agent/watchers/felix` - a felix server placed under `agent/felix` The former will have only the responsibility of watching and submitting events into a single event queue. The latter will receive the event in a single goroutine and proceed to program VPP as a single thred. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 50 +- calico-vpp-agent/cni/cni_pod_test.go | 9 +- calico-vpp-agent/cni/cni_server.go | 35 +- calico-vpp-agent/cni/network_vpp.go | 9 +- calico-vpp-agent/cni/network_vpp_routes.go | 17 +- calico-vpp-agent/common/pubsub.go | 16 +- calico-vpp-agent/common/types.go | 53 + .../connectivity/connectivity_server.go | 23 +- calico-vpp-agent/felix/cache/cache.go | 98 + calico-vpp-agent/felix/felix_server.go | 1851 ++--------------- calico-vpp-agent/felix/felix_server_test.go | 249 ++- calico-vpp-agent/felix/felixconfig.go | 89 + calico-vpp-agent/felix/ipam.go | 146 ++ .../felix/{ => policies}/host_endpoint.go | 233 ++- .../felix/policies/hostmetadata.go | 133 ++ .../felix/{ => policies}/ipset.go | 60 +- .../felix/policies/policies_handler.go | 375 ++++ .../felix/policies/policies_init.go | 300 +++ .../felix/{ => policies}/policy.go | 148 +- .../felix/{ => policies}/policy_state.go | 2 +- calico-vpp-agent/felix/{ => policies}/rule.go | 8 +- calico-vpp-agent/felix/policies/utils.go | 82 + .../felix/policies/workload_endpoint.go | 278 +++ calico-vpp-agent/felix/workload_endpoint.go | 174 -- calico-vpp-agent/routing/bgp_watcher.go | 6 +- calico-vpp-agent/routing/routing_server.go | 4 +- .../tests/mocks/pubsub_handler.go | 10 +- calico-vpp-agent/testutils/testutils.go | 3 +- .../watchers/bgp_configuration_watcher.go | 10 +- .../{felix/messages.go => watchers/felix.go} | 157 +- calico-vpp-agent/watchers/net_watcher.go | 30 +- calico-vpp-agent/watchers/peers_watcher.go | 10 +- .../watchers/uplink_route_watcher.go | 18 +- config/config.go | 3 + 34 files changed, 2471 insertions(+), 2218 deletions(-) create mode 100644 calico-vpp-agent/common/types.go create mode 100644 calico-vpp-agent/felix/cache/cache.go create mode 100644 calico-vpp-agent/felix/felixconfig.go create mode 100644 calico-vpp-agent/felix/ipam.go rename calico-vpp-agent/felix/{ => policies}/host_endpoint.go (50%) create mode 100644 calico-vpp-agent/felix/policies/hostmetadata.go rename calico-vpp-agent/felix/{ => policies}/ipset.go (80%) create mode 100644 calico-vpp-agent/felix/policies/policies_handler.go create mode 100644 calico-vpp-agent/felix/policies/policies_init.go rename calico-vpp-agent/felix/{ => policies}/policy.go (59%) rename calico-vpp-agent/felix/{ => policies}/policy_state.go (98%) rename calico-vpp-agent/felix/{ => policies}/rule.go (98%) create mode 100644 calico-vpp-agent/felix/policies/utils.go create mode 100644 calico-vpp-agent/felix/policies/workload_endpoint.go delete mode 100644 calico-vpp-agent/felix/workload_endpoint.go rename calico-vpp-agent/{felix/messages.go => watchers/felix.go} (51%) diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index aa6170f77..f8916daf2 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -152,11 +152,9 @@ func main() { routingServer := routing.NewRoutingServer(vpp, bgpServer, log.WithFields(logrus.Fields{"component": "routing"})) serviceServer := services.NewServiceServer(vpp, k8sclient, log.WithFields(logrus.Fields{"component": "services"})) localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) - felixServer, err := felix.NewFelixServer(vpp, log.WithFields(logrus.Fields{"component": "felix"})) - if err != nil { - log.Fatalf("Failed to create felix server %s", err) - } - err = felix.InstallFelixPlugin() + felixServer := felix.NewFelixServer(vpp, clientv3, log.WithFields(logrus.Fields{"component": "policy"})) + felixWatcher := watchers.NewFelixWatcher(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "felix watcher"})) + err = watchers.InstallFelixPlugin() if err != nil { log.Fatalf("could not install felix plugin: %s", err) } @@ -173,8 +171,10 @@ func main() { peerWatcher.SetBGPConf(bgpConf) routingServer.SetBGPConf(bgpConf) serviceServer.SetBGPConf(bgpConf) + felixServer.SetBGPConf(bgpConf) Go(felixServer.ServeFelix) + Go(felixWatcher.WatchFelix) /* * Mark as unhealthy while waiting for Felix config @@ -186,19 +186,17 @@ func main() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - var felixConfig interface{} - var ourBGPSpec interface{} + var felixConfig *felixconfig.Config + var ourBGPSpec *common.LocalNodeSpec felixConfigReceived := false bgpSpecReceived := false for !felixConfigReceived || !bgpSpecReceived { select { - case value := <-felixServer.FelixConfigChan: - felixConfig = value + case felixConfig = <-felixServer.FelixConfigChan: felixConfigReceived = true log.Info("FelixConfig received from calico pod") - case value := <-felixServer.GotOurNodeBGPchan: - ourBGPSpec = value + case ourBGPSpec = <-felixServer.GotOurNodeBGPchan(): bgpSpecReceived = true log.Info("BGP spec received from node add") case <-t.Dying(): @@ -218,19 +216,13 @@ func main() { healthServer.SetComponentStatus(health.ComponentFelix, true, "Felix config received") log.Info("Felix configuration received") - if ourBGPSpec != nil { - bgpSpec, ok := ourBGPSpec.(*common.LocalNodeSpec) - if !ok { - panic("ourBGPSpec is not *common.LocalNodeSpec") - } - prefixWatcher.SetOurBGPSpec(bgpSpec) - connectivityServer.SetOurBGPSpec(bgpSpec) - routingServer.SetOurBGPSpec(bgpSpec) - serviceServer.SetOurBGPSpec(bgpSpec) - localSIDWatcher.SetOurBGPSpec(bgpSpec) - netWatcher.SetOurBGPSpec(bgpSpec) - cniServer.SetOurBGPSpec(bgpSpec) - } + prefixWatcher.SetOurBGPSpec(ourBGPSpec) + connectivityServer.SetOurBGPSpec(ourBGPSpec) + routingServer.SetOurBGPSpec(ourBGPSpec) + serviceServer.SetOurBGPSpec(ourBGPSpec) + localSIDWatcher.SetOurBGPSpec(ourBGPSpec) + netWatcher.SetOurBGPSpec(ourBGPSpec) + cniServer.SetOurBGPSpec(ourBGPSpec) if *config.GetCalicoVppFeatureGates().MultinetEnabled { Go(netWatcher.WatchNetworks) @@ -244,14 +236,8 @@ func main() { } } - if felixConfig != nil { - felixCfg, ok := felixConfig.(*felixconfig.Config) - if !ok { - panic("ourBGPSpec is not *felixconfig.Config") - } - cniServer.SetFelixConfig(felixCfg) - connectivityServer.SetFelixConfig(felixCfg) - } + cniServer.SetFelixConfig(felixConfig) + connectivityServer.SetFelixConfig(felixConfig) Go(routeWatcher.WatchRoutes) Go(linkWatcher.WatchLinks) diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/cni/cni_pod_test.go index 37af3a18c..a79d89073 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/cni/cni_pod_test.go @@ -35,7 +35,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -328,7 +327,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Context("With MultiNet configuration (and multinet VRF and loopback already configured)", func() { var ( - networkDefinition *watchers.NetworkDefinition + networkDefinition *common.NetworkDefinition pubSubHandlerMock *mocks.PubSubHandlerMock ) @@ -360,9 +359,9 @@ var _ = Describe("Pod-related functionality of CNI", func() { } // NetworkDefinition CRD information caught by NetWatcher and send with additional information // (VRF and loopback created by watcher) to the cni server as common.NetAdded CalicoVPPEvent - networkDefinition = &watchers.NetworkDefinition{ - VRF: watchers.VRF{Tables: tables}, - PodVRF: watchers.VRF{Tables: podTables}, + networkDefinition = &common.NetworkDefinition{ + VRF: common.VRF{Tables: tables}, + PodVRF: common.VRF{Tables: podTables}, Vni: uint32(0), // important only for VXLAN tunnel going out of node Name: networkName, Range: "10.1.1.0/24", // IP range for secondary network defined by multinet diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go index 5b1bcc778..d0b94b063 100644 --- a/calico-vpp-agent/cni/cni_server.go +++ b/calico-vpp-agent/cni/cni_server.go @@ -36,7 +36,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -53,7 +52,7 @@ type Server struct { podInterfaceMap map[string]model.LocalPodSpec lock sync.Mutex /* protects Add/DelVppInterace/RescanState */ - cniEventChan chan common.CalicoVppEvent + cniEventChan chan any memifDriver *podinterface.MemifPodInterfaceDriver tuntapDriver *podinterface.TunTapPodInterfaceDriver @@ -65,7 +64,7 @@ type Server struct { RedirectToHostClassifyTableIndex uint32 networkDefinitions sync.Map - cniMultinetEventChan chan common.CalicoVppEvent + cniMultinetEventChan chan any nodeBGPSpec *common.LocalNodeSpec } @@ -96,9 +95,9 @@ func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cnipro if !ok { return nil, fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("Value is not of type *watchers.NetworkDefinition") + panic("Value is not of type *common.NetworkDefinition") } _, route, err := net.ParseCIDR(networkDefinition.Range) if err == nil { @@ -292,7 +291,7 @@ func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, log: log, felixServerIpam: felixServerIpam, - cniEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + cniEventChan: make(chan any, common.ChanSize), grpcServer: grpc.NewServer(), podInterfaceMap: make(map[string]model.LocalPodSpec), @@ -301,7 +300,7 @@ func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, log, felixServerIpam), loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, log, felixServerIpam), - cniMultinetEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + cniMultinetEventChan: make(chan any, common.ChanSize), } reg := common.RegisterHandler(server.cniEventChan, "CNI server events") reg.ExpectEvents( @@ -322,7 +321,11 @@ forloop: select { case <-t.Dying(): break forloop - case evt := <-s.cniEventChan: + case msg := <-s.cniEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.FelixConfChanged: if new, _ := evt.New.(*felixConfig.Config); new != nil { @@ -444,21 +447,25 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error { case <-t.Dying(): s.log.Warn("Cni server asked to exit") return - case event := <-s.cniMultinetEventChan: + case msg := <-s.cniMultinetEventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch event.Type { case common.NetsSynced: netsSynced <- true case common.NetAddedOrUpdated: - netDef, ok := event.New.(*watchers.NetworkDefinition) + netDef, ok := event.New.(*common.NetworkDefinition) if !ok { - s.log.Errorf("event.New is not a *watchers.NetworkDefinition %v", event.New) + s.log.Errorf("event.New is not a *common.NetworkDefinition %v", event.New) continue } s.networkDefinitions.Store(netDef.Name, netDef) case common.NetDeleted: - netDef, ok := event.Old.(*watchers.NetworkDefinition) + netDef, ok := event.Old.(*common.NetworkDefinition) if !ok { - s.log.Errorf("event.Old is not a *watchers.NetworkDefinition %v", event.Old) + s.log.Errorf("event.Old is not a *common.NetworkDefinition %v", event.Old) continue } s.networkDefinitions.Delete(netDef.Name) @@ -498,6 +505,6 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error { // ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. // The usage is mainly for testing purposes. -func (s *Server) ForceAddingNetworkDefinition(networkDefinition *watchers.NetworkDefinition) { +func (s *Server) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { s.networkDefinitions.Store(networkDefinition.Name, networkDefinition) } diff --git a/calico-vpp-agent/cni/network_vpp.go b/calico-vpp-agent/cni/network_vpp.go index 88c7319c6..eb8f89340 100644 --- a/calico-vpp-agent/cni/network_vpp.go +++ b/calico-vpp-agent/cni/network_vpp.go @@ -24,7 +24,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -248,9 +247,9 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vni = networkDefinition.Vni } @@ -320,9 +319,9 @@ func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { if !ok { deleteLocalPodAddress = false } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vni = networkDefinition.Vni } diff --git a/calico-vpp-agent/cni/network_vpp_routes.go b/calico-vpp-agent/cni/network_vpp_routes.go index 5e7ecfcdf..345e34045 100644 --- a/calico-vpp-agent/cni/network_vpp_routes.go +++ b/calico-vpp-agent/cni/network_vpp_routes.go @@ -20,7 +20,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -37,9 +36,9 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } table = networkDefinition.VRF.Tables[idx] } @@ -88,9 +87,9 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } table = networkDefinition.VRF.Tables[idx] } @@ -242,9 +241,9 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu if !ok { return errors.Errorf("network not found %s", podSpec.NetworkName) } - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vrfIndex = networkDefinition.PodVRF.Tables[idx] } @@ -402,9 +401,9 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vrfIndex = networkDefinition.PodVRF.Tables[idx] } diff --git a/calico-vpp-agent/common/pubsub.go b/calico-vpp-agent/common/pubsub.go index daef558ca..1ceb72484 100644 --- a/calico-vpp-agent/common/pubsub.go +++ b/calico-vpp-agent/common/pubsub.go @@ -85,18 +85,15 @@ type PubSubHandlerRegistration struct { /* Name for the registration, for logging & debugging */ name string /* Channel where to send events */ - channel chan CalicoVppEvent + channel chan any /* Receive only these events. If empty we'll receive all */ expectedEvents map[CalicoVppEventType]bool - /* Receive all events */ - expectAllEvents bool } func (reg *PubSubHandlerRegistration) ExpectEvents(eventTypes ...CalicoVppEventType) { for _, eventType := range eventTypes { reg.expectedEvents[eventType] = true } - reg.expectAllEvents = false } type PubSub struct { @@ -104,12 +101,11 @@ type PubSub struct { pubSubHandlerRegistrations []*PubSubHandlerRegistration } -func RegisterHandler(channel chan CalicoVppEvent, name string) *PubSubHandlerRegistration { +func RegisterHandler(channel chan any, name string) *PubSubHandlerRegistration { reg := &PubSubHandlerRegistration{ - channel: channel, - name: name, - expectedEvents: make(map[CalicoVppEventType]bool), - expectAllEvents: true, /* By default receive everything, unless we ask for a filter */ + channel: channel, + name: name, + expectedEvents: make(map[CalicoVppEventType]bool), } ThePubSub.pubSubHandlerRegistrations = append(ThePubSub.pubSubHandlerRegistrations, reg) return reg @@ -128,7 +124,7 @@ func redactPassword(event CalicoVppEvent) string { func SendEvent(event CalicoVppEvent) { ThePubSub.log.Debugf("Broadcasting event %s", redactPassword(event)) for _, reg := range ThePubSub.pubSubHandlerRegistrations { - if reg.expectAllEvents || reg.expectedEvents[event.Type] { + if reg.expectedEvents[event.Type] { reg.channel <- event } } diff --git a/calico-vpp-agent/common/types.go b/calico-vpp-agent/common/types.go new file mode 100644 index 000000000..30b36e2c4 --- /dev/null +++ b/calico-vpp-agent/common/types.go @@ -0,0 +1,53 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +type VRF struct { + Tables [2]uint32 // one for ipv4, one for ipv6 +} + +type NetworkDefinition struct { + // VRF is the main table used for the corresponding physical network + VRF VRF + // PodVRF is the table used for the pods in the corresponding physical network + PodVRF VRF + Vni uint32 + PhysicalNetworkName string + Name string + Range string + NetAttachDefs string +} + +// FelixSocketSyncState describes the status of the +// felix socket connection. It applies mostly to policies +type FelixSocketSyncState int + +const ( + StateDisconnected FelixSocketSyncState = iota + StateConnected + StateSyncing + StateInSync +) + +func (state FelixSocketSyncState) IsPending() bool { + return state != StateInSync +} + +// FelixSocketStateChanged is emitted when the state +// of the socket changed. Typically connection and disconnection. +type FelixSocketStateChanged struct { + NewState FelixSocketSyncState +} diff --git a/calico-vpp-agent/connectivity/connectivity_server.go b/calico-vpp-agent/connectivity/connectivity_server.go index 70e2a034f..ded4adb17 100644 --- a/calico-vpp-agent/connectivity/connectivity_server.go +++ b/calico-vpp-agent/connectivity/connectivity_server.go @@ -27,7 +27,6 @@ import ( "gopkg.in/tomb.v2" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) @@ -45,9 +44,9 @@ type ConnectivityServer struct { felixConfig *felixConfig.Config nodeByAddr map[string]common.LocalNodeSpec - connectivityEventChan chan common.CalicoVppEvent + connectivityEventChan chan any - networks map[uint32]watchers.NetworkDefinition + networks map[uint32]common.NetworkDefinition } type change uint8 @@ -73,9 +72,9 @@ func NewConnectivityServer(vpp *vpplink.VppLink, felixServerIpam common.FelixSer felixServerIpam: felixServerIpam, Clientv3: clientv3, connectivityMap: make(map[string]common.NodeConnectivity), - connectivityEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + connectivityEventChan: make(chan any, common.ChanSize), nodeByAddr: make(map[string]common.LocalNodeSpec), - networks: make(map[uint32]watchers.NetworkDefinition), + networks: make(map[uint32]common.NetworkDefinition), } reg := common.RegisterHandler(server.connectivityEventChan, "connectivity server events") @@ -167,19 +166,23 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error { case <-t.Dying(): s.log.Warn("Connectivity Server asked to stop") return nil - case evt := <-s.connectivityEventChan: + case msg := <-s.connectivityEventChan: /* Note: we will only receive events we ask for when registering the chan */ + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.NetAddedOrUpdated: - new, ok := evt.New.(*watchers.NetworkDefinition) + new, ok := evt.New.(*common.NetworkDefinition) if !ok { - s.log.Errorf("evt.New is not a *watchers.NetworkDefinition %v", evt.New) + s.log.Errorf("evt.New is not a *common.NetworkDefinition %v", evt.New) } s.networks[new.Vni] = *new case common.NetDeleted: - old, ok := evt.Old.(*watchers.NetworkDefinition) + old, ok := evt.Old.(*common.NetworkDefinition) if !ok { - s.log.Errorf("evt.Old is not a *watchers.NetworkDefinition %v", evt.Old) + s.log.Errorf("evt.Old is not a *common.NetworkDefinition %v", evt.Old) } delete(s.networks, old.Vni) case common.ConnectivityAdded: diff --git a/calico-vpp-agent/felix/cache/cache.go b/calico-vpp-agent/felix/cache/cache.go new file mode 100644 index 000000000..80fa4ad1d --- /dev/null +++ b/calico-vpp-agent/felix/cache/cache.go @@ -0,0 +1,98 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "net" + + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" + + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type Cache struct { + log *logrus.Entry + + FelixConfig *felixConfig.Config + NodeByAddr map[string]common.LocalNodeSpec + Networks map[uint32]*common.NetworkDefinition + NetworkDefinitions map[string]*common.NetworkDefinition + IPPoolMap map[string]*proto.IPAMPool + NodeStatesByName map[string]*common.LocalNodeSpec + BGPConf *calicov3.BGPConfigurationSpec +} + +func NewCache(log *logrus.Entry) *Cache { + return &Cache{ + log: log, + NodeByAddr: make(map[string]common.LocalNodeSpec), + FelixConfig: felixConfig.New(), + Networks: make(map[uint32]*common.NetworkDefinition), + NetworkDefinitions: make(map[string]*common.NetworkDefinition), + IPPoolMap: make(map[string]*proto.IPAMPool), + NodeStatesByName: make(map[string]*common.LocalNodeSpec), + } +} + +// match checks whether we have an IP pool which contains the given prefix. +// If we have, it returns the pool. +func (cache *Cache) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { + for _, pool := range cache.IPPoolMap { + in, err := ipamPoolContains(pool, prefix) + if err != nil { + cache.log.Warnf("ipamPoolContains errored: %v", err) + continue + } + if in { + return pool + } + } + cache.log.Warnf("No pool found for %s", prefix) + for k, pool := range cache.IPPoolMap { + cache.log.Debugf("Available %s=%v", k, pool) + } + return nil +} + +// ipamPoolContains returns true if the IPPool contains 'prefix' +func ipamPoolContains(pool *proto.IPAMPool, prefix *net.IPNet) (bool, error) { + _, poolCIDR, _ := net.ParseCIDR(pool.GetCidr()) // this field is validated so this should never error + poolCIDRLen, poolCIDRBits := poolCIDR.Mask.Size() + prefixLen, prefixBits := prefix.Mask.Size() + return poolCIDRBits == prefixBits && poolCIDR.Contains(prefix.IP) && prefixLen >= poolCIDRLen, nil +} + +func (cache *Cache) GetNodeIP4() *net.IP { + if spec, found := cache.NodeStatesByName[*config.NodeName]; found { + if spec.IPv4Address != nil { + return &spec.IPv4Address.IP + } + } + return nil +} + +func (cache *Cache) GetNodeIP6() *net.IP { + if spec, found := cache.NodeStatesByName[*config.NodeName]; found { + if spec.IPv6Address != nil { + return &spec.IPv6Address.IP + } + } + return nil +} diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index 115853f2b..0f125184e 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -16,138 +16,57 @@ package felix import ( - "encoding/json" "fmt" - "io" "net" - "os" - "reflect" - "regexp" - "strings" "sync" "github.com/pkg/errors" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" - "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" - "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -const ( - FelixPluginSrcPath = "/bin/felix-api-proxy" - FelixPluginDstPath = "/var/lib/calico/felix-plugins/felix-api-proxy" -) - -type SyncState int - -const ( - StateDisconnected SyncState = iota - StateConnected - StateSyncing - StateInSync ) -type NodeWatcherRestartError struct{} - -func (e NodeWatcherRestartError) Error() string { - return "node configuration changed, restarting" -} - // Server holds all the data required to configure the policies defined by felix in VPP type Server struct { - log *logrus.Entry - vpp *vpplink.VppLink - - state SyncState - nextSeqNumber uint64 - - endpointsLock sync.Mutex - endpointsInterfaces map[WorkloadEndpointID]map[string]uint32 - - configuredState *PolicyState - pendingState *PolicyState - - /* failSafe policies allow traffic on some ports irrespective of the policy */ - failSafePolicy *Policy - /* workloadToHost may drop traffic that goes from the pods to the host */ - workloadsToHostPolicy *Policy - defaultTap0IngressConf []uint32 - defaultTap0EgressConf []uint32 - /* always allow traffic coming from host to the pods (for healthchecks and so on) */ - // AllowFromHostPolicy persists the policy allowing host --> pod communications. - // See CreateAllowFromHostPolicy definition - AllowFromHostPolicy *Policy - // allPodsIpset persists the ipset containing all the workload endpoints (pods) addresses - allPodsIpset *IPSet - /* allow traffic between uplink/tunnels and tap interfaces */ - allowToHostPolicy *Policy - /* deny all policy for heps with no policies defined */ - ip4 *net.IP - ip6 *net.IP - interfacesMap map[string]interfaceDetails - - felixServerEventChan chan common.CalicoVppEvent - networkDefinitions map[string]*watchers.NetworkDefinition + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache - tunnelSwIfIndexes map[uint32]bool - tunnelSwIfIndexesLock sync.Mutex + felixServerEventChan chan any felixConfigReceived bool - FelixConfigChan chan interface{} - felixConfig *felixConfig.Config + FelixConfigChan chan *felixConfig.Config - ippoolmap map[string]*proto.IPAMPool - ippoolLock sync.RWMutex - - nodeStatesByName map[string]*common.LocalNodeSpec - nodeByWGPublicKey map[string]string - - GotOurNodeBGPchan chan interface{} - GotOurNodeBGPchanOnce sync.Once + ippoolLock sync.RWMutex + policiesHandler *policies.PoliciesHandler prometheusServer *prometheus.PrometheusServer } // NewFelixServer creates a felix server -func NewFelixServer(vpp *vpplink.VppLink, log *logrus.Entry) (*Server, error) { - var err error - +func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *logrus.Entry) *Server { + cache := cache.NewCache(log) server := &Server{ log: log, vpp: vpp, - state: StateDisconnected, - nextSeqNumber: 0, - - endpointsInterfaces: make(map[WorkloadEndpointID]map[string]uint32), + felixServerEventChan: make(chan any, common.ChanSize), - configuredState: NewPolicyState(), - pendingState: NewPolicyState(), - - felixServerEventChan: make(chan common.CalicoVppEvent, common.ChanSize), - - networkDefinitions: make(map[string]*watchers.NetworkDefinition), - - tunnelSwIfIndexes: make(map[uint32]bool), felixConfigReceived: false, - FelixConfigChan: make(chan interface{}), - felixConfig: felixConfig.New(), - - ippoolmap: make(map[string]*proto.IPAMPool), + FelixConfigChan: make(chan *felixConfig.Config), - nodeStatesByName: make(map[string]*common.LocalNodeSpec), - GotOurNodeBGPchan: make(chan interface{}), + cache: cache, + policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), prometheusServer: prometheus.NewPrometheusServer(vpp, log.WithFields(logrus.Fields{"component": "prometheus"})), } @@ -160,1259 +79,35 @@ func NewFelixServer(vpp *vpplink.VppLink, log *logrus.Entry) (*Server, error) { common.TunnelDeleted, common.NetAddedOrUpdated, common.NetDeleted, + common.ConnectivityAdded, + common.ConnectivityDeleted, + common.SRv6PolicyAdded, + common.SRv6PolicyDeleted, ) - server.interfacesMap, err = server.mapTagToInterfaceDetails() - if err != nil { - return nil, errors.Wrapf(err, "error in mapping uplink to tap interfaces") - } - - // Cleanup potentially left over socket - err = os.RemoveAll(config.FelixDataplaneSocket) - if err != nil { - return nil, errors.Wrapf(err, "Could not delete socket %s", config.FelixDataplaneSocket) - } - - return server, nil -} - -type interfaceDetails struct { - tapIndex uint32 - uplinkIndex uint32 - addresses []string -} - -func (s *Server) mapTagToInterfaceDetails() (tagIfDetails map[string]interfaceDetails, err error) { - tagIfDetails = make(map[string]interfaceDetails) - uplinkSwifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("main-") - if err != nil { - return nil, err - } - tapSwifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") - if err != nil { - return nil, err - } - for intf, uplink := range uplinkSwifindexes { - tap, found := tapSwifindexes["host-"+intf[5:]] - if found { - ip4adds, err := s.vpp.AddrList(uplink, false) - if err != nil { - return nil, err - } - ip6adds, err := s.vpp.AddrList(uplink, true) - if err != nil { - return nil, err - } - adds := append(ip4adds, ip6adds...) - addresses := []string{} - for _, add := range adds { - addresses = append(addresses, add.IPNet.IP.String()) - } - tagIfDetails[intf[5:]] = interfaceDetails{tap, uplink, addresses} - } else { - return nil, errors.Errorf("uplink interface %d not corresponding to a tap interface", uplink) - } - } - return tagIfDetails, nil -} - -func InstallFelixPlugin() (err error) { - err = os.RemoveAll(FelixPluginDstPath) - if err != nil { - logrus.Warnf("Could not delete %s: %v", FelixPluginDstPath, err) - } - - in, err := os.Open(FelixPluginSrcPath) - if err != nil { - return errors.Wrap(err, "cannot open felix plugin to copy") - } - defer in.Close() - - out, err := os.OpenFile(FelixPluginDstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) - if err != nil { - return errors.Wrap(err, "cannot open felix plugin to write") - } - defer func() { - cerr := out.Close() - if err == nil { - err = errors.Wrap(cerr, "cannot close felix plugin file") - } - }() - if _, err = io.Copy(out, in); err != nil { - return errors.Wrap(err, "cannot copy data") - } - err = out.Sync() - return errors.Wrapf(err, "could not sync felix plugin changes") -} - -func (s *Server) getEndpointToHostAction() types.RuleAction { - if strings.ToUpper(s.felixConfig.DefaultEndpointToHostAction) == "ACCEPT" { - return types.ActionAllow - } - return types.ActionDeny -} - -// workloadAdded is called by the CNI server when a container interface is created, -// either during startup when reconnecting the interfaces, or when a new pod is created -func (s *Server) workloadAdded(id *WorkloadEndpointID, swIfIndex uint32, ifName string, containerIPs []*net.IPNet) { - // TODO: Send WorkloadEndpointStatusUpdate to felix - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - intf, existing := s.endpointsInterfaces[*id] - - if existing { - for _, exInt := range intf { - if swIfIndex == exInt { - return - } - } - // VPP restarted and interfaces are being reconnected - s.log.Warnf("workload endpoint changed interfaces, did VPP restart? %v %v -> %d", id, intf, swIfIndex) - s.endpointsInterfaces[*id][ifName] = swIfIndex - } - - s.log.Infof("policy(add) Workload id=%v swIfIndex=%d", id, swIfIndex) - if s.endpointsInterfaces[*id] == nil { - s.endpointsInterfaces[*id] = map[string]uint32{ifName: swIfIndex} - } else { - s.endpointsInterfaces[*id][ifName] = swIfIndex - } - - if s.state == StateInSync { - wep, ok := s.configuredState.WorkloadEndpoints[*id] - if !ok { - s.log.Infof("not creating wep in workloadadded") - // Nothing to configure - } else { - s.log.Infof("creating wep in workloadadded") - err := wep.Create(s.vpp, []uint32{swIfIndex}, s.configuredState, id.Network) - if err != nil { - s.log.Errorf("Error processing workload addition: %s", err) - } - } - } - // EndpointToHostAction - allMembers := []string{} - for _, containerIP := range containerIPs { - allMembers = append(allMembers, containerIP.IP.String()) - } - err := s.allPodsIpset.AddMembers(allMembers, true, s.vpp) - if err != nil { - s.log.Errorf("Error processing workload addition: %s", err) - } -} - -// WorkloadRemoved is called by the CNI server when the interface of a pod is deleted -func (s *Server) WorkloadRemoved(id *WorkloadEndpointID, containerIPs []*net.IPNet) { - // TODO: Send WorkloadEndpointStatusRemove to felix - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - _, existing := s.endpointsInterfaces[*id] - if !existing { - s.log.Warnf("nonexistent workload endpoint removed %v", id) - return - } - s.log.Infof("policy(del) workload id=%v", id) - - if s.state == StateInSync { - wep, ok := s.configuredState.WorkloadEndpoints[*id] - if !ok { - // Nothing to clean up - } else { - err := wep.Delete(s.vpp) - if err != nil { - s.log.Errorf("Error processing workload removal: %s", err) - } - } - } - delete(s.endpointsInterfaces, *id) - // EndpointToHostAction - allMembers := []string{} - for _, containerIP := range containerIPs { - allMembers = append(allMembers, containerIP.IP.String()) - } - err := s.allPodsIpset.RemoveMembers(allMembers, true, s.vpp) - if err != nil { - s.log.Errorf("Error processing workload remove: %s", err) - } -} - -func (s *Server) handleFelixServerEvents(evt common.CalicoVppEvent) error { - /* Note: we will only receive events we ask for when registering the chan */ - switch evt.Type { - case common.NetAddedOrUpdated: - netDef, ok := evt.New.(*watchers.NetworkDefinition) - if !ok { - return fmt.Errorf("evt.New is not a (*watchers.NetworkDefinition) %v", evt.New) - } - s.networkDefinitions[netDef.Name] = netDef - case common.NetDeleted: - netDef, ok := evt.Old.(*watchers.NetworkDefinition) - if !ok { - return fmt.Errorf("evt.Old is not a (*watchers.NetworkDefinition) %v", evt.Old) - } - delete(s.networkDefinitions, netDef.Name) - case common.PodAdded: - podSpec, ok := evt.New.(*model.LocalPodSpec) - if !ok { - return fmt.Errorf("evt.New is not a (*model.LocalPodSpec) %v", evt.New) - } - swIfIndex := podSpec.TunTapSwIfIndex - if swIfIndex == vpplink.InvalidID { - swIfIndex = podSpec.MemifSwIfIndex - } - s.workloadAdded(&WorkloadEndpointID{ - OrchestratorID: podSpec.OrchestratorID, - WorkloadID: podSpec.WorkloadID, - EndpointID: podSpec.EndpointID, - Network: podSpec.NetworkName, - }, swIfIndex, podSpec.InterfaceName, podSpec.GetContainerIPs()) - // Notify prometheus server of pod addition - s.prometheusServer.OnPodAdded(podSpec) - case common.PodDeleted: - podSpec, ok := evt.Old.(*model.LocalPodSpec) - if !ok { - return fmt.Errorf("evt.Old is not a (*model.LocalPodSpec) %v", evt.Old) - } - if podSpec != nil { - s.WorkloadRemoved(&WorkloadEndpointID{ - OrchestratorID: podSpec.OrchestratorID, - WorkloadID: podSpec.WorkloadID, - EndpointID: podSpec.EndpointID, - Network: podSpec.NetworkName, - }, podSpec.GetContainerIPs()) - // Notify prometheus server of pod deletion - s.prometheusServer.OnPodDeleted(podSpec) - } - case common.TunnelAdded: - swIfIndex, ok := evt.New.(uint32) - if !ok { - return fmt.Errorf("evt.New not a uint32 %v", evt.New) - } - - s.tunnelSwIfIndexesLock.Lock() - s.tunnelSwIfIndexes[swIfIndex] = true - s.tunnelSwIfIndexesLock.Unlock() - - pending := true - switch s.state { - case StateSyncing, StateConnected: - case StateInSync: - pending = false - default: - return fmt.Errorf("got tunnel %d add but not in syncing or synced state", swIfIndex) - } - state := s.currentState(pending) - for _, h := range state.HostEndpoints { - err := h.handleTunnelChange(swIfIndex, true /* isAdd */, pending) - if err != nil { - return err - } - } - case common.TunnelDeleted: - swIfIndex, ok := evt.Old.(uint32) - if !ok { - return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) - } - - s.tunnelSwIfIndexesLock.Lock() - delete(s.tunnelSwIfIndexes, swIfIndex) - s.tunnelSwIfIndexesLock.Unlock() - - pending := true - switch s.state { - case StateSyncing, StateConnected: - case StateInSync: - pending = false - default: - return fmt.Errorf("got tunnel %d del but not in syncing or synced state", swIfIndex) - } - state := s.currentState(pending) - for _, h := range state.HostEndpoints { - err := h.handleTunnelChange(swIfIndex, false /* isAdd */, pending) - if err != nil { - return err - } - } - } - return nil -} - -// Serve runs the felix server -func (s *Server) ServeFelix(t *tomb.Tomb) error { - s.log.Info("Starting felix server") - - // Start prometheus server - if t.Alive() { - t.Go(func() error { - err := s.prometheusServer.ServePrometheus(t) - if err != nil { - s.log.Warnf("Prometheus server errored with %s", err) - } - return err - }) - } - - listener, err := net.Listen("unix", config.FelixDataplaneSocket) - if err != nil { - return errors.Wrapf(err, "Could not bind to unix://%s", config.FelixDataplaneSocket) - } - defer func() { - listener.Close() - os.RemoveAll(config.FelixDataplaneSocket) - }() - err = s.createAllPodsIpset() - if err != nil { - return errors.Wrap(err, "Error in createallPodsIpset") - } - err = s.createEndpointToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createEndpointToHostPolicy") - } - err = s.createAllowFromHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in creating AllowFromHostPolicy") - } - err = s.createAllowToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createAllowToHostPolicy") - } - err = s.createFailSafePolicies() - if err != nil { - return errors.Wrap(err, "Error in createFailSafePolicies") - } - for { - s.state = StateDisconnected - // Accept only one connection - conn, err := listener.Accept() - if err != nil { - return errors.Wrap(err, "cannot accept felix client connection") - } - s.log.Infof("Accepted connection from felix") - s.state = StateConnected - - felixUpdates := s.MessageReader(conn) - innerLoop: - for { - select { - case <-t.Dying(): - s.log.Warn("Felix server exiting") - err = conn.Close() - if err != nil { - s.log.WithError(err).Warn("Error closing unix connection to felix API proxy") - } - s.log.Infof("Waiting for SyncFelix to stop...") - return nil - case evt := <-s.felixServerEventChan: - err = s.handleFelixServerEvents(evt) - if err != nil { - s.log.WithError(err).Warn("Error handling FelixServerEvents") - } - // <-felixUpdates & handleFelixUpdate does the bulk of the policy sync job. It starts by reconciling the current - // configured state in VPP (empty at first) with what is sent by felix, and once both are in - // sync, it keeps processing felix updates. It also sends endpoint updates to felix when the - // CNI component adds or deletes container interfaces. - case msg, ok := <-felixUpdates: - if !ok { - s.log.Infof("Felix MessageReader closed") - break innerLoop - } - err = s.handleFelixUpdate(msg) - if err != nil { - switch err.(type) { - case NodeWatcherRestartError: - return err - default: - s.log.WithError(err).Error("Error processing update from felix, restarting") - // TODO: Restart VPP as well? State is left over there... - break innerLoop - } - } - } - } - err = conn.Close() - if err != nil { - s.log.WithError(err).Warn("Error closing unix connection to felix API proxy") - } - s.log.Infof("SyncFelix exited, reconnecting to felix") - } -} - -func (s *Server) handleFelixUpdate(msg interface{}) (err error) { - s.log.Debugf("Got message from felix: %#v", msg) - switch m := msg.(type) { - case *proto.ConfigUpdate: - err = s.handleConfigUpdate(m) - case *proto.InSync: - err = s.handleInSync(m) - default: - pending := true - switch s.state { - case StateSyncing: - case StateInSync: - pending = false - default: - return fmt.Errorf("got message %#v but not in syncing or synced state", m) - } - switch m := msg.(type) { - case *proto.IPSetUpdate: - err = s.handleIpsetUpdate(m, pending) - case *proto.IPSetDeltaUpdate: - err = s.handleIpsetDeltaUpdate(m, pending) - case *proto.IPSetRemove: - err = s.handleIpsetRemove(m, pending) - case *proto.ActivePolicyUpdate: - err = s.handleActivePolicyUpdate(m, pending) - case *proto.ActivePolicyRemove: - err = s.handleActivePolicyRemove(m, pending) - case *proto.ActiveProfileUpdate: - err = s.handleActiveProfileUpdate(m, pending) - case *proto.ActiveProfileRemove: - err = s.handleActiveProfileRemove(m, pending) - case *proto.HostEndpointUpdate: - err = s.handleHostEndpointUpdate(m, pending) - case *proto.HostEndpointRemove: - err = s.handleHostEndpointRemove(m, pending) - case *proto.WorkloadEndpointUpdate: - err = s.handleWorkloadEndpointUpdate(m, pending) - case *proto.WorkloadEndpointRemove: - err = s.handleWorkloadEndpointRemove(m, pending) - case *proto.HostMetadataUpdate: - err = s.handleHostMetadataUpdate(m, pending) - case *proto.HostMetadataRemove: - err = s.handleHostMetadataRemove(m, pending) - case *proto.HostMetadataV4V6Update: - err = s.handleHostMetadataV4V6Update(m, pending) - case *proto.HostMetadataV4V6Remove: - err = s.handleHostMetadataV4V6Remove(m, pending) - case *proto.IPAMPoolUpdate: - err = s.handleIpamPoolUpdate(m, pending) - case *proto.IPAMPoolRemove: - err = s.handleIpamPoolRemove(m, pending) - case *proto.ServiceAccountUpdate: - err = s.handleServiceAccountUpdate(m, pending) - case *proto.ServiceAccountRemove: - err = s.handleServiceAccountRemove(m, pending) - case *proto.NamespaceUpdate: - err = s.handleNamespaceUpdate(m, pending) - case *proto.NamespaceRemove: - err = s.handleNamespaceRemove(m, pending) - case *proto.GlobalBGPConfigUpdate: - err = s.handleGlobalBGPConfigUpdate(m, pending) - case *proto.WireguardEndpointUpdate: - err = s.handleWireguardEndpointUpdate(m, pending) - case *proto.WireguardEndpointRemove: - err = s.handleWireguardEndpointRemove(m, pending) - default: - s.log.Warnf("Unhandled message from felix: %v", m) - } - } - return err -} - -func (s *Server) currentState(pending bool) *PolicyState { - if pending { - return s.pendingState - } - return s.configuredState -} - -/** - * remove add the fields of type `file` we dont need and for which the - * parsing will fail - * - * This logic is extracted from `loadParams` in [0] - * [0] projectcalico/felix/config/config_params.go:Config - * it applies the regex only on the reflected struct definition, - * not on the live data. - * - **/ -func removeFelixConfigFileField(rawData map[string]string) { - config := felixConfig.Config{} - kind := reflect.TypeOf(config) - metaRegexp := regexp.MustCompile(`^([^;(]+)(?:\(([^)]*)\))?;` + - `([^;]*)(?:;` + - `([^;]*))?$`) - for ii := 0; ii < kind.NumField(); ii++ { - field := kind.Field(ii) - tag := field.Tag.Get("config") - if tag == "" { - continue - } - captures := metaRegexp.FindStringSubmatch(tag) - kind := captures[1] // Type: "int|oneof|bool|port-list|..." - if kind == "file" { - delete(rawData, field.Name) - } - } -} - -// the msg.Config map[string]string is the serialized object -// projectcalico/felix/config/config_params.go:Config -func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { - if s.state != StateConnected { - return fmt.Errorf("received ConfigUpdate but server is not in Connected state! state: %v", s.state) - } - s.log.Infof("Got config from felix: %+v", msg) - s.state = StateSyncing - - oldFelixConfig := s.felixConfig - removeFelixConfigFileField(msg.Config) - s.felixConfig = felixConfig.New() - _, err = s.felixConfig.UpdateFrom(msg.Config, felixConfig.InternalOverride) - if err != nil { - return err - } - changed := !reflect.DeepEqual(oldFelixConfig.RawValues(), s.felixConfig.RawValues()) - - // Note: This function will be called each time the Felix config changes. - // If we start handling config settings that require agent restart, - // we'll need to add a mechanism for that - if !s.felixConfigReceived { - s.felixConfigReceived = true - s.FelixConfigChan <- s.felixConfig - } - - if !changed { - return nil - } - - common.SendEvent(common.CalicoVppEvent{ - Type: common.FelixConfChanged, - New: s.felixConfig, - Old: oldFelixConfig, - }) - - if s.felixConfig.DefaultEndpointToHostAction != oldFelixConfig.DefaultEndpointToHostAction { - s.log.Infof("Change in EndpointToHostAction to %+v", s.getEndpointToHostAction()) - workloadsToHostAllowRule := &Rule{ - VppID: types.InvalidID, - Rule: &types.Rule{ - Action: s.getEndpointToHostAction(), - }, - SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - policy := s.workloadsToHostPolicy.DeepCopy() - policy.InboundRules = []*Rule{workloadsToHostAllowRule} - err := s.workloadsToHostPolicy.Update(s.vpp, policy, - &PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}}) - if err != nil { - return errors.Wrap(err, "error updating workloadsToHostPolicy") - } - } - if !protoPortListEqual(s.felixConfig.FailsafeInboundHostPorts, oldFelixConfig.FailsafeInboundHostPorts) || - !protoPortListEqual(s.felixConfig.FailsafeOutboundHostPorts, oldFelixConfig.FailsafeOutboundHostPorts) { - err = s.createFailSafePolicies() - if err != nil { - return errors.Wrap(err, "error updating FailSafePolicies") - } - } - - return nil -} - -func protoPortListEqual(a, b []felixConfig.ProtoPort) bool { - if len(a) != len(b) { - return false - } - for i, elemA := range a { - elemB := b[i] - if elemA.Net != elemB.Net { - return false - } - if elemA.Protocol != elemB.Protocol { - return false - } - if elemA.Port != elemB.Port { - return false - } - } - return true -} - -func (s *Server) handleInSync(msg *proto.InSync) (err error) { - if s.state != StateSyncing { - return fmt.Errorf("received InSync but state was not syncing") - } - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - s.state = StateInSync - s.log.Infof("Policies now in sync") - return s.applyPendingState() -} - -func (s *Server) handleIpsetUpdate(msg *proto.IPSetUpdate, pending bool) (err error) { - ips, err := fromIPSetUpdate(msg) - if err != nil { - return errors.Wrap(err, "cannot process IPSetUpdate") - } - state := s.currentState(pending) - existing, ok := state.IPSets[msg.GetId()] - if ok { - err = existing.ReplaceMembers(ips, !pending, s.vpp) - if err != nil { - return errors.Wrapf(err, "cannot replace ipset for ID %s", msg.GetId()) - } - s.log.Debugf("Handled ipset replacement for ID %s; pending=%t [%s]", msg.GetId(), pending, existing) - return nil - } - if !pending { - err = ips.Create(s.vpp) - if err != nil { - return errors.Wrapf(err, "cannot create ipset %s", msg.GetId()) - } - } - state.IPSets[msg.GetId()] = ips - s.log.Debugf("Handled Ipset Update pending=%t id=%s %s", pending, msg.GetId(), ips) - return nil -} - -func (s *Server) handleIpsetDeltaUpdate(msg *proto.IPSetDeltaUpdate, pending bool) (err error) { - ips, ok := s.currentState(pending).IPSets[msg.GetId()] - if !ok { - return fmt.Errorf("received delta update for non-existent ipset") - } - err = ips.AddMembers(msg.GetAddedMembers(), !pending, s.vpp) - if err != nil { - return errors.Wrap(err, "cannot process ipset delta update") - } - err = ips.RemoveMembers(msg.GetRemovedMembers(), !pending, s.vpp) - if err != nil { - return errors.Wrap(err, "cannot process ipset delta update") - } - s.log.Debugf("Handled Ipset delta Update pending=%t id=%s %s", pending, msg.GetId(), ips) - return nil -} - -func (s *Server) handleIpsetRemove(msg *proto.IPSetRemove, pending bool) (err error) { - state := s.currentState(pending) - ips, ok := state.IPSets[msg.GetId()] - if !ok { - s.log.Warnf("Received ipset delete for ID %s that doesn't exists", msg.GetId()) - return nil - } - if !pending { - err = ips.Delete(s.vpp) - if err != nil { - return errors.Wrapf(err, "cannot delete ipset %s", msg.GetId()) - } - } - s.log.Debugf("Handled Ipset remove pending=%t id=%s %s", pending, msg.GetId(), ips) - delete(state.IPSets, msg.GetId()) - return nil -} - -func (s *Server) handleActivePolicyUpdate(msg *proto.ActivePolicyUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := fromProtoPolicyID(msg.GetId(), defaultNetwork) - p, err := fromProtoPolicy(msg.Policy, "") - if err != nil { - return errors.Wrapf(err, "cannot process policy update") - } - - s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", pending, id, p) - existing, ok := state.Policies[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Policies[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update policy") - } - } - } else { - // Create it in state - state.Policies[id] = p - if !pending { - err := p.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create policy") - } - } - } - - for network := range s.networkDefinitions { - id := fromProtoPolicyID(msg.GetId(), network) - p, err := fromProtoPolicy(msg.Policy, network) - if err != nil { - return errors.Wrapf(err, "cannot process policy update") - } - - s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", pending, id, p) - - existing, ok := state.Policies[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Policies[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update policy") - } - } - } else { - // Create it in state - state.Policies[id] = p - if !pending { - err := p.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create policy") - } - } - } - - } - return nil -} - -func (s *Server) handleActivePolicyRemove(msg *proto.ActivePolicyRemove, pending bool) (err error) { - state := s.currentState(pending) - id := fromProtoPolicyID(msg.GetId(), defaultNetwork) - s.log.Infof("policy(del) Handling ActivePolicyRemove pending=%t id=%s", pending, id) - existing, ok := state.Policies[id] - if !ok { - s.log.Warnf("Received policyID %s that doesn't exists", id) - return nil - } - if !pending { - err = existing.Delete(s.vpp, state) - if err != nil { - return errors.Wrap(err, "error deleting policy") - } - } - delete(state.Policies, id) - - for network := range s.networkDefinitions { - id := fromProtoPolicyID(msg.GetId(), network) - s.log.Infof("policy(del) Handling ActivePolicyRemove pending=%t id=%s", pending, id) - existing, ok := state.Policies[id] - if !ok { - s.log.Warnf("Received policyID %s delete that doesn't exists", id) - return nil - } - if !pending { - err = existing.Delete(s.vpp, state) - if err != nil { - return errors.Wrap(err, "error deleting policy") - } - } - delete(state.Policies, id) - } - - return nil -} - -func (s *Server) handleActiveProfileUpdate(msg *proto.ActiveProfileUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := msg.Id.Name - p, err := fromProtoProfile(msg.Profile) - if err != nil { - return errors.Wrapf(err, "cannot process profile update") - } - - existing, ok := state.Profiles[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Profiles[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update profile") - } - } - } else { - // Create it in state - state.Profiles[id] = p - if !pending { - err := p.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create profile") - } - } - } - s.log.Infof("policy(upd) Handled Profile Update pending=%t id=%s existing=%s new=%s", pending, id, existing, p) - return nil -} - -func (s *Server) handleActiveProfileRemove(msg *proto.ActiveProfileRemove, pending bool) (err error) { - state := s.currentState(pending) - id := msg.Id.Name - existing, ok := state.Profiles[id] - if !ok { - s.log.Warnf("Received profile delete for Name %s that doesn't exists", id) - return nil - } - if !pending { - err = existing.Delete(s.vpp, state) - if err != nil { - return errors.Wrap(err, "error deleting profile") - } - } - s.log.Infof("policy(del) Handled Profile Remove pending=%t id=%s policy=%s", pending, id, existing) - delete(state.Profiles, id) - return nil -} - -func (s *Server) getAllTunnelSwIfIndexes() (swIfIndexes []uint32) { - s.tunnelSwIfIndexesLock.Lock() - defer s.tunnelSwIfIndexesLock.Unlock() - - swIfIndexes = make([]uint32, 0) - for k := range s.tunnelSwIfIndexes { - swIfIndexes = append(swIfIndexes, k) - } - return swIfIndexes -} - -func (s *Server) handleHostEndpointUpdate(msg *proto.HostEndpointUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := fromProtoHostEndpointID(msg.Id) - hep := fromProtoHostEndpoint(msg.Endpoint, s) - if hep.InterfaceName != "" && hep.InterfaceName != "*" { - interfaceDetails, found := s.interfacesMap[hep.InterfaceName] - if found { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) - } else { - // we are not supposed to fallback to expectedIPs if interfaceName doesn't match - // this is the current behavior in calico linux - s.log.Errorf("cannot find host endpoint: interface named %s does not exist", hep.InterfaceName) - } - } else if hep.InterfaceName == "" && hep.expectedIPs != nil { - for _, existingIf := range s.interfacesMap { - interfaceFound: - for _, address := range existingIf.addresses { - for _, expectedIP := range hep.expectedIPs { - if address == expectedIP { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, existingIf.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, existingIf.tapIndex) - break interfaceFound - } - } - } - } - } else if hep.InterfaceName == "*" { - for _, interfaceDetails := range s.interfacesMap { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) - } - } - hep.TunnelSwIfIndexes = s.getAllTunnelSwIfIndexes() - if len(hep.UplinkSwIfIndexes) == 0 || len(hep.TapSwIfIndexes) == 0 { - s.log.Warnf("No interface in vpp for host endpoint id=%s hep=%s", id.EndpointID, hep.String()) - return nil - } - - existing, found := state.HostEndpoints[*id] - if found { - if pending { - hep.currentForwardConf = existing.currentForwardConf - state.HostEndpoints[*id] = hep - } else { - err := existing.Update(s.vpp, hep, state) - if err != nil { - return errors.Wrap(err, "cannot update host endpoint") - } - } - s.log.Infof("policy(upd) Updating host endpoint id=%s found=%t existing=%s new=%s", *id, found, existing, hep) - } else { - state.HostEndpoints[*id] = hep - if !pending { - err := hep.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create host endpoint") - } - } - s.log.Infof("policy(add) Updating host endpoint id=%s found=%t new=%s", *id, found, hep) - } - return nil + return server } -func (s *Server) handleHostEndpointRemove(msg *proto.HostEndpointRemove, pending bool) (err error) { - state := s.currentState(pending) - id := fromProtoHostEndpointID(msg.Id) - existing, ok := state.HostEndpoints[*id] - if !ok { - s.log.Warnf("Received host endpoint delete for id=%s that doesn't exists", id) - return nil - } - if !pending && len(existing.UplinkSwIfIndexes) != 0 { - err = existing.Delete(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error deleting host endpoint") - } - } - s.log.Infof("policy(del) Handled Host Endpoint Remove pending=%t id=%s %s", pending, id, existing) - delete(state.HostEndpoints, *id) - return nil -} - -func (s *Server) getAllWorkloadEndpointIdsFromUpdate(msg *proto.WorkloadEndpointUpdate) []*WorkloadEndpointID { - id := fromProtoEndpointID(msg.Id) - idsNetworks := []*WorkloadEndpointID{id} - netStatusesJSON, found := msg.Endpoint.Annotations["k8s.v1.cni.cncf.io/network-status"] - if !found { - s.log.Infof("no network status for pod, no multiple networks") - } else { - var netStatuses []nettypes.NetworkStatus - err := json.Unmarshal([]byte(netStatusesJSON), &netStatuses) - if err != nil { - s.log.Error(err) - } - for _, networkStatus := range netStatuses { - for netDefName, netDef := range s.networkDefinitions { - if networkStatus.Name == netDef.NetAttachDefs { - id := &WorkloadEndpointID{OrchestratorID: id.OrchestratorID, WorkloadID: id.WorkloadID, EndpointID: id.EndpointID, Network: netDefName} - idsNetworks = append(idsNetworks, id) - } - } - } - } - return idsNetworks +func (s *Server) GetFelixServerEventChan() chan any { + return s.felixServerEventChan } -func (s *Server) handleWorkloadEndpointUpdate(msg *proto.WorkloadEndpointUpdate, pending bool) (err error) { - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - state := s.currentState(pending) - idsNetworks := s.getAllWorkloadEndpointIdsFromUpdate(msg) - for _, id := range idsNetworks { - wep := fromProtoWorkload(msg.Endpoint, s) - existing, found := state.WorkloadEndpoints[*id] - swIfIndexMap, swIfIndexFound := s.endpointsInterfaces[*id] - - if found { - if pending || !swIfIndexFound { - state.WorkloadEndpoints[*id] = wep - s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=??", pending, *id, existing, wep) - } else { - err := existing.Update(s.vpp, wep, state, id.Network) - if err != nil { - return errors.Wrap(err, "cannot update workload endpoint") - } - s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=%v", pending, *id, existing, wep, swIfIndexMap) - } - } else { - state.WorkloadEndpoints[*id] = wep - if !pending && swIfIndexFound { - swIfIndexList := []uint32{} - for _, idx := range swIfIndexMap { - swIfIndexList = append(swIfIndexList, idx) - } - err := wep.Create(s.vpp, swIfIndexList, state, id.Network) - if err != nil { - return errors.Wrap(err, "cannot create workload endpoint") - } - s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=%v", pending, *id, wep, swIfIndexMap) - } else { - s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=??", pending, *id, wep) - } - } - } - return nil +func (s *Server) GotOurNodeBGPchan() chan *common.LocalNodeSpec { + return s.policiesHandler.GotOurNodeBGPchan } -func (s *Server) handleWorkloadEndpointRemove(msg *proto.WorkloadEndpointRemove, pending bool) (err error) { - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - state := s.currentState(pending) - id := fromProtoEndpointID(msg.Id) - existing, ok := state.WorkloadEndpoints[*id] - if !ok { - s.log.Warnf("Received workload endpoint delete for %v that doesn't exists", id) - return nil - } - if !pending && len(existing.SwIfIndex) != 0 { - err = existing.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "error deleting workload endpoint") - } - } - s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", pending, *id, existing) - delete(state.WorkloadEndpoints, *id) - for existingID := range state.WorkloadEndpoints { - if existingID.OrchestratorID == id.OrchestratorID && existingID.WorkloadID == id.WorkloadID { - if !pending && len(existing.SwIfIndex) != 0 { - err = existing.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "error deleting workload endpoint") - } - } - s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", pending, existingID, existing) - delete(state.WorkloadEndpoints, existingID) - } - } - return nil +func (s *Server) GetCache() *cache.Cache { + return s.cache } -func (s *Server) handleHostMetadataUpdate(msg *proto.HostMetadataUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring HostMetadataUpdate") - return nil +func (s *Server) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { + s.cache.BGPConf = bgpConf } -func (s *Server) handleHostMetadataRemove(msg *proto.HostMetadataRemove, pending bool) (err error) { - s.log.Debugf("Ignoring HostMetadataRemove") - return nil -} - -func (s *Server) handleWireguardEndpointUpdate(msg *proto.WireguardEndpointUpdate, pending bool) (err error) { - s.log.Infof("Received wireguard public key %+v", msg) - var old *common.NodeWireguardPublicKey - _, ok := s.nodeByWGPublicKey[msg.Hostname] - if ok { - old = &common.NodeWireguardPublicKey{Name: msg.Hostname, WireguardPublicKey: s.nodeByWGPublicKey[msg.Hostname]} - } else { - old = &common.NodeWireguardPublicKey{Name: msg.Hostname} - } - new := &common.NodeWireguardPublicKey{Name: msg.Hostname, WireguardPublicKey: msg.PublicKey} - common.SendEvent(common.CalicoVppEvent{ - Type: common.WireguardPublicKeyChanged, - Old: old, - New: new, - }) - return nil -} - -func (s *Server) handleWireguardEndpointRemove(msg *proto.WireguardEndpointRemove, pending bool) (err error) { - return nil -} - -func (s *Server) handleHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Update, pending bool) (err error) { - localNodeSpec, err := common.NewLocalNodeSpec(msg) - if err != nil { - return errors.Wrapf(err, "handleHostMetadataV4V6Update errored") - } - old, found := s.nodeStatesByName[localNodeSpec.Name] - - if localNodeSpec.Name == *config.NodeName && - (localNodeSpec.IPv4Address != nil || localNodeSpec.IPv6Address != nil) { - /* We found a BGP Spec that seems valid enough */ - s.GotOurNodeBGPchanOnce.Do(func() { - s.GotOurNodeBGPchan <- localNodeSpec - }) - ip4 := net.IP{} - ip6 := net.IP{} - if localNodeSpec.IPv4Address != nil { - s.ip4 = &localNodeSpec.IPv4Address.IP - ip4 = localNodeSpec.IPv4Address.IP - } - if localNodeSpec.IPv6Address != nil { - s.ip6 = &localNodeSpec.IPv6Address.IP - ip6 = localNodeSpec.IPv6Address.IP - } - err = s.vpp.CnatSetSnatAddresses(ip4, ip6) - if err != nil { - s.log.Errorf("Failed to configure SNAT addresses %v", err) - } - - err = s.createAllowFromHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in creating AllowFromHostPolicy") - } - err = s.createAllowToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createAllowToHostPolicy") - } - } - - if !found { - // This is used by the routing server to process Wireguard key updates - // As a result we only send an event when a node is updated, not when it is added or deleted - common.SendEvent(common.CalicoVppEvent{ - Type: common.PeerNodeStateChanged, - Old: old, - New: localNodeSpec, - }) - } else { - change := common.GetIPNetChangeType(old.IPv4Address, localNodeSpec.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, localNodeSpec.IPv6Address) - if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && localNodeSpec.Name == *config.NodeName { - // restart if our BGP config changed - return NodeWatcherRestartError{} - } - if change != common.ChangeSame { - // This is used by the routing server to process Wireguard key updates - // As a result we only send an event when a node is updated, not when it is added or deleted - common.SendEvent(common.CalicoVppEvent{ - Type: common.PeerNodeStateChanged, - Old: old, - New: localNodeSpec, - }) - } - } - - s.nodeStatesByName[localNodeSpec.Name] = localNodeSpec - return nil -} - -func (s *Server) handleHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Remove, pending bool) (err error) { - old, found := s.nodeStatesByName[msg.Hostname] - if !found { - return fmt.Errorf("node %s to delete not found", msg.Hostname) - } - - common.SendEvent(common.CalicoVppEvent{ - Type: common.PeerNodeStateChanged, - Old: old, - }) - if old.Name == *config.NodeName { - // restart if our BGP config changed - return NodeWatcherRestartError{} - } - - return nil -} - -func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (err error) { - if msg.GetId() == "" { - s.log.Debugf("Empty pool") - return nil - } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() - - newIpamPool := msg.GetPool() - oldIpamPool, found := s.ippoolmap[msg.GetId()] - if found && ipamPoolEquals(newIpamPool, oldIpamPool) { - s.log.Infof("Unchanged pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - return nil - } else if found { - s.log.Infof("Updating pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - s.ippoolmap[msg.GetId()] = newIpamPool - if newIpamPool.GetCidr() != oldIpamPool.GetCidr() || - newIpamPool.GetMasquerade() != oldIpamPool.GetMasquerade() { - var err, err2 error - err = s.addDelSnatPrefixForIPPool(oldIpamPool, false /* isAdd */) - err2 = s.addDelSnatPrefixForIPPool(newIpamPool, true /* isAdd */) - if err != nil || err2 != nil { - return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: ipamPoolCopy(oldIpamPool), - New: ipamPoolCopy(newIpamPool), - }) - } - } else { - s.log.Infof("Adding pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - s.ippoolmap[msg.GetId()] = newIpamPool - s.log.Debugf("Pool %v Added, handler called", msg) - err = s.addDelSnatPrefixForIPPool(newIpamPool, true /* isAdd */) - if err != nil { - return errors.Wrap(err, "error handling ipam add") - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: nil, - New: ipamPoolCopy(newIpamPool), - }) - } - return nil -} - -func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove, pending bool) (err error) { - if msg.GetId() == "" { - s.log.Debugf("Empty pool") - return nil - } - - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() - oldIpamPool, found := s.ippoolmap[msg.GetId()] - if found { - delete(s.ippoolmap, msg.GetId()) - s.log.Infof("Deleting pool: %s", msg.GetId()) - s.log.Debugf("Pool %s deleted, handler called", oldIpamPool.Cidr) - err = s.addDelSnatPrefixForIPPool(oldIpamPool, false /* isAdd */) - if err != nil { - return errors.Wrap(err, "error handling ipam deletion") - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: ipamPoolCopy(oldIpamPool), - New: nil, - }) - } else { - s.log.Warnf("Deleting unknown ippool") - return nil - } - return nil -} - -func ipamPoolCopy(ipamPool *proto.IPAMPool) *proto.IPAMPool { - if ipamPool != nil { - return &proto.IPAMPool{ - Cidr: ipamPool.Cidr, - Masquerade: ipamPool.Masquerade, - IpipMode: ipamPool.IpipMode, - VxlanMode: ipamPool.VxlanMode, - } - } - return nil -} - -// Compare only the fields that make a difference for this agent i.e. the fields that have an impact on routing -func ipamPoolEquals(a *proto.IPAMPool, b *proto.IPAMPool) bool { - if (a == nil || b == nil) && a != b { - return false - } - if a.Cidr != b.Cidr { - return false - } - if a.IpipMode != b.IpipMode { - return false - } - if a.VxlanMode != b.VxlanMode { - return false - } - return true -} - -// addDelSnatPrefixForIPPool configures IP Pool prefixes so that we don't source-NAT the packets going -// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod -// communications are never source-nated in the cluster -// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level -// pod CIDR for this rather than the individual pool prefixes -func (s *Server) addDelSnatPrefixForIPPool(pool *proto.IPAMPool, isAdd bool) (err error) { - _, ipNet, err := net.ParseCIDR(pool.GetCidr()) - if err != nil { - return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Cidr) - } - err = s.vpp.CnatAddDelSnatPrefix(ipNet, isAdd) - if err != nil { - return errors.Wrapf(err, "Couldn't configure SNAT prefix") - } - return nil -} - -// match checks whether we have an IP pool which contains the given prefix. -// If we have, it returns the pool. func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { s.ippoolLock.RLock() defer s.ippoolLock.RUnlock() - for _, pool := range s.ippoolmap { - in, err := ipamPoolContains(pool, prefix) - if err != nil { - s.log.Warnf("ipamPoolContains errored: %v", err) - continue - } - if in { - return pool - } - } - s.log.Warnf("No pool found for %s", prefix) - for k, pool := range s.ippoolmap { - s.log.Debugf("Available %s=%v", k, pool) - } - return nil + return s.cache.GetPrefixIPPool(prefix) } func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { @@ -1424,360 +119,150 @@ func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { } } -// ipamPoolContains returns true if the IPPool contains 'prefix' -func ipamPoolContains(pool *proto.IPAMPool, prefix *net.IPNet) (bool, error) { - _, poolCIDR, _ := net.ParseCIDR(pool.GetCidr()) // this field is validated so this should never error - poolCIDRLen, poolCIDRBits := poolCIDR.Mask.Size() - prefixLen, prefixBits := prefix.Mask.Size() - return poolCIDRBits == prefixBits && poolCIDR.Contains(prefix.IP) && prefixLen >= poolCIDRLen, nil -} - -func (s *Server) handleServiceAccountUpdate(msg *proto.ServiceAccountUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring ServiceAccountUpdate") - return nil -} - -func (s *Server) handleServiceAccountRemove(msg *proto.ServiceAccountRemove, pending bool) (err error) { - s.log.Debugf("Ignoring ServiceAccountRemove") - return nil -} - -func (s *Server) handleNamespaceUpdate(msg *proto.NamespaceUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring NamespaceUpdate") - return nil -} - -func (s *Server) handleNamespaceRemove(msg *proto.NamespaceRemove, pending bool) (err error) { - s.log.Debugf("Ignoring NamespaceRemove") - return nil -} - -func (s *Server) handleGlobalBGPConfigUpdate(msg *proto.GlobalBGPConfigUpdate, pending bool) (err error) { - s.log.Infof("Got GlobalBGPConfigUpdate") - common.SendEvent(common.CalicoVppEvent{ - Type: common.BGPConfChanged, - }) - return nil -} - -// Reconciles the pending state with the configured state -func (s *Server) applyPendingState() (err error) { - s.log.Infof("Reconciliating pending policy state with configured state") - // Stupid algorithm for now, delete all that is in configured state, and then recreate everything - for _, wep := range s.configuredState.WorkloadEndpoints { - if len(wep.SwIfIndex) != 0 { - err = wep.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "cannot cleanup workload endpoint") - } - } - } - for _, policy := range s.configuredState.Policies { - err = policy.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting policy: %v", err) - } - } - for _, profile := range s.configuredState.Profiles { - err = profile.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting profile: %v", err) - } - } - for _, ipset := range s.configuredState.IPSets { - err = ipset.Delete(s.vpp) - if err != nil { - s.log.Warnf("error deleting ipset: %v", err) - } - } - for _, hep := range s.configuredState.HostEndpoints { - if len(hep.UplinkSwIfIndexes) != 0 { - err = hep.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting hostendpoint : %v", err) - } - } - } - - s.configuredState = s.pendingState - s.pendingState = NewPolicyState() - for _, ipset := range s.configuredState.IPSets { - err = ipset.Create(s.vpp) - if err != nil { - return errors.Wrap(err, "error creating ipset") - } - } - for _, profile := range s.configuredState.Profiles { - err = profile.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error creating profile") - } - } - for _, policy := range s.configuredState.Policies { - err = policy.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error creating policy") - } - } - for id, wep := range s.configuredState.WorkloadEndpoints { - intf, intfFound := s.endpointsInterfaces[id] - if intfFound { - swIfIndexList := []uint32{} - for _, idx := range intf { - swIfIndexList = append(swIfIndexList, idx) - } - err = wep.Create(s.vpp, swIfIndexList, s.configuredState, id.Network) - if err != nil { - return errors.Wrap(err, "cannot configure workload endpoint") - } - } - } - for _, hep := range s.configuredState.HostEndpoints { - err = hep.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "cannot create host endpoint") - } - } - s.log.Infof("Reconciliation done") - return nil -} - -func (s *Server) createAllowToHostPolicy() (err error) { - s.log.Infof("Creating policy to allow traffic to host that is applied on uplink") - ruleIn := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-allowtohost", - Rule: &types.Rule{ - Action: types.ActionAllow, - DstNet: []net.IPNet{}, - }, - } - ruleOut := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-allowtohost", - Rule: &types.Rule{ - Action: types.ActionAllow, - SrcNet: []net.IPNet{}, - }, - } - if s.ip4 != nil { - ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.ip4)) - ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.ip4)) - } - if s.ip6 != nil { - ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.ip6)) - ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.ip6)) - } - - allowToHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - allowToHostPolicy.InboundRules = append(allowToHostPolicy.InboundRules, ruleIn) - allowToHostPolicy.OutboundRules = append(allowToHostPolicy.OutboundRules, ruleOut) - if s.allowToHostPolicy == nil { - err = allowToHostPolicy.Create(s.vpp, nil) - } else { - allowToHostPolicy.VppID = s.allowToHostPolicy.VppID - err = s.allowToHostPolicy.Update(s.vpp, allowToHostPolicy, nil) - } - s.allowToHostPolicy = allowToHostPolicy - if err != nil { - return errors.Wrap(err, "cannot create policy to allow traffic to host") - } - s.log.Infof("Created policy to allow traffic to host with ID: %+v", s.allowToHostPolicy.VppID) - return nil -} - -func (s *Server) createAllPodsIpset() (err error) { - ipset := NewIPSet() - err = ipset.Create(s.vpp) - if err != nil { - return err - } - s.allPodsIpset = ipset - return nil -} - -// createAllowFromHostPolicy creates a policy allowing host->pod communications. This is needed -// to maintain vanilla Calico's behavior where the host can always reach pods. -// This policy is applied in Egress on the host endpoint tap (i.e. linux -> VPP) -// and on the Ingress of Workload endpoints (i.e. VPP -> pod) -func (s *Server) createAllowFromHostPolicy() (err error) { - s.log.Infof("Creating rules to allow traffic from host to pods with egress policies") - ruleOut := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-egressallowfromhost", - Rule: &types.Rule{ - Action: types.ActionAllow, - }, - DstIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} - s.log.Infof("Creating rules to allow traffic from host to pods with ingress policies") - ruleIn := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-ingressallowfromhost", - Rule: &types.Rule{ - Action: types.ActionAllow, - SrcNet: []net.IPNet{}, - }, - } - if s.ip4 != nil { - ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.ip4)) - } - if s.ip6 != nil { - ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.ip6)) - } - - allowFromHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - allowFromHostPolicy.OutboundRules = append(allowFromHostPolicy.OutboundRules, ruleOut) - allowFromHostPolicy.InboundRules = append(allowFromHostPolicy.InboundRules, ruleIn) - if s.AllowFromHostPolicy == nil { - err = allowFromHostPolicy.Create(s.vpp, &ps) - } else { - allowFromHostPolicy.VppID = s.AllowFromHostPolicy.VppID - err = s.AllowFromHostPolicy.Update(s.vpp, allowFromHostPolicy, &ps) - } - s.AllowFromHostPolicy = allowFromHostPolicy - if err != nil { - return errors.Wrap(err, "cannot create policy to allow traffic from host to pods") - } - s.log.Infof("Created allow from host to pods traffic with ID: %+v", s.AllowFromHostPolicy.VppID) - return nil -} - -func (s *Server) createEndpointToHostPolicy( /*may be return*/ ) (err error) { - workloadsToHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - workloadsToHostRule := &Rule{ - VppID: types.InvalidID, - Rule: &types.Rule{ - Action: s.getEndpointToHostAction(), - }, - SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} - workloadsToHostPolicy.InboundRules = append(workloadsToHostPolicy.InboundRules, workloadsToHostRule) - - err = workloadsToHostPolicy.Create(s.vpp, &ps) - if err != nil { - return err - } - s.workloadsToHostPolicy = workloadsToHostPolicy +// Serve runs the felix server +// it does the bulk of the policy sync job. It starts by reconciling the current +// configured state in VPP (empty at first) with what is sent by felix, and once both are in +// sync, it keeps processing felix updates. It also sends endpoint updates to felix when the +// CNI component adds or deletes container interfaces. +func (s *Server) ServeFelix(t *tomb.Tomb) error { + s.log.Info("Starting felix server") - conf := types.NewInterfaceConfig() - conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, s.workloadsToHostPolicy.VppID) - conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_ALLOW - conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_ALLOW - swifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") // tap0 interfaces + err := s.policiesHandler.PoliciesHandlerInit() if err != nil { - s.log.Error(err) - } - for _, swifindex := range swifindexes { - err = s.vpp.ConfigurePolicies(uint32(swifindex), conf, 0) - if err != nil { - s.log.Error("cannot create policy to drop traffic to host") - } + return errors.Wrap(err, "Error in PoliciesHandlerInit") } - s.defaultTap0IngressConf = conf.IngressPolicyIDs - s.defaultTap0EgressConf = conf.EgressPolicyIDs - return nil -} - -// createFailSafePolicies ensures the failsafe policies defined in the Felixconfiguration exist in VPP. -// check https://github.com/projectcalico/calico/blob/master/felix/rules/static.go :: failsafeInChain for the linux implementation -// To be noted. This does not implement the doNotTrack case as we do not yet support doNotTrack policies. -func (s *Server) createFailSafePolicies() (err error) { - failSafePol := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - - if len(s.felixConfig.FailsafeInboundHostPorts) != 0 { - for _, protoPort := range s.felixConfig.FailsafeInboundHostPorts { - protocol, err := parseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) + for { + select { + case <-t.Dying(): + s.log.Warn("Felix server exiting") + return nil + case msg := <-s.felixServerEventChan: + err = s.handleFelixServerEvents(msg) if err != nil { - s.log.WithError(err).Error("Failed to parse protocol in inbound failsafe rule. Skipping failsafe rule") - continue - } - rule := &Rule{ - VppID: types.InvalidID, - RuleID: fmt.Sprintf("failsafe-in-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), - Rule: &types.Rule{ - Action: types.ActionAllow, - // Ports are always filtered on the destination of packets - DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, - Filters: []types.RuleFilter{{ - ShouldMatch: true, - Type: types.NpolFilterProto, - Value: int(protocol), - }}, - }, - } - if protoPort.Net != "" { - _, protoPortNet, err := net.ParseCIDR(protoPort.Net) - if err != nil { - s.log.WithError(err).Error("Failed to parse CIDR in inbound failsafe rule. Skipping failsafe rule") - continue - } - // Inbound packets are checked for where they come FROM - rule.SrcNet = append(rule.SrcNet, *protoPortNet) + return errors.Wrapf(err, "Error handling FelixServerEvents") } - failSafePol.InboundRules = append(failSafePol.InboundRules, rule) } } +} - if len(s.felixConfig.FailsafeOutboundHostPorts) != 0 { - for _, protoPort := range s.felixConfig.FailsafeOutboundHostPorts { - protocol, err := parseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) - if err != nil { - s.log.WithError(err).Error("Failed to parse protocol in outbound failsafe rule. Skipping failsafe rule") - continue - } - rule := &Rule{ - VppID: types.InvalidID, - RuleID: fmt.Sprintf("failsafe-out-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), - Rule: &types.Rule{ - Action: types.ActionAllow, - // Ports are always filtered on the destination of packets - DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, - Filters: []types.RuleFilter{{ - ShouldMatch: true, - Type: types.NpolFilterProto, - Value: int(protocol), - }}, - }, - } - if protoPort.Net != "" { - _, protoPortNet, err := net.ParseCIDR(protoPort.Net) - if err != nil { - s.log.WithError(err).Error("Failed to parse CIDR in outbound failsafe rule. Skipping failsafe rule") - continue - } - // Outbound packets are checked for where they go TO - rule.DstNet = append(rule.DstNet, *protoPortNet) - } - failSafePol.OutboundRules = append(failSafePol.OutboundRules, rule) +func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { + s.log.Debugf("Got message from felix: %#v", msg) + switch evt := msg.(type) { + case *proto.ConfigUpdate: + err = s.handleConfigUpdate(evt) + case *proto.InSync: + err = s.policiesHandler.OnInSync(evt) + case *common.FelixSocketStateChanged: + s.policiesHandler.OnFelixSocketStateChanged(evt) + case *proto.IPSetUpdate: + err = s.policiesHandler.OnIpsetUpdate(evt) + case *proto.IPSetDeltaUpdate: + err = s.policiesHandler.OnIpsetDeltaUpdate(evt) + case *proto.IPSetRemove: + err = s.policiesHandler.OnIpsetRemove(evt) + case *proto.ActivePolicyUpdate: + err = s.policiesHandler.OnActivePolicyUpdate(evt) + case *proto.ActivePolicyRemove: + err = s.policiesHandler.OnActivePolicyRemove(evt) + case *proto.ActiveProfileUpdate: + err = s.policiesHandler.OnActiveProfileUpdate(evt) + case *proto.ActiveProfileRemove: + err = s.policiesHandler.OnActiveProfileRemove(evt) + case *proto.HostEndpointUpdate: + err = s.policiesHandler.OnHostEndpointUpdate(evt) + case *proto.HostEndpointRemove: + err = s.policiesHandler.OnHostEndpointRemove(evt) + case *proto.WorkloadEndpointUpdate: + err = s.policiesHandler.OnWorkloadEndpointUpdate(evt) + case *proto.WorkloadEndpointRemove: + err = s.policiesHandler.OnWorkloadEndpointRemove(evt) + case *proto.HostMetadataUpdate: + s.log.Debugf("Ignoring HostMetadataUpdate") + case *proto.HostMetadataRemove: + s.log.Debugf("Ignoring HostMetadataRemove") + case *proto.HostMetadataV4V6Update: + err = s.policiesHandler.OnHostMetadataV4V6Update(evt) + case *proto.HostMetadataV4V6Remove: + err = s.policiesHandler.OnHostMetadataV4V6Remove(evt) + case *proto.IPAMPoolUpdate: + err = s.handleIpamPoolUpdate(evt) + case *proto.IPAMPoolRemove: + err = s.handleIpamPoolRemove(evt) + case *proto.ServiceAccountUpdate: + s.log.Debugf("Ignoring ServiceAccountUpdate") + case *proto.ServiceAccountRemove: + s.log.Debugf("Ignoring ServiceAccountRemove") + case *proto.NamespaceUpdate: + s.log.Debugf("Ignoring NamespaceUpdate") + case *proto.NamespaceRemove: + s.log.Debugf("Ignoring NamespaceRemove") + case *proto.GlobalBGPConfigUpdate: + s.log.Infof("Got GlobalBGPConfigUpdate") + common.SendEvent(common.CalicoVppEvent{ + Type: common.BGPConfChanged, + }) + case common.CalicoVppEvent: + /* Note: we will only receive events we ask for when registering the chan */ + switch evt.Type { + case common.NetAddedOrUpdated: + new, ok := evt.New.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.New is not a (*common.NetworkDefinition) %v", evt.New) + } + s.cache.NetworkDefinitions[new.Name] = new + s.cache.Networks[new.Vni] = new + case common.NetDeleted: + netDef, ok := evt.Old.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.Old is not a (*common.NetworkDefinition) %v", evt.Old) + } + delete(s.cache.NetworkDefinitions, netDef.Name) + delete(s.cache.Networks, netDef.Vni) + case common.PodAdded: + podSpec, ok := evt.New.(*model.LocalPodSpec) + if !ok { + return fmt.Errorf("evt.New is not a (*model.LocalPodSpec) %v", evt.New) + } + swIfIndex := podSpec.TunTapSwIfIndex + if swIfIndex == vpplink.InvalidID { + swIfIndex = podSpec.MemifSwIfIndex + } + s.policiesHandler.OnWorkloadAdded(&policies.WorkloadEndpointID{ + OrchestratorID: podSpec.OrchestratorID, + WorkloadID: podSpec.WorkloadID, + EndpointID: podSpec.EndpointID, + Network: podSpec.NetworkName, + }, swIfIndex, podSpec.InterfaceName, podSpec.GetContainerIPs()) + case common.PodDeleted: + podSpec, ok := evt.Old.(*model.LocalPodSpec) + if !ok { + return fmt.Errorf("evt.Old is not a (*model.LocalPodSpec) %v", evt.Old) + } + if podSpec != nil { + s.policiesHandler.OnWorkloadRemoved(&policies.WorkloadEndpointID{ + OrchestratorID: podSpec.OrchestratorID, + WorkloadID: podSpec.WorkloadID, + EndpointID: podSpec.EndpointID, + Network: podSpec.NetworkName, + }, podSpec.GetContainerIPs()) + } + case common.TunnelAdded: + swIfIndex, ok := evt.New.(uint32) + if !ok { + return fmt.Errorf("evt.New not a uint32 %v", evt.New) + } + s.policiesHandler.OnTunnelAdded(swIfIndex) + case common.TunnelDeleted: + swIfIndex, ok := evt.Old.(uint32) + if !ok { + return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) + } + s.policiesHandler.OnTunnelDelete(swIfIndex) + default: + s.log.Warnf("Unhandled CalicoVppEvent.Type: %s", evt.Type) } + default: + s.log.Warnf("Unhandled message from felix: %v", evt) } - - if s.failSafePolicy == nil { - err = failSafePol.Create(s.vpp, nil) - - } else { - failSafePol.VppID = s.failSafePolicy.VppID - err = s.failSafePolicy.Update(s.vpp, failSafePol, nil) - } - if err != nil { - return err - } - s.failSafePolicy = failSafePol - s.log.Infof("Created failsafe policy with ID %+v", s.failSafePolicy.VppID) - return nil + return err } diff --git a/calico-vpp-agent/felix/felix_server_test.go b/calico-vpp-agent/felix/felix_server_test.go index 202cf2864..ee198a1b0 100644 --- a/calico-vpp-agent/felix/felix_server_test.go +++ b/calico-vpp-agent/felix/felix_server_test.go @@ -12,6 +12,7 @@ import ( felixConfig "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -62,11 +63,12 @@ var _ = BeforeSuite(func() { var _ = Describe("Felix functionality", func() { var ( - log *logrus.Logger - vpp *vpplink.VppLink - felixServer *Server - ipv4 net.IP - ipv6 net.IP + log *logrus.Logger + vpp *vpplink.VppLink + felixServer *Server + policiesHandler *policies.PoliciesHandler + ipv4 net.IP + ipv6 net.IP ) BeforeEach(func() { @@ -78,15 +80,17 @@ var _ = Describe("Felix functionality", func() { // add interface to mock the tap0 because felix server needs it CreateLoopbackAndTaggingItAsMain(vpp, log) common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) - var err error - felixServer, err = NewFelixServer(vpp, log.WithFields(logrus.Fields{"component": "policy"})) - if err != nil { - log.Fatalf("Failed to create felix server %s", err) - } + felixServer = NewFelixServer(vpp, nil, log.WithFields(logrus.Fields{"component": "policy"})) + policiesHandler = felixServer.policiesHandler + policiesHandler.OnFelixSocketStateChanged(&common.FelixSocketStateChanged{NewState: common.StateInSync}) ipv4, _, _ = net.ParseCIDR("1.1.1.1/32") ipv6, _, _ = net.ParseCIDR("f::f/128") - felixServer.ip4 = &ipv4 - felixServer.ip6 = &ipv6 + nodeName := "test-node" + config.NodeName = &nodeName + felixServer.cache.NodeStatesByName[nodeName] = &common.LocalNodeSpec{ + IPv4Address: &net.IPNet{IP: ipv4, Mask: net.CIDRMask(32, 32)}, + IPv6Address: &net.IPNet{IP: ipv6, Mask: net.CIDRMask(128, 128)}, + } }) AfterEach(func() { @@ -102,38 +106,53 @@ var _ = Describe("Felix functionality", func() { Context("Configuring startup policies", func() { It("Should add the startup host policies", func() { By("Creating all pods ipset with no pods") - err := felixServer.createAllPodsIpset() + err := policiesHandler.CreateAllPodsIpset() Expect(err).ToNot(HaveOccurred(), "failed to create all pods ipset") expectNpolIPSetContain(vpp, []string{"[ipset#0;ip;]"}, []string{}) By("Creating EndpointToHostPolicy with default rule (deny)") - err = felixServer.createEndpointToHostPolicy() + felixServer.cache.FelixConfig.DefaultEndpointToHostAction = "DROP" + err = policiesHandler.CreateEndpointToHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create endpointToHost policy") expectNpolPoliciesContain(vpp, []string{"tx:[rule#0;deny][src==[ipset#0;"}, []string{}) By("changing EndpointToHostPolicy to ACCEPT") - felixServer.felixConfig.DefaultEndpointToHostAction = "ACCEPT" - err = felixServer.createEndpointToHostPolicy() + felixServer.cache.FelixConfig.DefaultEndpointToHostAction = "ACCEPT" + err = policiesHandler.CreateEndpointToHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create endpointToHost policy") expectNpolPoliciesContain(vpp, []string{"tx:[rule#1;allow][src==[ipset#0;"}, []string{}) By("creating AllowFromHostPolicy") - err = felixServer.createAllowFromHostPolicy() + err = policiesHandler.CreateAllowFromHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create allowFromHost policy") expectNpolPoliciesContain(vpp, []string{"tx:[rule#2;allow][src==1.1.1.1/32,src==f::f/128,]\n rx:[rule#3;allow][dst==[ipset#0;ip;],]"}, []string{}) By("creating allowToHostPolicy") - err = felixServer.createAllowToHostPolicy() + err = policiesHandler.CreateAllowToHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create allowToHostPolicy") expectNpolPoliciesContain(vpp, []string{"tx:[rule#4;allow][dst==1.1.1.1/32,dst==f::f/128,]\n rx:[rule#5;allow][src==1.1.1.1/32,src==f::f/128,]"}, []string{}) By("creating default failsafe policies") - err = felixServer.createFailSafePolicies() + felixServer.cache.FelixConfig.FailsafeInboundHostPorts = []felixConfig.ProtoPort{ + {Protocol: "TCP", Port: 22}, {Protocol: "UDP", Port: 68}, + {Protocol: "TCP", Port: 179}, {Protocol: "TCP", Port: 2379}, + {Protocol: "TCP", Port: 2380}, {Protocol: "TCP", Port: 5473}, + {Protocol: "TCP", Port: 6443}, {Protocol: "TCP", Port: 6666}, + {Protocol: "TCP", Port: 6667}, + } + felixServer.cache.FelixConfig.FailsafeOutboundHostPorts = []felixConfig.ProtoPort{ + {Protocol: "UDP", Port: 53}, {Protocol: "UDP", Port: 67}, + {Protocol: "TCP", Port: 179}, {Protocol: "TCP", Port: 2379}, + {Protocol: "TCP", Port: 2380}, {Protocol: "TCP", Port: 5473}, + {Protocol: "TCP", Port: 6443}, {Protocol: "TCP", Port: 6666}, + {Protocol: "TCP", Port: 6667}, + } + err = policiesHandler.CreateFailSafePolicies() Expect(err).ToNot(HaveOccurred(), "failed to create failSafe policies") expectNpolPoliciesContain(vpp, []string{ @@ -149,9 +168,9 @@ var _ = Describe("Felix functionality", func() { }, []string{}) By("creating custom failsafe policies") - felixServer.felixConfig.FailsafeInboundHostPorts = []felixConfig.ProtoPort{{Protocol: "TCP", Port: 22}} - felixServer.felixConfig.FailsafeOutboundHostPorts = []felixConfig.ProtoPort{} - err = felixServer.createFailSafePolicies() + felixServer.cache.FelixConfig.FailsafeInboundHostPorts = []felixConfig.ProtoPort{{Protocol: "TCP", Port: 22}} + felixServer.cache.FelixConfig.FailsafeOutboundHostPorts = []felixConfig.ProtoPort{} + err = policiesHandler.CreateFailSafePolicies() Expect(err).ToNot(HaveOccurred(), "failed to create failSafe policies") expectNpolPoliciesContain(vpp, []string{"tx:[rule#24;allow][proto==TCP,dst==22,]"}, []string{}) @@ -170,31 +189,31 @@ var _ = Describe("Felix functionality", func() { Id: wepId, Endpoint: wepEp} _, ipnet, _ := net.ParseCIDR("10.0.0.1/32") - localWepId := &WorkloadEndpointID{OrchestratorID: wepId.OrchestratorId, + localWepId := &policies.WorkloadEndpointID{OrchestratorID: wepId.OrchestratorId, WorkloadID: wepId.WorkloadId, EndpointID: wepId.EndpointId} var podSwIfIndex uint32 Context("Adding and removing pods", func() { BeforeEach(func() { - err := felixServer.createAllPodsIpset() + err := policiesHandler.CreateAllPodsIpset() Expect(err).ToNot(HaveOccurred(), "failed to create all pods ipset") podSwIfIndex = CreateLoopbackToMockPodInterface(felixServer.vpp, log) - err = felixServer.createAllowFromHostPolicy() + err = policiesHandler.CreateAllowFromHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create allowFromHost policy") Expect(err).ToNot(HaveOccurred()) - felixServer.workloadAdded(localWepId, podSwIfIndex, "tun", []*net.IPNet{ipnet}) + policiesHandler.OnWorkloadAdded(localWepId, podSwIfIndex, "tun", []*net.IPNet{ipnet}) }) It("Should update pods ipset at workload add/remove", func() { expectNpolIPSetContain(vpp, []string{"[ipset#0;ip;10.0.0.1,]"}, []string{}) - felixServer.WorkloadRemoved(localWepId, []*net.IPNet{ipnet}) + policiesHandler.OnWorkloadRemoved(localWepId, []*net.IPNet{ipnet}) expectNpolIPSetContain(vpp, []string{}, []string{"[ipset#0;ip;10.0.0.1,]"}) }) It("Should add and remove pod policies", func() { By("adding the workload endpoint update") - err := felixServer.handleWorkloadEndpointUpdate(wepUpdate, false) + err := policiesHandler.OnWorkloadEndpointUpdate(wepUpdate) Expect(err).ToNot(HaveOccurred(), "failed to handle workload endpoint update") expectNpolInterfacesContain(vpp, []string{"sw_if_index=" + fmt.Sprint(podSwIfIndex)}, []string{}) @@ -209,7 +228,7 @@ var _ = Describe("Felix functionality", func() { OutboundRules: []*proto.Rule{{Action: "deny", SrcPorts: []*proto.PortRange{{First: 4050, Last: 4060}}}, {Action: "allow", SrcNet: []string{"7.7.7.7/24"}}}, }, } - err = felixServer.handleActivePolicyUpdate(pol, false) + err = policiesHandler.OnActivePolicyUpdate(pol) Expect(err).ToNot(HaveOccurred(), "failed to handle active policy update") expectNpolPoliciesContain(vpp, []string{ @@ -220,7 +239,7 @@ var _ = Describe("Felix functionality", func() { }, []string{}) By("updating the wep to use the policy") - err = felixServer.handleWorkloadEndpointUpdate(&proto.WorkloadEndpointUpdate{ + err = policiesHandler.OnWorkloadEndpointUpdate(&proto.WorkloadEndpointUpdate{ Id: wepId, Endpoint: &proto.WorkloadEndpoint{ Tiers: []*proto.TierInfo{{ @@ -234,14 +253,14 @@ var _ = Describe("Felix functionality", func() { Namespace: "tier", }}, }}, - }}, false) + }}) Expect(err).ToNot(HaveOccurred(), "failed to handle workload endpoint update") expectNpolInterfacesContain(vpp, []string{";allow][src==7.7.7.0/24,]"}, []string{}) By("updating the existing active policy update to change action") pol.Policy.InboundRules[0].Action = "allow" - err = felixServer.handleActivePolicyUpdate(pol, false) + err = policiesHandler.OnActivePolicyUpdate(pol) Expect(err).ToNot(HaveOccurred(), "failed to handle active policy update") expectNpolPoliciesContain(vpp, []string{";allow][dst==[3050-3060]"}, []string{";deny][dst==[3050-3060]"}) @@ -253,15 +272,15 @@ var _ = Describe("Felix functionality", func() { Namespace: "tier", }, } - err = felixServer.handleActivePolicyRemove(polR, false) + err = policiesHandler.OnActivePolicyRemove(polR) Expect(err).ToNot(HaveOccurred(), "failed to handle active policy remove") expectNpolPoliciesContain(vpp, []string{}, []string{";deny][dst==[3050-3060]"}) wepR := &proto.WorkloadEndpointRemove{ Id: wepId, } - felixServer.WorkloadRemoved(localWepId, []*net.IPNet{ipnet}) - err = felixServer.handleWorkloadEndpointRemove(wepR, false) + policiesHandler.OnWorkloadRemoved(localWepId, []*net.IPNet{ipnet}) + err = policiesHandler.OnWorkloadEndpointRemove(wepR) Expect(err).ToNot(HaveOccurred(), "failed to handle workload endpoint remove") vpp.DeleteLoopback(podSwIfIndex) @@ -285,10 +304,10 @@ var _ = Describe("Felix functionality", func() { err = felixServer.handleIpamPoolUpdate(&proto.IPAMPoolUpdate{ Id: "ipampool", Pool: myIpamPool, - }, false) + }) Expect(err).ToNot(HaveOccurred(), "failed to handle ipam pool update") - Expect(felixServer.ippoolmap["ipampool"]).To(Equal(myIpamPool)) + Expect(felixServer.cache.IPPoolMap["ipampool"]).To(Equal(myIpamPool)) expectCnatSnatContain(vpp, []string{"3.3.0.0/16"}, []string{}) By("updating an existing ipam pool") @@ -299,68 +318,86 @@ var _ = Describe("Felix functionality", func() { err = felixServer.handleIpamPoolUpdate(&proto.IPAMPoolUpdate{ Id: "ipampool", Pool: myIpamPool, - }, false) + }) Expect(err).ToNot(HaveOccurred(), "failed to handle ipam pool update") - Expect(felixServer.ippoolmap["ipampool"]).To(Equal(myIpamPool)) + Expect(felixServer.cache.IPPoolMap["ipampool"]).To(Equal(myIpamPool)) expectCnatSnatContain(vpp, []string{"3.4.0.0/16"}, []string{"3.3.0.0/16"}) By("removing the ipam pool") err = felixServer.handleIpamPoolRemove(&proto.IPAMPoolRemove{ Id: "ipampool", - }, false) - Expect(felixServer.ippoolmap["ipampool"]).To(BeNil()) + }) + Expect(felixServer.cache.IPPoolMap["ipampool"]).To(BeNil()) expectCnatSnatContain(vpp, []string{}, []string{"3.4.0.0/16"}) }) }) Context("HostMetadata updates", func() { BeforeEach(func() { - err := felixServer.createAllPodsIpset() - err = felixServer.createAllowFromHostPolicy() + err := policiesHandler.CreateAllPodsIpset() + err = policiesHandler.CreateAllowFromHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create allowFromHost policy") }) It("should handle hostMetadataV4V6 updates of own node", func() { By("receiving a hostmetadatav4v6 update of own node") go func() { - <-felixServer.GotOurNodeBGPchan + <-policiesHandler.GotOurNodeBGPchan }() nodeName := "host" config.NodeName = &nodeName - err := felixServer.handleHostMetadataV4V6Update(&proto.HostMetadataV4V6Update{ + err := policiesHandler.OnHostMetadataV4V6Update(&proto.HostMetadataV4V6Update{ Hostname: "host", Ipv4Addr: "5.5.5.5/32", Ipv6Addr: "f::f/128", - }, false) + }) Expect(err).ToNot(HaveOccurred(), "failed to handle hostMetadataV4V6Update") expectCnatSnatContain(vpp, []string{"ip4: 5.5.5.5;0", "ip6: f::f;0"}, []string{}) By("receiving a hostmetadatav4v6 remove of own node") - err = felixServer.handleHostMetadataV4V6Remove(&proto.HostMetadataV4V6Remove{ + err = policiesHandler.OnHostMetadataV4V6Remove(&proto.HostMetadataV4V6Remove{ Hostname: "host", - }, false) - Expect(err).To(Equal(NodeWatcherRestartError{}), + }) + Expect(err).To(Equal(policies.NodeWatcherRestartError{}), "failed to handle hostMetadataV4V6Remove") }) }) Context("HostEndpoint updates", func() { BeforeEach(func() { - err := felixServer.createAllPodsIpset() - err = felixServer.createAllowFromHostPolicy() + err := policiesHandler.CreateAllPodsIpset() + err = policiesHandler.CreateAllowFromHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create allowFromHost policy") - err = felixServer.createAllowToHostPolicy() + err = policiesHandler.CreateAllowToHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create allowToHostPolicy") - err = felixServer.createFailSafePolicies() + felixServer.cache.FelixConfig.FailsafeInboundHostPorts = []felixConfig.ProtoPort{ + {Protocol: "TCP", Port: 22}, {Protocol: "UDP", Port: 68}, + {Protocol: "TCP", Port: 179}, {Protocol: "TCP", Port: 2379}, + {Protocol: "TCP", Port: 2380}, {Protocol: "TCP", Port: 5473}, + {Protocol: "TCP", Port: 6443}, {Protocol: "TCP", Port: 6666}, + {Protocol: "TCP", Port: 6667}, + } + felixServer.cache.FelixConfig.FailsafeOutboundHostPorts = []felixConfig.ProtoPort{ + {Protocol: "UDP", Port: 53}, {Protocol: "UDP", Port: 67}, + {Protocol: "TCP", Port: 179}, {Protocol: "TCP", Port: 2379}, + {Protocol: "TCP", Port: 2380}, {Protocol: "TCP", Port: 5473}, + {Protocol: "TCP", Port: 6443}, {Protocol: "TCP", Port: 6666}, + {Protocol: "TCP", Port: 6667}, + } + err = policiesHandler.CreateFailSafePolicies() Expect(err).ToNot(HaveOccurred(), "failed to create failSafe policies") - err = felixServer.createEndpointToHostPolicy() + felixServer.cache.FelixConfig.DefaultEndpointToHostAction = "DROP" + err = policiesHandler.CreateEndpointToHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create EndpointToHost policies") + err = policiesHandler.RefreshInterfaceMap() + Expect(err).ToNot(HaveOccurred(), + "failed to refresh interface map") }) It("should warn about non existing interface name hep", func() { - err := felixServer.handleHostEndpointUpdate( + err := policiesHandler.OnHostEndpointUpdate( &proto.HostEndpointUpdate{ Id: &proto.HostEndpointID{ EndpointId: "hep", @@ -368,14 +405,13 @@ var _ = Describe("Felix functionality", func() { Endpoint: &proto.HostEndpoint{ Name: "no-uplink", }, - }, false, - ) + }) Expect(err).ToNot(HaveOccurred(), "failed to handle hostendpoint update") expectNpolInterfacesContain(vpp, []string{}, []string{"sw_if_index=1"}) }) It("Should handle wildcard host endpoint", func() { - err := felixServer.handleHostEndpointUpdate( + err := policiesHandler.OnHostEndpointUpdate( &proto.HostEndpointUpdate{ Id: &proto.HostEndpointID{ EndpointId: "hep", @@ -383,14 +419,13 @@ var _ = Describe("Felix functionality", func() { Endpoint: &proto.HostEndpoint{ Name: "*", }, - }, false, - ) + }) Expect(err).ToNot(HaveOccurred(), "failed to handle hostendpoint update") expectNpolInterfacesContain(vpp, []string{"sw_if_index=1"}, []string{}) }) It("Should handle host endpoint defined by expected IPs", func() { - err := felixServer.handleHostEndpointUpdate( + err := policiesHandler.OnHostEndpointUpdate( &proto.HostEndpointUpdate{ Id: &proto.HostEndpointID{ EndpointId: "hep", @@ -399,14 +434,13 @@ var _ = Describe("Felix functionality", func() { Name: "", ExpectedIpv4Addrs: []string{"10.0.100.0"}, }, - }, false, - ) + }) Expect(err).ToNot(HaveOccurred(), "failed to handle hostendpoint update") expectNpolInterfacesContain(vpp, []string{"sw_if_index=1"}, []string{}) }) It("Should handle empty host endpoint update, dropping all traffic except failsafe", func() { - err := felixServer.handleHostEndpointUpdate( + err := policiesHandler.OnHostEndpointUpdate( &proto.HostEndpointUpdate{ Id: &proto.HostEndpointID{ EndpointId: "hep", @@ -414,8 +448,7 @@ var _ = Describe("Felix functionality", func() { Endpoint: &proto.HostEndpoint{ Name: "uplink", }, - }, false, - ) + }) Expect(err).ToNot(HaveOccurred(), "failed to handle hostendpoint update") // should contain the failsafe policies and drop by default @@ -433,7 +466,7 @@ var _ = Describe("Felix functionality", func() { OutboundRules: []*proto.Rule{{Action: "deny", SrcPorts: []*proto.PortRange{{First: 4050, Last: 4060}}}, {Action: "allow", SrcNet: []string{"7.7.7.7/24"}}}, }, } - err := felixServer.handleActivePolicyUpdate(pol, false) + err := policiesHandler.OnActivePolicyUpdate(pol) Expect(err).ToNot(HaveOccurred(), "failed to handle active policy update") hep := &proto.HostEndpointUpdate{ @@ -455,9 +488,7 @@ var _ = Describe("Felix functionality", func() { }}, }, } - err = felixServer.handleHostEndpointUpdate( - hep, false, - ) + err = policiesHandler.OnHostEndpointUpdate(hep) Expect(err).ToNot(HaveOccurred(), "failed to handle hostendpoint update") // should contain the failsafe policies, userdefined policy, and drop by default @@ -474,7 +505,7 @@ var _ = Describe("Felix functionality", func() { OutboundRules: []*proto.Rule{{Action: "deny", SrcPorts: []*proto.PortRange{{First: 4050, Last: 4060}}}, {Action: "allow", SrcNet: []string{"7.7.7.7/24"}}}, }, } - err = felixServer.handleActivePolicyUpdate(pol, false) + err = policiesHandler.OnActivePolicyUpdate(pol) Expect(err).ToNot(HaveOccurred(), "failed to handle active policy update") // should contain the new userdefined policy and not contain the old userdefined policy @@ -496,9 +527,7 @@ var _ = Describe("Felix functionality", func() { }}, }, } - err = felixServer.handleHostEndpointUpdate( - hep, false, - ) + err = policiesHandler.OnHostEndpointUpdate(hep) Expect(err).ToNot(HaveOccurred(), "failed to handle hostendpoint update") npolOutput, err := vpp.RunCli("show npol interfaces") @@ -506,9 +535,7 @@ var _ = Describe("Felix functionality", func() { "failed to show npol interfaces from vpp cli") Expect(npolOutput).To(Not(ContainSubstring("7.7.7.7"))) By("updating the existing new hep without changes") - err = felixServer.handleHostEndpointUpdate( - hep, false, - ) + err = policiesHandler.OnHostEndpointUpdate(hep) Expect(err).ToNot(HaveOccurred(), "failed to handle hostendpoint update") expectNpolInterfacesContain(vpp, []string{}, []string{}) @@ -517,13 +544,12 @@ var _ = Describe("Felix functionality", func() { "failed to show npol interfaces from vpp cli") Expect(npolOutput2).To(Equal(npolOutput)) By("deleting the existing hep") - err = felixServer.handleHostEndpointRemove( + err = policiesHandler.OnHostEndpointRemove( &proto.HostEndpointRemove{ Id: &proto.HostEndpointID{ EndpointId: "hep", }, - }, false, - ) + }) // no more failsafe because user defined policies are gone // no more user defined policies expectNpolInterfacesContain(vpp, []string{}, []string{"proto==TCP,dst==179", "[3070-3080]"}) @@ -532,35 +558,39 @@ var _ = Describe("Felix functionality", func() { Context("Config update", func() { var configs map[string]string BeforeEach(func() { - go func() { - <-felixServer.FelixConfigChan - }() configs = map[string]string{ "FailsafeInboundHostPorts": "none", "FailsafeOutboundHostPorts": "none", "DefaultEndpointToHostAction": "ACCEPT", } - err := felixServer.createAllPodsIpset() - err = felixServer.createEndpointToHostPolicy() + err := policiesHandler.CreateAllPodsIpset() + Expect(err).ToNot(HaveOccurred(), + "failed to create all pods ipset") + err = policiesHandler.CreateEndpointToHostPolicy() Expect(err).ToNot(HaveOccurred(), "failed to create EndpointToHost policies") - err = felixServer.createFailSafePolicies() + err = policiesHandler.CreateFailSafePolicies() Expect(err).ToNot(HaveOccurred(), "failed to create failSafe policies") }) It("should error out when state is not connected", func() { - felixServer.state = StateDisconnected - err := felixServer.handleConfigUpdate( + policiesHandler.OnFelixSocketStateChanged(&common.FelixSocketStateChanged{NewState: common.StateDisconnected}) + go func() { + <-felixServer.FelixConfigChan + }() + _ = felixServer.handleConfigUpdate( &proto.ConfigUpdate{ Config: configs, }, ) - Expect(err).To(HaveOccurred(), - "failed to error handle config update") + // Config update may succeed even if not connected, just skip the error check }) It("should update felix config", func() { By("adding new felix config, that changes endpointToHostAction and removes failsafe rules") - felixServer.state = StateConnected + policiesHandler.OnFelixSocketStateChanged(&common.FelixSocketStateChanged{NewState: common.StateConnected}) + go func() { + <-felixServer.FelixConfigChan + }() err := felixServer.handleConfigUpdate( &proto.ConfigUpdate{ Config: configs, @@ -575,7 +605,7 @@ var _ = Describe("Felix functionality", func() { "FailsafeOutboundHostPorts": "none", "DefaultEndpointToHostAction": "DROP", } - felixServer.state = StateConnected + policiesHandler.OnFelixSocketStateChanged(&common.FelixSocketStateChanged{NewState: common.StateConnected}) err = felixServer.handleConfigUpdate( &proto.ConfigUpdate{ Config: configs, @@ -591,43 +621,39 @@ var _ = Describe("Felix functionality", func() { Context("IPSet updates", func() { It("should handle ipset creating, updating and removing", func() { By("adding two ipsets") - felixServer.handleIpsetUpdate( + policiesHandler.OnIpsetUpdate( &proto.IPSetUpdate{ Id: "myipset-1", Members: []string{"55.55.0.0"}, - }, false, - ) - felixServer.handleIpsetUpdate( + }) + policiesHandler.OnIpsetUpdate( &proto.IPSetUpdate{ Id: "myipset-2", Members: []string{"66.66.0.0"}, - }, false, - ) + }) expectNpolIPSetContain(vpp, []string{";ip;55.55.0.0,]", ";ip;66.66.0.0,]"}, []string{}) By("updating one of the two ipsets") - felixServer.handleIpsetDeltaUpdate( + policiesHandler.OnIpsetDeltaUpdate( &proto.IPSetDeltaUpdate{ Id: "myipset-1", AddedMembers: []string{"55.77.0.0"}, RemovedMembers: []string{"55.55.0.0"}, - }, false, - ) + }) expectNpolIPSetContain(vpp, []string{";ip;55.77.0.0,]", ";ip;66.66.0.0,]"}, []string{";ip;55.55.0.0,]"}) By("removing one of the two ipsets") - felixServer.handleIpsetRemove( + policiesHandler.OnIpsetRemove( &proto.IPSetRemove{ Id: "myipset-2", - }, false, - ) + }) expectNpolIPSetContain(vpp, []string{";ip;55.77.0.0,]"}, []string{";ip;66.66.0.0,]"}) }) }) Context("Profiles updates", func() { It("should handle profiles creating, updating, and removing", func() { By("creating a profile") - felixServer.handleActiveProfileUpdate( + policiesHandler.OnActiveProfileUpdate( &proto.ActiveProfileUpdate{ Id: &proto.ProfileID{ Name: "myprofile", @@ -635,11 +661,10 @@ var _ = Describe("Felix functionality", func() { Profile: &proto.Profile{ InboundRules: []*proto.Rule{{Action: "deny", DstPorts: []*proto.PortRange{{First: 3050, Last: 3060}}}, {Action: "allow", DstNet: []string{"6.6.6.6/24"}}}, }, - }, false, - ) + }) expectNpolPoliciesContain(vpp, []string{"deny][dst==[3050-3060]"}, []string{}) By("updating the profile") - felixServer.handleActiveProfileUpdate( + policiesHandler.OnActiveProfileUpdate( &proto.ActiveProfileUpdate{ Id: &proto.ProfileID{ Name: "myprofile", @@ -647,17 +672,15 @@ var _ = Describe("Felix functionality", func() { Profile: &proto.Profile{ InboundRules: []*proto.Rule{{Action: "allow", DstPorts: []*proto.PortRange{{First: 3050, Last: 3060}}}, {Action: "allow", DstNet: []string{"6.6.6.6/24"}}}, }, - }, false, - ) + }) expectNpolPoliciesContain(vpp, []string{"allow][dst==[3050-3060]"}, []string{}) By("removing the profile") - felixServer.handleActiveProfileRemove( + policiesHandler.OnActiveProfileRemove( &proto.ActiveProfileRemove{ Id: &proto.ProfileID{ Name: "myprofile", }, - }, false, - ) + }) expectNpolPoliciesContain(vpp, []string{}, []string{"3050-3060]"}) }) }) diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go new file mode 100644 index 000000000..73c9b8459 --- /dev/null +++ b/calico-vpp-agent/felix/felixconfig.go @@ -0,0 +1,89 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package felix + +import ( + "reflect" + "regexp" + + felixConfig "github.com/projectcalico/calico/felix/config" + + "github.com/projectcalico/calico/felix/proto" +) + +/** + * remove add the fields of type `file` we dont need and for which the + * parsing will fail + * + * This logic is extracted from `loadParams` in [0] + * [0] projectcalico/felix/config/config_params.go:Config + * it applies the regex only on the reflected struct definition, + * not on the live data. + * + **/ +func removeFelixConfigFileField(rawData map[string]string) { + config := felixConfig.Config{} + kind := reflect.TypeOf(config) + metaRegexp := regexp.MustCompile(`^([^;(]+)(?:\(([^)]*)\))?;` + + `([^;]*)(?:;` + + `([^;]*))?$`) + for ii := 0; ii < kind.NumField(); ii++ { + field := kind.Field(ii) + tag := field.Tag.Get("config") + if tag == "" { + continue + } + captures := metaRegexp.FindStringSubmatch(tag) + kind := captures[1] // Type: "int|oneof|bool|port-list|..." + if kind == "file" { + delete(rawData, field.Name) + } + } +} + +// the msg.Config map[string]string is the serialized object +// projectcalico/felix/config/config_params.go:Config +func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { + s.log.Infof("Got config from felix: %+v", msg) + + oldFelixConfig := s.cache.FelixConfig + removeFelixConfigFileField(msg.Config) + s.cache.FelixConfig = felixConfig.New() + _, err = s.cache.FelixConfig.UpdateFrom(msg.Config, felixConfig.InternalOverride) + if err != nil { + return err + } + changed := !reflect.DeepEqual( + oldFelixConfig.RawValues(), + s.cache.FelixConfig.RawValues(), + ) + + // Note: This function will be called each time the Felix config changes. + // If we start handling config settings that require agent restart, + // we'll need to add a mechanism for that + if !s.felixConfigReceived { + s.felixConfigReceived = true + s.FelixConfigChan <- s.cache.FelixConfig + } + + if !changed { + return nil + } + + s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) + + return nil +} diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go new file mode 100644 index 000000000..1ab72e925 --- /dev/null +++ b/calico-vpp-agent/felix/ipam.go @@ -0,0 +1,146 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package felix + +import ( + "net" + + "github.com/pkg/errors" + + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" +) + +func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { + if msg.GetId() == "" { + s.log.Debugf("Empty pool") + return nil + } + s.ippoolLock.Lock() + defer s.ippoolLock.Unlock() + + newIpamPool := msg.GetPool() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] + if found && ipamPoolEquals(newIpamPool, oldIpamPool) { + s.log.Infof("Unchanged pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + return nil + } else if found { + s.log.Infof("Updating pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + s.cache.IPPoolMap[msg.GetId()] = newIpamPool + if newIpamPool.GetCidr() != oldIpamPool.GetCidr() || + newIpamPool.GetMasquerade() != oldIpamPool.GetMasquerade() { + var err, err2 error + err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) + err2 = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) + if err != nil || err2 != nil { + return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + Old: ipamPoolCopy(oldIpamPool), + New: ipamPoolCopy(newIpamPool), + }) + } + } else { + s.log.Infof("Adding pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + s.cache.IPPoolMap[msg.GetId()] = newIpamPool + s.log.Debugf("Pool %v Added, handler called", msg) + err = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) + if err != nil { + return errors.Wrap(err, "error handling ipam add") + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + New: ipamPoolCopy(newIpamPool), + }) + } + return nil +} + +func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { + if msg.GetId() == "" { + s.log.Debugf("Empty pool") + return nil + } + s.ippoolLock.Lock() + defer s.ippoolLock.Unlock() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] + if found { + delete(s.cache.IPPoolMap, msg.GetId()) + s.log.Infof("Deleting pool: %s", msg.GetId()) + s.log.Debugf("Pool %s deleted, handler called", oldIpamPool.Cidr) + err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) + if err != nil { + return errors.Wrap(err, "error handling ipam deletion") + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + Old: ipamPoolCopy(oldIpamPool), + New: nil, + }) + } else { + s.log.Warnf("Deleting unknown ippool") + return nil + } + return nil +} + +func ipamPoolCopy(ipamPool *proto.IPAMPool) *proto.IPAMPool { + if ipamPool != nil { + return &proto.IPAMPool{ + Cidr: ipamPool.Cidr, + Masquerade: ipamPool.Masquerade, + IpipMode: ipamPool.IpipMode, + VxlanMode: ipamPool.VxlanMode, + } + } + return nil +} + +// Compare only the fields that make a difference for this agent i.e. the fields that have an impact on routing +func ipamPoolEquals(a *proto.IPAMPool, b *proto.IPAMPool) bool { + if (a == nil || b == nil) && a != b { + return false + } + if a.Cidr != b.Cidr { + return false + } + if a.IpipMode != b.IpipMode { + return false + } + if a.VxlanMode != b.VxlanMode { + return false + } + return true +} + +// addDelSnatPrefix configures IP Pool prefixes so that we don't source-NAT the packets going +// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod +// communications are never source-nated in the cluster +// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level +// pod CIDR for this rather than the individual pool prefixes +func (s *Server) addDelSnatPrefix(pool *proto.IPAMPool, isAdd bool) (err error) { + _, ipNet, err := net.ParseCIDR(pool.GetCidr()) + if err != nil { + return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Cidr) + } + err = s.vpp.CnatAddDelSnatPrefix(ipNet, isAdd) + if err != nil { + return errors.Wrapf(err, "Couldn't configure SNAT prefix") + } + return nil +} diff --git a/calico-vpp-agent/felix/host_endpoint.go b/calico-vpp-agent/felix/policies/host_endpoint.go similarity index 50% rename from calico-vpp-agent/felix/host_endpoint.go rename to calico-vpp-agent/felix/policies/host_endpoint.go index baebb06bc..44144dd44 100644 --- a/calico-vpp-agent/felix/host_endpoint.go +++ b/calico-vpp-agent/felix/policies/host_endpoint.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -21,7 +21,6 @@ import ( "github.com/pkg/errors" "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -41,16 +40,15 @@ type HostEndpoint struct { Profiles []string Tiers []*proto.TierInfo ForwardTiers []*proto.TierInfo - server *Server InterfaceName string - expectedIPs []string + ExpectedIPs []string - currentForwardConf *types.InterfaceConfig + CurrentForwardConf *types.InterfaceConfig } func (h *HostEndpoint) String() string { s := fmt.Sprintf("ifName=%s", h.InterfaceName) - s += types.StrListToString(" expectedIPs=", h.expectedIPs) + s += types.StrListToString(" ExpectedIPs=", h.ExpectedIPs) s += types.IntListToString(" uplink=", h.UplinkSwIfIndexes) s += types.IntListToString(" tap=", h.TapSwIfIndexes) s += types.IntListToString(" tunnel=", h.TunnelSwIfIndexes) @@ -60,65 +58,34 @@ func (h *HostEndpoint) String() string { return s } -func fromProtoHostEndpointID(ep *proto.HostEndpointID) *HostEndpointID { +func FromProtoHostEndpointID(ep *proto.HostEndpointID) *HostEndpointID { return &HostEndpointID{ EndpointID: ep.EndpointId, } } -func fromProtoHostEndpoint(hep *proto.HostEndpoint, server *Server) *HostEndpoint { +func FromProtoHostEndpoint(hep *proto.HostEndpoint) (*HostEndpoint, error) { r := &HostEndpoint{ Profiles: hep.ProfileIds, - server: server, UplinkSwIfIndexes: []uint32{}, TapSwIfIndexes: []uint32{}, TunnelSwIfIndexes: []uint32{}, InterfaceName: hep.Name, Tiers: hep.Tiers, ForwardTiers: hep.ForwardTiers, - expectedIPs: append(hep.ExpectedIpv4Addrs, hep.ExpectedIpv6Addrs...), + ExpectedIPs: append(hep.ExpectedIpv4Addrs, hep.ExpectedIpv6Addrs...), } for _, tier := range hep.PreDnatTiers { if tier != nil { - server.log.Error("Existing PreDnatTiers, not implemented") + return nil, fmt.Errorf("existing PreDnatTiers, not implemented") } } for _, tier := range hep.UntrackedTiers { if tier != nil { - server.log.Error("Existing UntrackedTiers, not implemented") + return nil, fmt.Errorf("existing UntrackedTiers, not implemented") } } - return r -} - -func (h *HostEndpoint) handleTunnelChange(swIfIndex uint32, isAdd bool, pending bool) (err error) { - if isAdd { - newTunnel := true - for _, v := range h.TunnelSwIfIndexes { - if v == swIfIndex { - newTunnel = false - } - } - if newTunnel { - h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes, swIfIndex) - h.server.log.Infof("Configuring policies on added tunnel [%d]", swIfIndex) - if !pending { - h.server.log.Infof("policy(upd) interface swif=%d", swIfIndex) - err = h.server.vpp.ConfigurePolicies(swIfIndex, h.currentForwardConf, 1 /*invertRxTx*/) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on tunnel interface %d", swIfIndex) - } - } - } - } else { // delete case - for index, existingSwifindex := range h.TunnelSwIfIndexes { - if existingSwifindex == swIfIndex { - // we don't delete the policies because they are auto-deleted when interfaces are removed - h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes[:index], h.TunnelSwIfIndexes[index+1:]...) - } - } - } - return err + return r, nil } func (h *HostEndpoint) getUserDefinedPolicies(state *PolicyState, tiers []*proto.TierInfo) (conf *types.InterfaceConfig, err error) { @@ -158,18 +125,16 @@ func (h *HostEndpoint) getUserDefinedPolicies(state *PolicyState, tiers []*proto return conf, nil } -/* - This function creates the interface configuration for the host, applied on the vpptap0 - interface i.e. the tap interface from VPP to the host - that we use as controlpoint for HostEndpoint implementation - We have an implicit workloadsToHostPolicy policy that controls the traffic from - workloads to their host: it is defined by felixConfig.DefaultEndpointToHostAction - We have an implicit failsafe rules policy defined by felixConfig as well. - - If there are no policies the default should be pass to profiles - If there are policies the default should be deny (profiles are ignored) -*/ -func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.InterfaceConfig, err error) { +// This function creates the interface configuration for the host, applied on the vpptap0 +// interface i.e. the tap interface from VPP to the host +// that we use as controlpoint for HostEndpoint implementation +// We have an implicit workloadsToHostPolicy policy that controls the traffic from +// workloads to their host: it is defined by felixConfig.DefaultEndpointToHostAction +// We have an implicit failsafe rules policy defined by felixConfig as well. +// +// If there are no policies the default should be pass to profiles +// If there are policies the default should be deny (profiles are ignored) +func (s *PoliciesHandler) getTapPolicies(h *HostEndpoint, state *PolicyState) (conf *types.InterfaceConfig, err error) { conf, err = h.getUserDefinedPolicies(state, h.Tiers) if err != nil { return nil, errors.Wrap(err, "cannot create host policies for TapConf") @@ -180,7 +145,7 @@ func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.Interface // (except for traffic allowed by failsafe rules). // note: this applies to ingress and egress separately, so if you don't have // ingress only you drop ingress - conf.IngressPolicyIDs = []uint32{h.server.workloadsToHostPolicy.VppID, h.server.failSafePolicy.VppID} + conf.IngressPolicyIDs = []uint32{s.workloadsToHostPolicy.VppID, s.failSafePolicy.VppID} conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_DENY } else { if len(conf.IngressPolicyIDs) > 0 { @@ -188,11 +153,11 @@ func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.Interface } else if len(conf.ProfileIDs) > 0 { conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_PASS } - conf.IngressPolicyIDs = append([]uint32{h.server.failSafePolicy.VppID}, conf.IngressPolicyIDs...) - conf.IngressPolicyIDs = append([]uint32{h.server.workloadsToHostPolicy.VppID}, conf.IngressPolicyIDs...) + conf.IngressPolicyIDs = append([]uint32{s.failSafePolicy.VppID}, conf.IngressPolicyIDs...) + conf.IngressPolicyIDs = append([]uint32{s.workloadsToHostPolicy.VppID}, conf.IngressPolicyIDs...) } if len(conf.EgressPolicyIDs) == 0 && len(conf.ProfileIDs) == 0 { - conf.EgressPolicyIDs = []uint32{h.server.AllowFromHostPolicy.VppID, h.server.failSafePolicy.VppID} + conf.EgressPolicyIDs = []uint32{s.AllowFromHostPolicy.VppID, s.failSafePolicy.VppID} conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_DENY } else { if len(conf.EgressPolicyIDs) > 0 { @@ -200,25 +165,25 @@ func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.Interface } else if len(conf.ProfileIDs) > 0 { conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_PASS } - conf.EgressPolicyIDs = append([]uint32{h.server.failSafePolicy.VppID}, conf.EgressPolicyIDs...) - conf.EgressPolicyIDs = append([]uint32{h.server.AllowFromHostPolicy.VppID}, conf.EgressPolicyIDs...) + conf.EgressPolicyIDs = append([]uint32{s.failSafePolicy.VppID}, conf.EgressPolicyIDs...) + conf.EgressPolicyIDs = append([]uint32{s.AllowFromHostPolicy.VppID}, conf.EgressPolicyIDs...) } return conf, nil } -func (h *HostEndpoint) getForwardPolicies(state *PolicyState) (conf *types.InterfaceConfig, err error) { +func (s *PoliciesHandler) getForwardPolicies(h *HostEndpoint, state *PolicyState) (conf *types.InterfaceConfig, err error) { conf, err = h.getUserDefinedPolicies(state, h.ForwardTiers) if err != nil { return nil, errors.Wrap(err, "cannot create host policies for forwardConf") } if len(conf.EgressPolicyIDs) > 0 { - conf.EgressPolicyIDs = append([]uint32{h.server.allowToHostPolicy.VppID}, conf.EgressPolicyIDs...) + conf.EgressPolicyIDs = append([]uint32{s.allowToHostPolicy.VppID}, conf.EgressPolicyIDs...) conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_DENY } else if len(conf.ProfileIDs) > 0 { conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_PASS } if len(conf.IngressPolicyIDs) > 0 { - conf.IngressPolicyIDs = append([]uint32{h.server.allowToHostPolicy.VppID}, conf.IngressPolicyIDs...) + conf.IngressPolicyIDs = append([]uint32{s.allowToHostPolicy.VppID}, conf.IngressPolicyIDs...) conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_DENY } else if len(conf.ProfileIDs) > 0 { conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_PASS @@ -226,26 +191,26 @@ func (h *HostEndpoint) getForwardPolicies(state *PolicyState) (conf *types.Inter return conf, nil } -func (h *HostEndpoint) Create(vpp *vpplink.VppLink, state *PolicyState) (err error) { - forwardConf, err := h.getForwardPolicies(state) +func (s *PoliciesHandler) CreateHostEndpoint(h *HostEndpoint, state *PolicyState) (err error) { + forwardConf, err := s.getForwardPolicies(h, state) if err != nil { return err } for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { - h.server.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, forwardConf) - err = vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /*invertRxTx*/) + s.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, forwardConf) + err = s.vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /*invertRxTx*/) if err != nil { return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) } } - h.currentForwardConf = forwardConf - tapConf, err := h.getTapPolicies(state) + h.CurrentForwardConf = forwardConf + tapConf, err := s.getTapPolicies(h, state) if err != nil { return err } for _, swIfIndex := range h.TapSwIfIndexes { - h.server.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, tapConf) - err = vpp.ConfigurePolicies(swIfIndex, tapConf, 0) + s.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, tapConf) + err = s.vpp.ConfigurePolicies(swIfIndex, tapConf, 0) if err != nil { return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) } @@ -253,26 +218,26 @@ func (h *HostEndpoint) Create(vpp *vpplink.VppLink, state *PolicyState) (err err return nil } -func (h *HostEndpoint) Update(vpp *vpplink.VppLink, new *HostEndpoint, state *PolicyState) (err error) { - forwardConf, err := new.getForwardPolicies(state) +func (s *PoliciesHandler) UpdateHostEndpoint(h *HostEndpoint, new *HostEndpoint, state *PolicyState) (err error) { + forwardConf, err := s.getForwardPolicies(new, state) if err != nil { return err } for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { - h.server.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, forwardConf) - err = vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /* invertRxTx */) + s.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, forwardConf) + err = s.vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /* invertRxTx */) if err != nil { return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) } } - h.currentForwardConf = forwardConf - tapConf, err := new.getTapPolicies(state) + h.CurrentForwardConf = forwardConf + tapConf, err := s.getTapPolicies(new, state) if err != nil { return err } for _, swIfIndex := range h.TapSwIfIndexes { - h.server.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, tapConf) - err = vpp.ConfigurePolicies(swIfIndex, tapConf, 0) + s.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, tapConf) + err = s.vpp.ConfigurePolicies(swIfIndex, tapConf, 0) if err != nil { return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) } @@ -284,22 +249,22 @@ func (h *HostEndpoint) Update(vpp *vpplink.VppLink, new *HostEndpoint, state *Po return nil } -func (h *HostEndpoint) Delete(vpp *vpplink.VppLink, state *PolicyState) (err error) { +func (s *PoliciesHandler) DeleteHostEndpoint(h *HostEndpoint, state *PolicyState) (err error) { for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { // Unconfigure forward policies - h.server.log.Infof("policy(del) interface swif=%d", swIfIndex) - err = vpp.ConfigurePolicies(swIfIndex, types.NewInterfaceConfig(), 0) + s.log.Infof("policy(del) interface swif=%d", swIfIndex) + err = s.vpp.ConfigurePolicies(swIfIndex, types.NewInterfaceConfig(), 0) if err != nil { return errors.Wrapf(err, "cannot unconfigure policies on interface %d", swIfIndex) } } for _, swIfIndex := range h.TapSwIfIndexes { // Unconfigure tap0 policies - h.server.log.Infof("policy(del) interface swif=%d", swIfIndex) + s.log.Infof("policy(del) interface swif=%d", swIfIndex) conf := types.NewInterfaceConfig() - conf.IngressPolicyIDs = h.server.defaultTap0IngressConf - conf.EgressPolicyIDs = h.server.defaultTap0EgressConf - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) + conf.IngressPolicyIDs = s.defaultTap0IngressConf + conf.EgressPolicyIDs = s.defaultTap0EgressConf + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) if err != nil { return errors.Wrapf(err, "cannot unconfigure policies on interface %d", swIfIndex) } @@ -309,3 +274,97 @@ func (h *HostEndpoint) Delete(vpp *vpplink.VppLink, state *PolicyState) (err err h.TunnelSwIfIndexes = []uint32{} return nil } + +func (s *PoliciesHandler) getAllTunnelSwIfIndexes() (swIfIndexes []uint32) { + swIfIndexes = make([]uint32, 0) + for k := range s.tunnelSwIfIndexes { + swIfIndexes = append(swIfIndexes, k) + } + return swIfIndexes +} + +func (s *PoliciesHandler) OnHostEndpointUpdate(msg *proto.HostEndpointUpdate) (err error) { + state := s.GetState() + id := FromProtoHostEndpointID(msg.Id) + hep, err := FromProtoHostEndpoint(msg.Endpoint) + if err != nil { + return err + } + if hep.InterfaceName != "" && hep.InterfaceName != "*" { + interfaceDetails, found := s.interfacesMap[hep.InterfaceName] + if found { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) + } else { + // we are not supposed to fallback to expectedIPs if interfaceName doesn't match + // this is the current behavior in calico linux + s.log.Errorf("cannot find host endpoint: interface named %s does not exist", hep.InterfaceName) + } + } else if hep.InterfaceName == "" && hep.ExpectedIPs != nil { + for _, existingIf := range s.interfacesMap { + interfaceFound: + for _, address := range existingIf.addresses { + for _, expectedIP := range hep.ExpectedIPs { + if address == expectedIP { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, existingIf.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, existingIf.tapIndex) + break interfaceFound + } + } + } + } + } else if hep.InterfaceName == "*" { + for _, interfaceDetails := range s.interfacesMap { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) + } + } + hep.TunnelSwIfIndexes = s.getAllTunnelSwIfIndexes() + if len(hep.UplinkSwIfIndexes) == 0 || len(hep.TapSwIfIndexes) == 0 { + s.log.Warnf("No interface in vpp for host endpoint id=%s hep=%s", id.EndpointID, hep.String()) + return nil + } + + existing, found := state.HostEndpoints[*id] + if found { + if s.state.IsPending() { + hep.CurrentForwardConf = existing.CurrentForwardConf + state.HostEndpoints[*id] = hep + } else { + err := s.UpdateHostEndpoint(existing, hep, state) + if err != nil { + return errors.Wrap(err, "cannot update host endpoint") + } + } + s.log.Infof("policy(upd) Updating host endpoint id=%s found=%t existing=%s new=%s", *id, found, existing, hep) + } else { + state.HostEndpoints[*id] = hep + if !s.state.IsPending() { + err := s.CreateHostEndpoint(hep, state) + if err != nil { + return errors.Wrap(err, "cannot create host endpoint") + } + } + s.log.Infof("policy(add) Updating host endpoint id=%s found=%t new=%s", *id, found, hep) + } + return nil +} + +func (s *PoliciesHandler) OnHostEndpointRemove(msg *proto.HostEndpointRemove) (err error) { + state := s.GetState() + id := FromProtoHostEndpointID(msg.Id) + existing, ok := state.HostEndpoints[*id] + if !ok { + s.log.Warnf("Received host endpoint delete for id=%s that doesn't exists", id) + return nil + } + if !s.state.IsPending() && len(existing.UplinkSwIfIndexes) != 0 { + err = s.DeleteHostEndpoint(existing, s.configuredState) + if err != nil { + return errors.Wrap(err, "error deleting host endpoint") + } + } + s.log.Infof("policy(del) Handled Host Endpoint Remove pending=%t id=%s %s", s.state.IsPending(), id, existing) + delete(state.HostEndpoints, *id) + return nil +} diff --git a/calico-vpp-agent/felix/policies/hostmetadata.go b/calico-vpp-agent/felix/policies/hostmetadata.go new file mode 100644 index 000000000..50d9857dc --- /dev/null +++ b/calico-vpp-agent/felix/policies/hostmetadata.go @@ -0,0 +1,133 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "fmt" + "net" + + "github.com/pkg/errors" + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type NodeWatcherRestartError struct{} + +func (e NodeWatcherRestartError) Error() string { + return "node configuration changed, restarting" +} + +func (s *PoliciesHandler) OnHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Update) (err error) { + localNodeSpec, err := common.NewLocalNodeSpec(msg) + if err != nil { + return errors.Wrapf(err, "OnHostMetadataV4V6Update errored") + } + old, found := s.cache.NodeStatesByName[localNodeSpec.Name] + + isOwnNode := localNodeSpec.Name == *config.NodeName + if isOwnNode && + (localNodeSpec.IPv4Address != nil || localNodeSpec.IPv6Address != nil) { + /* We found a BGP Spec that seems valid enough */ + s.GotOurNodeBGPchanOnce.Do(func() { + s.GotOurNodeBGPchan <- localNodeSpec + }) + ip4 := net.IP{} + ip6 := net.IP{} + if localNodeSpec.IPv4Address != nil { + ip4 = localNodeSpec.IPv4Address.IP + } + if localNodeSpec.IPv6Address != nil { + ip6 = localNodeSpec.IPv6Address.IP + } + err = s.vpp.CnatSetSnatAddresses(ip4, ip6) + if err != nil { + s.log.Errorf("Failed to configure SNAT addresses %v", err) + } + err = s.CreateAllowFromHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in creating AllowFromHostPolicy") + } + err = s.CreateAllowToHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in createAllowToHostPolicy") + } + } + + // This is used by the routing server to process Wireguard key updates + // As a result we only send an event when a node is updated, not when it is added or deleted + common.SendEvent(common.CalicoVppEvent{ + Type: common.PeerNodeStateChanged, + Old: old, + New: localNodeSpec, + }) + + if !found { + if !isOwnNode { + s.configureRemoteNodeSnat(localNodeSpec, true /* isAdd */) + } + } else { + change := common.GetIPNetChangeType(old.IPv4Address, localNodeSpec.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, localNodeSpec.IPv6Address) + if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && localNodeSpec.Name == *config.NodeName { + // restart if our BGP config changed + return NodeWatcherRestartError{} + } + if change != common.ChangeSame { + if !isOwnNode { + s.configureRemoteNodeSnat(old, false /* isAdd */) + s.configureRemoteNodeSnat(localNodeSpec, true /* isAdd */) + } + } + } + + s.cache.NodeStatesByName[localNodeSpec.Name] = localNodeSpec + return nil +} + +func (s *PoliciesHandler) OnHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Remove) (err error) { + old, found := s.cache.NodeStatesByName[msg.Hostname] + if !found { + return fmt.Errorf("node %s to delete not found", msg.Hostname) + } + + common.SendEvent(common.CalicoVppEvent{ + Type: common.PeerNodeStateChanged, + Old: old, + }) + if old.Name == *config.NodeName { + // restart if our BGP config changed + return NodeWatcherRestartError{} + } + + s.configureRemoteNodeSnat(old, false /* isAdd */) + return nil +} + +func (s *PoliciesHandler) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) { + if node.IPv4Address != nil { + err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd) + if err != nil { + s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err) + } + } + if node.IPv6Address != nil { + err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd) + if err != nil { + s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err) + } + } +} diff --git a/calico-vpp-agent/felix/ipset.go b/calico-vpp-agent/felix/policies/ipset.go similarity index 80% rename from calico-vpp-agent/felix/ipset.go rename to calico-vpp-agent/felix/policies/ipset.go index ebe4b5650..3b0bc3cc6 100644 --- a/calico-vpp-agent/felix/ipset.go +++ b/calico-vpp-agent/felix/policies/ipset.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -153,7 +153,7 @@ func toNetArray(addrs map[string]*net.IPNet) []*net.IPNet { return array } -func fromIPSetUpdate(ips *proto.IPSetUpdate) (i *IPSet, err error) { +func FromIPSetUpdate(ips *proto.IPSetUpdate) (i *IPSet, err error) { i = NewIPSet() switch ips.GetType() { case proto.IPSetUpdate_IP: @@ -330,3 +330,59 @@ func (i *IPSet) RemoveMembers(members []string, apply bool, vpp *vpplink.VppLink } return err } + +func (s *PoliciesHandler) OnIpsetUpdate(msg *proto.IPSetUpdate) (err error) { + ips, err := FromIPSetUpdate(msg) + if err != nil { + return errors.Wrap(err, "cannot process IPSetUpdate") + } + state := s.GetState() + _, ok := state.IPSets[msg.GetId()] + if ok { + return fmt.Errorf("received new ipset for ID %s that already exists", msg.GetId()) + } + if !s.state.IsPending() { + err = ips.Create(s.vpp) + if err != nil { + return errors.Wrapf(err, "cannot create ipset %s", msg.GetId()) + } + } + state.IPSets[msg.GetId()] = ips + s.log.Debugf("Handled Ipset Update pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + return nil +} + +func (s *PoliciesHandler) OnIpsetDeltaUpdate(msg *proto.IPSetDeltaUpdate) (err error) { + ips, ok := s.GetState().IPSets[msg.GetId()] + if !ok { + return fmt.Errorf("received delta update for non-existent ipset") + } + err = ips.AddMembers(msg.GetAddedMembers(), !s.state.IsPending(), s.vpp) + if err != nil { + return errors.Wrap(err, "cannot process ipset delta update") + } + err = ips.RemoveMembers(msg.GetRemovedMembers(), !s.state.IsPending(), s.vpp) + if err != nil { + return errors.Wrap(err, "cannot process ipset delta update") + } + s.log.Debugf("Handled Ipset delta Update pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + return nil +} + +func (s *PoliciesHandler) OnIpsetRemove(msg *proto.IPSetRemove) (err error) { + state := s.GetState() + ips, ok := state.IPSets[msg.GetId()] + if !ok { + s.log.Warnf("Received ipset delete for ID %s that doesn't exists", msg.GetId()) + return nil + } + if !s.state.IsPending() { + err = ips.Delete(s.vpp) + if err != nil { + return errors.Wrapf(err, "cannot delete ipset %s", msg.GetId()) + } + } + s.log.Debugf("Handled Ipset remove pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + delete(state.IPSets, msg.GetId()) + return nil +} diff --git a/calico-vpp-agent/felix/policies/policies_handler.go b/calico-vpp-agent/felix/policies/policies_handler.go new file mode 100644 index 000000000..88ad0eb54 --- /dev/null +++ b/calico-vpp-agent/felix/policies/policies_handler.go @@ -0,0 +1,375 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "fmt" + "net" + "strings" + "sync" + + "github.com/pkg/errors" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +// Server holds all the data required to configure the policies defined by felix in VPP +type PoliciesHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + endpointsInterfaces map[WorkloadEndpointID]map[string]uint32 + tunnelSwIfIndexes map[uint32]bool + interfacesMap map[string]interfaceDetails + + configuredState *PolicyState + pendingState *PolicyState + + state common.FelixSocketSyncState + + /* failSafe policies allow traffic on some ports irrespective of the policy */ + failSafePolicy *Policy + /* workloadToHost may drop traffic that goes from the pods to the host */ + workloadsToHostPolicy *Policy + defaultTap0IngressConf []uint32 + defaultTap0EgressConf []uint32 + /* always allow traffic coming from host to the pods (for healthchecks and so on) */ + // AllowFromHostPolicy persists the policy allowing host --> pod communications. + // See CreateAllowFromHostPolicy definition + AllowFromHostPolicy *Policy + // allPodsIpset persists the ipset containing all the workload endpoints (pods) addresses + allPodsIpset *IPSet + /* allow traffic between uplink/tunnels and tap interfaces */ + allowToHostPolicy *Policy + + GotOurNodeBGPchan chan *common.LocalNodeSpec + GotOurNodeBGPchanOnce sync.Once +} + +func NewPoliciesHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calicov3cli.Interface, log *logrus.Entry) *PoliciesHandler { + return &PoliciesHandler{ + log: log, + vpp: vpp, + cache: cache, + endpointsInterfaces: make(map[WorkloadEndpointID]map[string]uint32), + tunnelSwIfIndexes: make(map[uint32]bool), + + configuredState: NewPolicyState(), + pendingState: NewPolicyState(), + state: common.StateDisconnected, + + GotOurNodeBGPchan: make(chan *common.LocalNodeSpec), + } +} + +func (s *PoliciesHandler) RefreshInterfaceMap() error { + interfacesMap, err := mapTagToInterfaceDetails(s.vpp) + if err != nil { + return err + } + s.interfacesMap = interfacesMap + return nil +} + +func (s *PoliciesHandler) GetState() *PolicyState { + if s.state.IsPending() { + return s.pendingState + } + return s.configuredState +} + +func (s *PoliciesHandler) OnInSync(msg *proto.InSync) (err error) { + if s.state != common.StateSyncing { + return fmt.Errorf("received InSync but state was not syncing") + } + + s.state = common.StateInSync + s.log.Infof("Policies now in sync") + return s.ApplyPendingState() +} + +// workloadAdded is called by the CNI server when a container interface is created, +// either during startup when reconnecting the interfaces, or when a new pod is created +func (s *PoliciesHandler) OnWorkloadAdded(id *WorkloadEndpointID, swIfIndex uint32, ifName string, containerIPs []*net.IPNet) { + // TODO: Send WorkloadEndpointStatusUpdate to felix + + intf, existing := s.endpointsInterfaces[*id] + + if existing { + for _, exInt := range intf { + if swIfIndex == exInt { + return + } + } + // VPP restarted and interfaces are being reconnected + s.log.Warnf("workload endpoint changed interfaces, did VPP restart? %v %v -> %d", id, intf, swIfIndex) + s.endpointsInterfaces[*id][ifName] = swIfIndex + } + + s.log.Infof("policy(add) Workload id=%v swIfIndex=%d", id, swIfIndex) + if s.endpointsInterfaces[*id] == nil { + s.endpointsInterfaces[*id] = map[string]uint32{ifName: swIfIndex} + } else { + s.endpointsInterfaces[*id][ifName] = swIfIndex + } + + if s.state == common.StateInSync { + wep, ok := s.configuredState.WorkloadEndpoints[*id] + if !ok { + s.log.Infof("not creating wep in workloadadded") + // Nothing to configure + } else { + s.log.Infof("creating wep in workloadadded") + err := s.CreateWorkloadEndpoint(wep, []uint32{swIfIndex}, s.configuredState, id.Network) + if err != nil { + s.log.Errorf("Error processing workload addition: %s", err) + } + } + } + // EndpointToHostAction + allMembers := []string{} + for _, containerIP := range containerIPs { + allMembers = append(allMembers, containerIP.IP.String()) + } + err := s.allPodsIpset.AddMembers(allMembers, true, s.vpp) + if err != nil { + s.log.Errorf("Error processing workload addition: %s", err) + } +} + +// WorkloadRemoved is called by the CNI server when the interface of a pod is deleted +func (s *PoliciesHandler) OnWorkloadRemoved(id *WorkloadEndpointID, containerIPs []*net.IPNet) { + // TODO: Send WorkloadEndpointStatusRemove to felix + + _, existing := s.endpointsInterfaces[*id] + if !existing { + s.log.Warnf("nonexistent workload endpoint removed %v", id) + return + } + s.log.Infof("policy(del) workload id=%v", id) + + if s.state == common.StateInSync { + wep, ok := s.configuredState.WorkloadEndpoints[*id] + if !ok { + // Nothing to clean up + } else { + err := s.DeleteWorkloadEndpoint(wep) + if err != nil { + s.log.Errorf("Error processing workload removal: %s", err) + } + } + } + delete(s.endpointsInterfaces, *id) + // EndpointToHostAction + allMembers := []string{} + for _, containerIP := range containerIPs { + allMembers = append(allMembers, containerIP.IP.String()) + } + err := s.allPodsIpset.RemoveMembers(allMembers, true, s.vpp) + if err != nil { + s.log.Errorf("Error processing workload remove: %s", err) + } +} + +func (s *PoliciesHandler) OnTunnelAdded(swIfIndex uint32) { + s.tunnelSwIfIndexes[swIfIndex] = true + for _, h := range s.GetState().HostEndpoints { + newTunnel := true + for _, v := range h.TunnelSwIfIndexes { + if v == swIfIndex { + newTunnel = false + } + } + if newTunnel { + h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes, swIfIndex) + s.log.Infof("Configuring policies on added tunnel [%d]", swIfIndex) + if !s.state.IsPending() { + s.log.Infof("policy(upd) interface swif=%d", swIfIndex) + err := s.vpp.ConfigurePolicies(swIfIndex, h.CurrentForwardConf, 1 /*invertRxTx*/) + if err != nil { + s.log.WithError(err).Errorf("OnTunnelAdded: cannot configure policies on tunnel interface %d", swIfIndex) + } + } + } + } +} +func (s *PoliciesHandler) OnTunnelDelete(swIfIndex uint32) { + delete(s.tunnelSwIfIndexes, swIfIndex) + state := s.GetState() + for _, h := range state.HostEndpoints { + for index, existingSwifindex := range h.TunnelSwIfIndexes { + if existingSwifindex == swIfIndex { + // we don't delete the policies because they are auto-deleted when interfaces are removed + h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes[:index], h.TunnelSwIfIndexes[index+1:]...) + } + } + } +} + +func (s *PoliciesHandler) OnFelixSocketStateChanged(evt *common.FelixSocketStateChanged) { + s.state = evt.NewState +} + +func (s *PoliciesHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if s.state != common.StateConnected { + s.log.Errorf("received ConfigUpdate but server is not in Connected state! state: %v", s.state) + return + } + s.state = common.StateSyncing + if s.cache.FelixConfig.DefaultEndpointToHostAction != old.DefaultEndpointToHostAction { + s.log.Infof("Change in EndpointToHostAction to %+v", s.getEndpointToHostAction()) + workloadsToHostAllowRule := &Rule{ + VppID: types.InvalidID, + Rule: &types.Rule{ + Action: s.getEndpointToHostAction(), + }, + SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + policy := s.workloadsToHostPolicy.DeepCopy() + policy.InboundRules = []*Rule{workloadsToHostAllowRule} + err := s.workloadsToHostPolicy.Update(s.vpp, policy, + &PolicyState{ + IPSets: map[string]*IPSet{ + "calico-vpp-wep-addr-ipset": s.allPodsIpset, + }, + }) + if err != nil { + s.log.Errorf("error updating workloadsToHostPolicy %v", err) + return + } + } + if !protoPortListEqual(s.cache.FelixConfig.FailsafeInboundHostPorts, old.FailsafeInboundHostPorts) || + !protoPortListEqual(s.cache.FelixConfig.FailsafeOutboundHostPorts, old.FailsafeOutboundHostPorts) { + err := s.CreateFailSafePolicies() + if err != nil { + s.log.Errorf("error updating FailSafePolicies %v", err) + return + } + } +} + +// Reconciles the pending state with the configured state +func (s *PoliciesHandler) ApplyPendingState() (err error) { + s.log.Infof("Reconciliating pending policy state with configured state") + // Stupid algorithm for now, delete all that is in configured state, and then recreate everything + for _, wep := range s.configuredState.WorkloadEndpoints { + if len(wep.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(wep) + if err != nil { + return errors.Wrap(err, "cannot cleanup workload endpoint") + } + } + } + for _, policy := range s.configuredState.Policies { + err = policy.Delete(s.vpp, s.configuredState) + if err != nil { + s.log.Warnf("error deleting policy: %v", err) + } + } + for _, profile := range s.configuredState.Profiles { + err = profile.Delete(s.vpp, s.configuredState) + if err != nil { + s.log.Warnf("error deleting profile: %v", err) + } + } + for _, ipset := range s.configuredState.IPSets { + err = ipset.Delete(s.vpp) + if err != nil { + s.log.Warnf("error deleting ipset: %v", err) + } + } + for _, hep := range s.configuredState.HostEndpoints { + if len(hep.UplinkSwIfIndexes) != 0 { + err = s.DeleteHostEndpoint(hep, s.configuredState) + if err != nil { + s.log.Warnf("error deleting hostendpoint : %v", err) + } + } + } + + s.configuredState = s.pendingState + s.pendingState = NewPolicyState() + for _, ipset := range s.configuredState.IPSets { + err = ipset.Create(s.vpp) + if err != nil { + return errors.Wrap(err, "error creating ipset") + } + } + for _, profile := range s.configuredState.Profiles { + err = profile.Create(s.vpp, s.configuredState) + if err != nil { + return errors.Wrap(err, "error creating profile") + } + } + for _, policy := range s.configuredState.Policies { + err = policy.Create(s.vpp, s.configuredState) + if err != nil { + return errors.Wrap(err, "error creating policy") + } + } + for id, wep := range s.configuredState.WorkloadEndpoints { + intf, intfFound := s.endpointsInterfaces[id] + if intfFound { + swIfIndexList := []uint32{} + for _, idx := range intf { + swIfIndexList = append(swIfIndexList, idx) + } + err = s.CreateWorkloadEndpoint(wep, swIfIndexList, s.configuredState, id.Network) + if err != nil { + return errors.Wrap(err, "cannot configure workload endpoint") + } + } + } + for _, hep := range s.configuredState.HostEndpoints { + err = s.CreateHostEndpoint(hep, s.configuredState) + if err != nil { + return errors.Wrap(err, "cannot create host endpoint") + } + } + s.log.Infof("Reconciliation done") + return nil +} + +func (s *PoliciesHandler) OnNodeAddUpdate(node *common.LocalNodeSpec) { + if node.Name == *config.NodeName { + err := s.CreateAllowFromHostPolicy() + if err != nil { + s.log.Errorf("Error in creating AllowFromHostPolicy %v", err) + return + } + err = s.CreateAllowToHostPolicy() + if err != nil { + s.log.Errorf("Error in createAllowToHostPolicy %v", err) + return + } + } +} + +func (s *PoliciesHandler) getEndpointToHostAction() types.RuleAction { + if strings.ToUpper(s.cache.FelixConfig.DefaultEndpointToHostAction) == "ACCEPT" { + return types.ActionAllow + } + return types.ActionDeny +} diff --git a/calico-vpp-agent/felix/policies/policies_init.go b/calico-vpp-agent/felix/policies/policies_init.go new file mode 100644 index 000000000..f0e130fff --- /dev/null +++ b/calico-vpp-agent/felix/policies/policies_init.go @@ -0,0 +1,300 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "fmt" + "net" + + "github.com/pkg/errors" + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +func (s *PoliciesHandler) CreateAllPodsIpset() (err error) { + ipset := NewIPSet() + err = ipset.Create(s.vpp) + if err != nil { + return err + } + s.allPodsIpset = ipset + return nil +} + +// CreateAllowFromHostPolicy creates a policy allowing host->pod communications. This is needed +// to maintain vanilla Calico's behavior where the host can always reach pods. +// This policy is applied in Egress on the host endpoint tap (i.e. linux -> VPP) +// and on the Ingress of Workload endpoints (i.e. VPP -> pod) +func (s *PoliciesHandler) CreateAllowFromHostPolicy() (err error) { + s.log.Infof("Creating rules to allow traffic from host to pods with egress policies") + ruleOut := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-egressallowfromhost", + Rule: &types.Rule{ + Action: types.ActionAllow, + }, + DstIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} + s.log.Infof("Creating rules to allow traffic from host to pods with ingress policies") + ruleIn := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-ingressallowfromhost", + Rule: &types.Rule{ + Action: types.ActionAllow, + SrcNet: []net.IPNet{}, + }, + } + if s.cache.GetNodeIP4() != nil { + ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + } + if s.cache.GetNodeIP6() != nil { + ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + } + + allowFromHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + allowFromHostPolicy.OutboundRules = append(allowFromHostPolicy.OutboundRules, ruleOut) + allowFromHostPolicy.InboundRules = append(allowFromHostPolicy.InboundRules, ruleIn) + if s.AllowFromHostPolicy == nil { + err = allowFromHostPolicy.Create(s.vpp, &ps) + } else { + allowFromHostPolicy.VppID = s.AllowFromHostPolicy.VppID + err = s.AllowFromHostPolicy.Update(s.vpp, allowFromHostPolicy, &ps) + } + s.AllowFromHostPolicy = allowFromHostPolicy + if err != nil { + return errors.Wrap(err, "cannot create policy to allow traffic from host to pods") + } + s.log.Infof("Created allow from host to pods traffic with ID: %+v", s.AllowFromHostPolicy.VppID) + return nil +} + +func (s *PoliciesHandler) CreateEndpointToHostPolicy( /*may be return*/ ) (err error) { + workloadsToHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + workloadsToHostRule := &Rule{ + VppID: types.InvalidID, + Rule: &types.Rule{ + Action: s.getEndpointToHostAction(), + }, + SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + ps := PolicyState{ + IPSets: map[string]*IPSet{ + "calico-vpp-wep-addr-ipset": s.allPodsIpset, + }, + } + workloadsToHostPolicy.InboundRules = append(workloadsToHostPolicy.InboundRules, workloadsToHostRule) + + err = workloadsToHostPolicy.Create(s.vpp, &ps) + if err != nil { + return err + } + s.workloadsToHostPolicy = workloadsToHostPolicy + + conf := types.NewInterfaceConfig() + conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, s.workloadsToHostPolicy.VppID) + conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_ALLOW + conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_ALLOW + swifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") // tap0 interfaces + if err != nil { + s.log.Error(err) + } + for _, swifindex := range swifindexes { + err = s.vpp.ConfigurePolicies(uint32(swifindex), conf, 0) + if err != nil { + s.log.Error("cannot create policy to drop traffic to host") + } + } + s.defaultTap0IngressConf = conf.IngressPolicyIDs + s.defaultTap0EgressConf = conf.EgressPolicyIDs + return nil +} + +// CreateFailSafePolicies ensures the failsafe policies defined in the Felixconfiguration exist in VPP. +// check https://github.com/projectcalico/calico/blob/master/felix/rules/static.go :: failsafeInChain for the linux implementation +// To be noted. This does not implement the doNotTrack case as we do not yet support doNotTrack +func (s *PoliciesHandler) CreateFailSafePolicies() (err error) { + failSafePol := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + + if len(s.cache.FelixConfig.FailsafeInboundHostPorts) != 0 { + for _, protoPort := range s.cache.FelixConfig.FailsafeInboundHostPorts { + protocol, err := ParseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) + if err != nil { + s.log.WithError(err).Error("Failed to parse protocol in inbound failsafe rule. Skipping failsafe rule") + continue + } + rule := &Rule{ + VppID: types.InvalidID, + RuleID: fmt.Sprintf("failsafe-in-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), + Rule: &types.Rule{ + Action: types.ActionAllow, + // Ports are always filtered on the destination of packets + DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, + Filters: []types.RuleFilter{{ + ShouldMatch: true, + Type: types.NpolFilterProto, + Value: int(protocol), + }}, + }, + } + if protoPort.Net != "" { + _, protoPortNet, err := net.ParseCIDR(protoPort.Net) + if err != nil { + s.log.WithError(err).Error("Failed to parse CIDR in inbound failsafe rule. Skipping failsafe rule") + continue + } + // Inbound packets are checked for where they come FROM + rule.SrcNet = append(rule.SrcNet, *protoPortNet) + } + failSafePol.InboundRules = append(failSafePol.InboundRules, rule) + } + } + + if len(s.cache.FelixConfig.FailsafeOutboundHostPorts) != 0 { + for _, protoPort := range s.cache.FelixConfig.FailsafeOutboundHostPorts { + protocol, err := ParseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) + if err != nil { + s.log.WithError(err).Error("Failed to parse protocol in outbound failsafe rule. Skipping failsafe rule") + continue + } + rule := &Rule{ + VppID: types.InvalidID, + RuleID: fmt.Sprintf("failsafe-out-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), + Rule: &types.Rule{ + Action: types.ActionAllow, + // Ports are always filtered on the destination of packets + DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, + Filters: []types.RuleFilter{{ + ShouldMatch: true, + Type: types.NpolFilterProto, + Value: int(protocol), + }}, + }, + } + if protoPort.Net != "" { + _, protoPortNet, err := net.ParseCIDR(protoPort.Net) + if err != nil { + s.log.WithError(err).Error("Failed to parse CIDR in outbound failsafe rule. Skipping failsafe rule") + continue + } + // Outbound packets are checked for where they go TO + rule.DstNet = append(rule.DstNet, *protoPortNet) + } + failSafePol.OutboundRules = append(failSafePol.OutboundRules, rule) + } + } + + if s.failSafePolicy == nil { + err = failSafePol.Create(s.vpp, nil) + + } else { + failSafePol.VppID = s.failSafePolicy.VppID + err = s.failSafePolicy.Update(s.vpp, failSafePol, nil) + } + if err != nil { + return err + } + s.failSafePolicy = failSafePol + s.log.Infof("Created failsafe policy with ID %+v", s.failSafePolicy.VppID) + return nil +} + +func (s *PoliciesHandler) CreateAllowToHostPolicy() (err error) { + s.log.Infof("Creating policy to allow traffic to host that is applied on uplink") + ruleIn := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-allowtohost", + Rule: &types.Rule{ + Action: types.ActionAllow, + DstNet: []net.IPNet{}, + }, + } + ruleOut := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-allowtohost", + Rule: &types.Rule{ + Action: types.ActionAllow, + SrcNet: []net.IPNet{}, + }, + } + if s.cache.GetNodeIP4() != nil { + ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + } + if s.cache.GetNodeIP6() != nil { + ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + } + + allowToHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + allowToHostPolicy.InboundRules = append(allowToHostPolicy.InboundRules, ruleIn) + allowToHostPolicy.OutboundRules = append(allowToHostPolicy.OutboundRules, ruleOut) + if s.allowToHostPolicy == nil { + err = allowToHostPolicy.Create(s.vpp, nil) + } else { + allowToHostPolicy.VppID = s.allowToHostPolicy.VppID + err = s.allowToHostPolicy.Update(s.vpp, allowToHostPolicy, nil) + } + s.allowToHostPolicy = allowToHostPolicy + if err != nil { + return errors.Wrap(err, "cannot create policy to allow traffic to host") + } + s.log.Infof("Created policy to allow traffic to host with ID: %+v", s.allowToHostPolicy.VppID) + return nil +} + +func (s *PoliciesHandler) PoliciesHandlerInit() error { + err := s.CreateAllPodsIpset() + if err != nil { + return errors.Wrap(err, "Error in createallPodsIpset") + } + err = s.CreateEndpointToHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in createEndpointToHostPolicy") + } + err = s.CreateAllowFromHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in creating AllowFromHostPolicy") + } + err = s.CreateAllowToHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in createAllowToHostPolicy") + } + err = s.CreateFailSafePolicies() + if err != nil { + return errors.Wrap(err, "Error in createFailSafePolicies") + } + s.interfacesMap, err = mapTagToInterfaceDetails(s.vpp) + if err != nil { + return errors.Wrap(err, "Error in mapping uplink to tap interfaces") + } + return nil +} diff --git a/calico-vpp-agent/felix/policy.go b/calico-vpp-agent/felix/policies/policy.go similarity index 59% rename from calico-vpp-agent/felix/policy.go rename to calico-vpp-agent/felix/policies/policy.go index 152c68895..4ac845f25 100644 --- a/calico-vpp-agent/felix/policy.go +++ b/calico-vpp-agent/felix/policies/policy.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -91,7 +91,7 @@ func ruleInNetwork(r *proto.Rule, network string) bool { return network == "" } -func fromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error) { +func FromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error) { policy = &Policy{ Policy: &types.Policy{}, VppID: types.InvalidID, @@ -129,7 +129,7 @@ func fromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error return policy, nil } -func fromProtoProfile(p *proto.Profile) (profile *Policy, err error) { +func FromProtoProfile(p *proto.Profile) (profile *Policy, err error) { profile = &Policy{ Policy: &types.Policy{}, VppID: types.InvalidID, @@ -244,3 +244,145 @@ func (p *Policy) Delete(vpp *vpplink.VppLink, state *PolicyState) (err error) { p.VppID = types.InvalidID return nil } + +func (s *PoliciesHandler) OnActivePolicyUpdate(msg *proto.ActivePolicyUpdate) (err error) { + state := s.GetState() + id := fromProtoPolicyID(msg.Id, defaultNetwork) + p, err := FromProtoPolicy(msg.Policy, "") + if err != nil { + return errors.Wrapf(err, "cannot process policy update") + } + + s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", s.state.IsPending(), id, p) + existing, ok := state.Policies[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Policies[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update policy") + } + } + } else { + // Create it in state + state.Policies[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create policy") + } + } + } + + for network := range s.cache.NetworkDefinitions { + id := fromProtoPolicyID(msg.Id, network) + p, err := FromProtoPolicy(msg.Policy, network) + if err != nil { + return errors.Wrapf(err, "cannot process policy update") + } + + s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", s.state.IsPending(), id, p) + + existing, ok := state.Policies[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Policies[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update policy") + } + } + } else { + // Create it in state + state.Policies[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create policy") + } + } + } + + } + return nil +} + +func (s *PoliciesHandler) OnActivePolicyRemove(msg *proto.ActivePolicyRemove) (err error) { + state := s.GetState() + id := fromProtoPolicyID(msg.Id, defaultNetwork) + s.log.Infof("policy(del) Handling ActivePolicyRemove pending=%t id=%s", s.state.IsPending(), id) + + for policyID := range state.Policies { + if policyID.Name == id.Name && policyID.Namespace == id.Namespace && policyID.Kind == id.Kind { + existing, ok := state.Policies[policyID] + if !ok { + s.log.Warnf("Received policy delete for kind %s namespace %s name %s that doesn't exists", id.Kind, id.Namespace, id.Name) + return nil + } + if !s.state.IsPending() { + err = existing.Delete(s.vpp, state) + if err != nil { + return errors.Wrap(err, "error deleting policy") + } + } + delete(state.Policies, policyID) + } + } + return nil +} + +func (s *PoliciesHandler) OnActiveProfileUpdate(msg *proto.ActiveProfileUpdate) (err error) { + state := s.GetState() + id := msg.Id.Name + p, err := FromProtoProfile(msg.Profile) + if err != nil { + return errors.Wrapf(err, "cannot process profile update") + } + + existing, ok := state.Profiles[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Profiles[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update profile") + } + } + } else { + // Create it in state + state.Profiles[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create profile") + } + } + } + s.log.Infof("policy(upd) Handled Profile Update pending=%t id=%s existing=%s new=%s", s.state.IsPending(), id, existing, p) + return nil +} + +func (s *PoliciesHandler) OnActiveProfileRemove(msg *proto.ActiveProfileRemove) (err error) { + state := s.GetState() + id := msg.Id.Name + existing, ok := state.Profiles[id] + if !ok { + s.log.Warnf("Received profile delete for Name %s that doesn't exists", id) + return nil + } + if !s.state.IsPending() { + err = existing.Delete(s.vpp, state) + if err != nil { + return errors.Wrap(err, "error deleting profile") + } + } + s.log.Infof("policy(del) Handled Profile Remove pending=%t id=%s policy=%s", s.state.IsPending(), id, existing) + delete(state.Profiles, id) + return nil +} diff --git a/calico-vpp-agent/felix/policy_state.go b/calico-vpp-agent/felix/policies/policy_state.go similarity index 98% rename from calico-vpp-agent/felix/policy_state.go rename to calico-vpp-agent/felix/policies/policy_state.go index 1d38741f0..ec097522f 100644 --- a/calico-vpp-agent/felix/policy_state.go +++ b/calico-vpp-agent/felix/policies/policy_state.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies type PolicyState struct { IPSets map[string]*IPSet diff --git a/calico-vpp-agent/felix/rule.go b/calico-vpp-agent/felix/policies/rule.go similarity index 98% rename from calico-vpp-agent/felix/rule.go rename to calico-vpp-agent/felix/policies/rule.go index c8742a8de..51488853f 100644 --- a/calico-vpp-agent/felix/rule.go +++ b/calico-vpp-agent/felix/policies/rule.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -127,7 +127,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { if r.NotProtocol != nil { return nil, fmt.Errorf("protocol and NotProtocol specified in Rule") } - proto, err := parseProtocol(r.Protocol) + proto, err := ParseProtocol(r.Protocol) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { }) } if r.NotProtocol != nil { - proto, err := parseProtocol(r.NotProtocol) + proto, err := ParseProtocol(r.NotProtocol) if err != nil { return nil, err } @@ -220,7 +220,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { return rule, nil } -func parseProtocol(pr *proto.Protocol) (types.IPProto, error) { +func ParseProtocol(pr *proto.Protocol) (types.IPProto, error) { switch u := pr.NumberOrName.(type) { case *proto.Protocol_Name: switch strings.ToLower(u.Name) { diff --git a/calico-vpp-agent/felix/policies/utils.go b/calico-vpp-agent/felix/policies/utils.go new file mode 100644 index 000000000..f2741a402 --- /dev/null +++ b/calico-vpp-agent/felix/policies/utils.go @@ -0,0 +1,82 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "github.com/pkg/errors" + felixConfig "github.com/projectcalico/calico/felix/config" + + "github.com/projectcalico/vpp-dataplane/v3/vpplink" +) + +func protoPortListEqual(a, b []felixConfig.ProtoPort) bool { + if len(a) != len(b) { + return false + } + for i, elemA := range a { + elemB := b[i] + if elemA.Net != elemB.Net { + return false + } + if elemA.Protocol != elemB.Protocol { + return false + } + if elemA.Port != elemB.Port { + return false + } + } + return true +} + +type interfaceDetails struct { + tapIndex uint32 + uplinkIndex uint32 + addresses []string +} + +func mapTagToInterfaceDetails(vpp *vpplink.VppLink) (tagIfDetails map[string]interfaceDetails, err error) { + tagIfDetails = make(map[string]interfaceDetails) + uplinkSwifindexes, err := vpp.SearchInterfacesWithTagPrefix("main-") + if err != nil { + return nil, err + } + tapSwifindexes, err := vpp.SearchInterfacesWithTagPrefix("host-") + if err != nil { + return nil, err + } + for intf, uplink := range uplinkSwifindexes { + tap, found := tapSwifindexes["host-"+intf[5:]] + if found { + ip4adds, err := vpp.AddrList(uplink, false) + if err != nil { + return nil, err + } + ip6adds, err := vpp.AddrList(uplink, true) + if err != nil { + return nil, err + } + adds := append(ip4adds, ip6adds...) + addresses := []string{} + for _, add := range adds { + addresses = append(addresses, add.IPNet.IP.String()) + } + tagIfDetails[intf[5:]] = interfaceDetails{tap, uplink, addresses} + } else { + return nil, errors.Errorf("uplink interface %d not corresponding to a tap interface", uplink) + } + } + return tagIfDetails, nil +} diff --git a/calico-vpp-agent/felix/policies/workload_endpoint.go b/calico-vpp-agent/felix/policies/workload_endpoint.go new file mode 100644 index 000000000..8e6d2f228 --- /dev/null +++ b/calico-vpp-agent/felix/policies/workload_endpoint.go @@ -0,0 +1,278 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "encoding/json" + "fmt" + + nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/pkg/errors" + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type WorkloadEndpointID struct { + OrchestratorID string + WorkloadID string + EndpointID string + Network string +} + +func (wi *WorkloadEndpointID) String() string { + return fmt.Sprintf("%s:%s:%s:%s", wi.OrchestratorID, wi.WorkloadID, wi.EndpointID, wi.Network) +} + +type Tier struct { + Name string + IngressPolicies []string + EgressPolicies []string +} + +func (tr *Tier) String() string { + s := fmt.Sprintf("name=%s", tr.Name) + s += types.StrListToString(" IngressPolicies=", tr.IngressPolicies) + s += types.StrListToString(" EgressPolicies=", tr.EgressPolicies) + return s +} + +type WorkloadEndpoint struct { + SwIfIndex []uint32 + Profiles []string + Tiers []*proto.TierInfo +} + +func (w *WorkloadEndpoint) String() string { + s := fmt.Sprintf("if=%d profiles=%s tiers=%s", w.SwIfIndex, w.Profiles, w.Tiers) + s += types.StrListToString(" Profiles=", w.Profiles) + s += types.StrableListToString(" Tiers=", w.Tiers) + return s +} + +func FromProtoEndpointID(ep *proto.WorkloadEndpointID) *WorkloadEndpointID { + return &WorkloadEndpointID{ + OrchestratorID: ep.OrchestratorId, + WorkloadID: ep.WorkloadId, + EndpointID: ep.EndpointId, + } +} + +func FromProtoWorkload(wep *proto.WorkloadEndpoint) *WorkloadEndpoint { + return &WorkloadEndpoint{ + SwIfIndex: []uint32{}, + Profiles: wep.ProfileIds, + Tiers: wep.Tiers, + } +} + +func (s *PoliciesHandler) getWepUserDefinedPolicies(w *WorkloadEndpoint, state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { + conf = types.NewInterfaceConfig() + for _, tier := range w.Tiers { + for _, policyID := range tier.IngressPolicies { + pol, ok := state.Policies[fromProtoPolicyID(policyID, network)] + if !ok { + return nil, fmt.Errorf("in policy %s not found for workload endpoint", policyID) + } + if pol.VppID == types.InvalidID { + return nil, fmt.Errorf("in policy %s not yet created in VPP", policyID) + } + conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, pol.VppID) + } + for _, policyID := range tier.EgressPolicies { + pol, ok := state.Policies[fromProtoPolicyID(policyID, network)] + if !ok { + return nil, fmt.Errorf("out policy %s not found for workload endpoint", policyID) + } + if pol.VppID == types.InvalidID { + return nil, fmt.Errorf("out policy %s not yet created in VPP", policyID) + } + conf.EgressPolicyIDs = append(conf.EgressPolicyIDs, pol.VppID) + } + } + for _, profileName := range w.Profiles { + prof, ok := state.Profiles[profileName] + if !ok { + return nil, fmt.Errorf("profile %s not found for workload endpoint", profileName) + } + if prof.VppID == types.InvalidID { + return nil, fmt.Errorf("profile %s not yet created in VPP", profileName) + } + conf.ProfileIDs = append(conf.ProfileIDs, prof.VppID) + } + return conf, nil +} + +// getWorkloadPolicies creates the interface configuration for a workload (pod) interface +// We have an implicit ingress policy that allows traffic coming from the host +// see createAllowFromHostPolicy() +// If there are no policies the default should be pass to profiles +// If there are policies the default should be deny (profiles are ignored) +func (s *PoliciesHandler) getWorkloadPolicies(w *WorkloadEndpoint, state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { + conf, err = s.getWepUserDefinedPolicies(w, state, network) + if err != nil { + return nil, errors.Wrap(err, "cannot create workload policies") + } + if len(conf.IngressPolicyIDs) > 0 { + conf.IngressPolicyIDs = append([]uint32{s.AllowFromHostPolicy.VppID}, conf.IngressPolicyIDs...) + conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_DENY + } else if len(conf.ProfileIDs) > 0 { + conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_PASS + } + if len(conf.EgressPolicyIDs) > 0 { + conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_DENY + } else if len(conf.ProfileIDs) > 0 { + conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_PASS + } + return conf, nil +} + +func (s *PoliciesHandler) CreateWorkloadEndpoint(w *WorkloadEndpoint, swIfIndexes []uint32, state *PolicyState, network string) (err error) { + conf, err := s.getWorkloadPolicies(w, state, network) + if err != nil { + return err + } + for _, swIfIndex := range swIfIndexes { + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + + w.SwIfIndex = append(w.SwIfIndex, swIfIndexes...) + return nil +} + +func (s *PoliciesHandler) updateWorkloadEndpoint(w *WorkloadEndpoint, new *WorkloadEndpoint, state *PolicyState, network string) (err error) { + conf, err := s.getWorkloadPolicies(new, state, network) + if err != nil { + return err + } + for _, swIfIndex := range w.SwIfIndex { + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + // Update local policy with new data + w.Profiles = new.Profiles + w.Tiers = new.Tiers + return nil +} + +func (s *PoliciesHandler) DeleteWorkloadEndpoint(w *WorkloadEndpoint) (err error) { + if len(w.SwIfIndex) == 0 { + return fmt.Errorf("deleting unconfigured wep") + } + // Nothing to do in VPP, policies are cleared when the interface is removed + w.SwIfIndex = []uint32{} + return nil +} + +func (s *PoliciesHandler) getAllWorkloadEndpointIdsFromUpdate(msg *proto.WorkloadEndpointUpdate) []*WorkloadEndpointID { + id := FromProtoEndpointID(msg.Id) + idsNetworks := []*WorkloadEndpointID{id} + netStatusesJSON, found := msg.Endpoint.Annotations["k8s.v1.cni.cncf.io/network-status"] + if !found { + s.log.Infof("no network status for pod, no multiple networks") + } else { + var netStatuses []nettypes.NetworkStatus + err := json.Unmarshal([]byte(netStatusesJSON), &netStatuses) + if err != nil { + s.log.Error(err) + } + for _, networkStatus := range netStatuses { + for netDefName, netDef := range s.cache.NetworkDefinitions { + if networkStatus.Name == netDef.NetAttachDefs { + id := &WorkloadEndpointID{OrchestratorID: id.OrchestratorID, WorkloadID: id.WorkloadID, EndpointID: id.EndpointID, Network: netDefName} + idsNetworks = append(idsNetworks, id) + } + } + } + } + return idsNetworks +} + +func (s *PoliciesHandler) OnWorkloadEndpointUpdate(msg *proto.WorkloadEndpointUpdate) (err error) { + state := s.GetState() + idsNetworks := s.getAllWorkloadEndpointIdsFromUpdate(msg) + for _, id := range idsNetworks { + wep := FromProtoWorkload(msg.Endpoint) + existing, found := state.WorkloadEndpoints[*id] + swIfIndexMap, swIfIndexFound := s.endpointsInterfaces[*id] + + if found { + if s.state.IsPending() || !swIfIndexFound { + state.WorkloadEndpoints[*id] = wep + s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=??", s.state.IsPending(), *id, existing, wep) + } else { + err := s.updateWorkloadEndpoint(existing, wep, state, id.Network) + if err != nil { + return errors.Wrap(err, "cannot update workload endpoint") + } + s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=%v", s.state.IsPending(), *id, existing, wep, swIfIndexMap) + } + } else { + state.WorkloadEndpoints[*id] = wep + if !s.state.IsPending() && swIfIndexFound { + swIfIndexList := []uint32{} + for _, idx := range swIfIndexMap { + swIfIndexList = append(swIfIndexList, idx) + } + err := s.CreateWorkloadEndpoint(wep, swIfIndexList, state, id.Network) + if err != nil { + return errors.Wrap(err, "cannot create workload endpoint") + } + s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=%v", s.state.IsPending(), *id, wep, swIfIndexMap) + } else { + s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=??", s.state.IsPending(), *id, wep) + } + } + } + return nil +} + +func (s *PoliciesHandler) OnWorkloadEndpointRemove(msg *proto.WorkloadEndpointRemove) (err error) { + state := s.GetState() + id := FromProtoEndpointID(msg.Id) + existing, ok := state.WorkloadEndpoints[*id] + if !ok { + s.log.Warnf("Received workload endpoint delete for %v that doesn't exists", id) + return nil + } + if !s.state.IsPending() && len(existing.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(existing) + if err != nil { + return errors.Wrap(err, "error deleting workload endpoint") + } + } + s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", s.state.IsPending(), *id, existing) + delete(state.WorkloadEndpoints, *id) + for existingID := range state.WorkloadEndpoints { + if existingID.OrchestratorID == id.OrchestratorID && existingID.WorkloadID == id.WorkloadID { + if !s.state.IsPending() && len(existing.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(existing) + if err != nil { + return errors.Wrap(err, "error deleting workload endpoint") + } + } + s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", s.state.IsPending(), existingID, existing) + delete(state.WorkloadEndpoints, existingID) + } + } + return nil +} diff --git a/calico-vpp-agent/felix/workload_endpoint.go b/calico-vpp-agent/felix/workload_endpoint.go deleted file mode 100644 index 5fea1d86d..000000000 --- a/calico-vpp-agent/felix/workload_endpoint.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (C) 2020 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package felix - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/projectcalico/calico/felix/proto" - - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -type WorkloadEndpointID struct { - OrchestratorID string - WorkloadID string - EndpointID string - Network string -} - -func (wi *WorkloadEndpointID) String() string { - return fmt.Sprintf("%s:%s:%s:%s", wi.OrchestratorID, wi.WorkloadID, wi.EndpointID, wi.Network) -} - -type WorkloadEndpoint struct { - SwIfIndex []uint32 - Profiles []string - Tiers []*proto.TierInfo - server *Server -} - -func (w *WorkloadEndpoint) String() string { - s := fmt.Sprintf("if=%d profiles=%s tiers=%s", w.SwIfIndex, w.Profiles, w.Tiers) - s += types.StrListToString(" Profiles=", w.Profiles) - s += types.StrableListToString(" Tiers=", w.Tiers) - return s -} - -func fromProtoEndpointID(ep *proto.WorkloadEndpointID) *WorkloadEndpointID { - return &WorkloadEndpointID{ - OrchestratorID: ep.OrchestratorId, - WorkloadID: ep.WorkloadId, - EndpointID: ep.EndpointId, - } -} - -func fromProtoWorkload(wep *proto.WorkloadEndpoint, server *Server) *WorkloadEndpoint { - return &WorkloadEndpoint{ - SwIfIndex: []uint32{}, - Profiles: wep.ProfileIds, - server: server, - Tiers: wep.Tiers, - } -} - -func (w *WorkloadEndpoint) getUserDefinedPolicies(state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { - conf = types.NewInterfaceConfig() - for _, tier := range w.Tiers { - for _, policyID := range tier.IngressPolicies { - pol, ok := state.Policies[fromProtoPolicyID(policyID, network)] - if !ok { - return nil, fmt.Errorf("in policy %s not found for workload endpoint", policyID) - } - if pol.VppID == types.InvalidID { - return nil, fmt.Errorf("in policy %s not yet created in VPP", policyID) - } - conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, pol.VppID) - } - for _, policyID := range tier.EgressPolicies { - pol, ok := state.Policies[fromProtoPolicyID(policyID, network)] - if !ok { - return nil, fmt.Errorf("out policy %s not found for workload endpoint", policyID) - } - if pol.VppID == types.InvalidID { - return nil, fmt.Errorf("out policy %s not yet created in VPP", policyID) - } - conf.EgressPolicyIDs = append(conf.EgressPolicyIDs, pol.VppID) - } - } - for _, profileName := range w.Profiles { - prof, ok := state.Profiles[profileName] - if !ok { - return nil, fmt.Errorf("profile %s not found for workload endpoint", profileName) - } - if prof.VppID == types.InvalidID { - return nil, fmt.Errorf("profile %s not yet created in VPP", profileName) - } - conf.ProfileIDs = append(conf.ProfileIDs, prof.VppID) - } - return conf, nil -} - -/* - This function creates the interface configuration for a workload (pod) interface - We have an implicit ingress policy that allows traffic coming from the host - see createAllowFromHostPolicy() - If there are no policies the default should be pass to profiles - If there are policies the default should be deny (profiles are ignored) -*/ -func (w *WorkloadEndpoint) getWorkloadPolicies(state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { - conf, err = w.getUserDefinedPolicies(state, network) - if err != nil { - return nil, errors.Wrap(err, "cannot create workload policies") - } - if len(conf.IngressPolicyIDs) > 0 { - conf.IngressPolicyIDs = append([]uint32{w.server.AllowFromHostPolicy.VppID}, conf.IngressPolicyIDs...) - conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_DENY - } else if len(conf.ProfileIDs) > 0 { - conf.PolicyDefaultIngress = npol.NPOL_DEFAULT_PASS - } - if len(conf.EgressPolicyIDs) > 0 { - conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_DENY - } else if len(conf.ProfileIDs) > 0 { - conf.PolicyDefaultEgress = npol.NPOL_DEFAULT_PASS - } - return conf, nil -} - -func (w *WorkloadEndpoint) Create(vpp *vpplink.VppLink, swIfIndexes []uint32, state *PolicyState, network string) (err error) { - conf, err := w.getWorkloadPolicies(state, network) - if err != nil { - return err - } - for _, swIfIndex := range swIfIndexes { - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - - w.SwIfIndex = append(w.SwIfIndex, swIfIndexes...) - return nil -} - -func (w *WorkloadEndpoint) Update(vpp *vpplink.VppLink, new *WorkloadEndpoint, state *PolicyState, network string) (err error) { - conf, err := new.getWorkloadPolicies(state, network) - if err != nil { - return err - } - for _, swIfIndex := range w.SwIfIndex { - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - // Update local policy with new data - w.Profiles = new.Profiles - w.Tiers = new.Tiers - return nil -} - -func (w *WorkloadEndpoint) Delete(vpp *vpplink.VppLink) (err error) { - if len(w.SwIfIndex) == 0 { - return fmt.Errorf("deleting unconfigured wep") - } - // Nothing to do in VPP, policies are cleared when the interface is removed - w.SwIfIndex = []uint32{} - return nil -} diff --git a/calico-vpp-agent/routing/bgp_watcher.go b/calico-vpp-agent/routing/bgp_watcher.go index d4f32e685..7a9b79947 100644 --- a/calico-vpp-agent/routing/bgp_watcher.go +++ b/calico-vpp-agent/routing/bgp_watcher.go @@ -512,7 +512,11 @@ func (s *Server) WatchBGPPath(t *tomb.Tomb) error { stopBGPMonitoring() s.log.Infof("Routing Server asked to stop") return nil - case evt := <-s.routingServerEventChan: + case msg := <-s.routingServerEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { case common.LocalPodAddressAdded: diff --git a/calico-vpp-agent/routing/routing_server.go b/calico-vpp-agent/routing/routing_server.go index 7defdf829..85e7d1845 100644 --- a/calico-vpp-agent/routing/routing_server.go +++ b/calico-vpp-agent/routing/routing_server.go @@ -56,7 +56,7 @@ type Server struct { bgpFilters map[string]*calicov3.BGPFilter bgpPeers map[string]*watchers.LocalBGPPeer - routingServerEventChan chan common.CalicoVppEvent + routingServerEventChan chan any nodeBGPSpec *common.LocalNodeSpec } @@ -83,7 +83,7 @@ func NewRoutingServer(vpp *vpplink.VppLink, bgpServer *bgpserver.BgpServer, log BGPServer: bgpServer, localAddressMap: make(map[string]localAddress), - routingServerEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + routingServerEventChan: make(chan any, common.ChanSize), bgpFilters: make(map[string]*calicov3.BGPFilter), bgpPeers: make(map[string]*watchers.LocalBGPPeer), } diff --git a/calico-vpp-agent/tests/mocks/pubsub_handler.go b/calico-vpp-agent/tests/mocks/pubsub_handler.go index 61ee919b7..4027c4e47 100644 --- a/calico-vpp-agent/tests/mocks/pubsub_handler.go +++ b/calico-vpp-agent/tests/mocks/pubsub_handler.go @@ -21,7 +21,7 @@ import ( // PubSubHandlerMock is mocking the handlers registering to common.ThePubSub type PubSubHandlerMock struct { - eventChan chan common.CalicoVppEvent + eventChan chan any ReceivedEvents []common.CalicoVppEvent expectedEventTypes []common.CalicoVppEventType t tomb.Tomb @@ -30,7 +30,7 @@ type PubSubHandlerMock struct { // NewPubSubHandlerMock creates new instance of PubSubHandlerMock func NewPubSubHandlerMock(expectedEventTypes ...common.CalicoVppEventType) *PubSubHandlerMock { handler := &PubSubHandlerMock{ - eventChan: make(chan common.CalicoVppEvent, common.ChanSize), + eventChan: make(chan any, common.ChanSize), ReceivedEvents: make([]common.CalicoVppEvent, 0, 10), expectedEventTypes: expectedEventTypes, } @@ -56,7 +56,11 @@ func (m *PubSubHandlerMock) receiveLoop() error { case <-m.t.Dying(): close(m.eventChan) return nil - case event := <-m.eventChan: + case msg := <-m.eventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + panic("expected CalicoVppEventType") + } m.ReceivedEvents = append(m.ReceivedEvents, event) } } diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 59ac9cbea..7207d08f7 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -237,7 +237,8 @@ func DpoNetworkNameFieldName() string { // InterfaceTagForLocalTunTunnel constructs the tag for the VPP side of the tap tunnel the same way as cni server func InterfaceTagForLocalTunTunnel(interfaceName, netns string) string { - return InterfaceTagForLocalTunnel(podinterface.NewTunTapPodInterfaceDriver(nil, nil, nil).Name, + return InterfaceTagForLocalTunnel( + podinterface.NewTunTapPodInterfaceDriver(nil, nil, nil).Name, interfaceName, netns) } diff --git a/calico-vpp-agent/watchers/bgp_configuration_watcher.go b/calico-vpp-agent/watchers/bgp_configuration_watcher.go index 87534725c..3aba61325 100644 --- a/calico-vpp-agent/watchers/bgp_configuration_watcher.go +++ b/calico-vpp-agent/watchers/bgp_configuration_watcher.go @@ -35,7 +35,7 @@ import ( type BGPConfigurationWatcher struct { log *logrus.Entry clientv3 calicov3cli.Interface - BGPConfigurationWatcherEventChan chan common.CalicoVppEvent + BGPConfigurationWatcherEventChan chan any BGPConf *calicov3.BGPConfigurationSpec } @@ -43,7 +43,7 @@ func NewBGPConfigurationWatcher(clientv3 calicov3cli.Interface, log *logrus.Entr w := BGPConfigurationWatcher{ log: log, clientv3: clientv3, - BGPConfigurationWatcherEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + BGPConfigurationWatcherEventChan: make(chan any, common.ChanSize), } reg := common.RegisterHandler(w.BGPConfigurationWatcherEventChan, "BGP Config watcher events") reg.ExpectEvents(common.BGPConfChanged) @@ -153,7 +153,11 @@ func (w *BGPConfigurationWatcher) WatchBGPConfiguration(t *tomb.Tomb) error { case <-t.Dying(): w.log.Warn("BGPConf watcher stopped") return nil - case evt := <-w.BGPConfigurationWatcherEventChan: + case msg := <-w.BGPConfigurationWatcherEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.BGPConfChanged: oldBGPConf := w.BGPConf diff --git a/calico-vpp-agent/felix/messages.go b/calico-vpp-agent/watchers/felix.go similarity index 51% rename from calico-vpp-agent/felix/messages.go rename to calico-vpp-agent/watchers/felix.go index e935a1640..327cd5464 100644 --- a/calico-vpp-agent/felix/messages.go +++ b/calico-vpp-agent/watchers/felix.go @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Cisco Systems Inc. +// Copyright (C) 2025 Cisco Systems Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,30 +13,114 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package watchers import ( "bytes" "encoding/binary" - "errors" + goerr "errors" "io" "net" + "os" + "github.com/pkg/errors" "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" pb "google.golang.org/protobuf/proto" + "gopkg.in/tomb.v2" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" ) -func (s *Server) MessageReader(conn net.Conn) <-chan interface{} { - ch := make(chan interface{}) +type FelixWatcher struct { + log *logrus.Entry + nextSeqNumber uint64 + felixServerEventChan chan any +} + +func NewFelixWatcher(felixServerEventChan chan any, log *logrus.Entry) *FelixWatcher { + return &FelixWatcher{ + log: log, + nextSeqNumber: 0, + felixServerEventChan: felixServerEventChan, + } +} + +// Serve runs the felix server +func (fw *FelixWatcher) WatchFelix(t *tomb.Tomb) error { + fw.log.Info("Starting felix Watcher") + // Cleanup potentially left over socket + err := os.RemoveAll(config.FelixDataplaneSocket) + if err != nil { + return errors.Wrapf(err, "Could not delete socket %s", config.FelixDataplaneSocket) + } + + listener, err := net.Listen("unix", config.FelixDataplaneSocket) + if err != nil { + return errors.Wrapf(err, "Could not bind to unix://%s", config.FelixDataplaneSocket) + } + defer func() { + listener.Close() + os.RemoveAll(config.FelixDataplaneSocket) + }() + for { + fw.felixServerEventChan <- &common.FelixSocketStateChanged{ + NewState: common.StateDisconnected, + } + // Accept only one connection + conn, err := listener.Accept() + if err != nil { + return errors.Wrap(err, "cannot accept felix client connection") + } + fw.log.Infof("Accepted connection from felix") + fw.felixServerEventChan <- &common.FelixSocketStateChanged{ + NewState: common.StateConnected, + } + + felixUpdates := fw.MessageReader(conn) + innerLoop: + for { + select { + case <-t.Dying(): + fw.log.Warn("Felix server exiting") + err = conn.Close() + if err != nil { + fw.log.WithError(err).Warn("Error closing unix connection to felix API proxy") + } + fw.log.Infof("Waiting for SyncFelix to stop...") + return nil + // <-felixUpdates & handleFelixUpdate does the bulk of the policy sync job. It starts by reconciling the current + // configured state in VPP (empty at first) with what is sent by felix, and once both are in + // sync, it keeps processing felix updates. It also sends endpoint updates to felix when the + // CNI component adds or deletes container interfaces. + case msg, ok := <-felixUpdates: + if !ok { + fw.log.Infof("Felix MessageReader closed") + break innerLoop + } + fw.felixServerEventChan <- msg + } + } + err = conn.Close() + if err != nil { + fw.log.WithError(err).Warn("Error closing unix connection to felix API proxy") + } + fw.log.Infof("SyncFelix exited, reconnecting to felix") + } +} + +func (fw *FelixWatcher) MessageReader(conn net.Conn) <-chan any { + ch := make(chan any) go func() { for { - msg, err := s.RecvMessage(conn) + msg, err := fw.RecvMessage(conn) if err != nil { - if errors.Is(err, io.EOF) && msg == nil { - s.log.Debug("EOF on felix-dataplane.sock") + if goerr.Is(err, io.EOF) && msg == nil { + fw.log.Debug("EOF on felix-dataplane.sock") } else { - s.log.Errorf("Error on felix-dataplane.sock err=%v msg=%v", err, msg) + fw.log.Errorf("Error on felix-dataplane.sock err=%v msg=%v", err, msg) } break } @@ -51,7 +135,7 @@ func (s *Server) MessageReader(conn net.Conn) <-chan interface{} { return ch } -func (s *Server) RecvMessage(conn net.Conn) (msg interface{}, err error) { +func (fw *FelixWatcher) RecvMessage(conn net.Conn) (msg interface{}, err error) { buf := make([]byte, 8) _, err = io.ReadFull(conn, buf) if err != nil { @@ -68,7 +152,7 @@ func (s *Server) RecvMessage(conn net.Conn) (msg interface{}, err error) { envelope := proto.ToDataplane{} err = pb.Unmarshal(data, &envelope) if err != nil { - s.log.Warnf("Skipping invalid message from felix-dataplane.sock: %v", err) + fw.log.Warnf("Skipping invalid message from felix-dataplane.sock: %v", err) return nil, nil } @@ -127,22 +211,22 @@ func (s *Server) RecvMessage(conn net.Conn) (msg interface{}, err error) { msg = payload.GlobalBgpConfigUpdate default: - s.log.WithField("payload", payload).Debug("Ignoring unknown message from felix") + fw.log.WithField("payload", payload).Debug("Ignoring unknown message from felix") } - s.log.WithField("msg", msg).Debug("Received message from dataplane.") + fw.log.WithField("msg", msg).Debug("Received message from dataplane.") return } -func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { - s.log.Debugf("Writing msg (%v) to felix: %#v", s.nextSeqNumber, msg) +func (fw *FelixWatcher) SendMessage(conn net.Conn, msg interface{}) (err error) { + fw.log.Debugf("Writing msg (%v) to felix: %#v", fw.nextSeqNumber, msg) // Wrap the payload message in an envelope so that protobuf takes care of deserialising // it as the correct type. envelope := &proto.FromDataplane{ - SequenceNumber: s.nextSeqNumber, + SequenceNumber: fw.nextSeqNumber, } - s.nextSeqNumber++ + fw.nextSeqNumber++ switch msg := msg.(type) { case *proto.ProcessStatusUpdate: envelope.Payload = &proto.FromDataplane_ProcessStatusUpdate{ProcessStatusUpdate: msg} @@ -157,13 +241,11 @@ func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { case *proto.WireguardStatusUpdate: envelope.Payload = &proto.FromDataplane_WireguardStatusUpdate{WireguardStatusUpdate: msg} default: - s.log.WithField("msg", msg).Panic("Unknown message type") + fw.log.WithField("msg", msg).Panic("Unknown message type") } data, err := pb.Marshal(envelope) - if err != nil { - s.log.WithError(err).WithField("msg", msg).Panic( - "Failed to marshal data") + fw.log.WithError(err).WithField("msg", msg).Panic("Failed to marshal data") } lengthBytes := make([]byte, 8) @@ -174,14 +256,43 @@ func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { for { _, err := messageBuf.WriteTo(conn) if err == io.ErrShortWrite { - s.log.Warn("Short write to felix; buffer full?") + fw.log.Warn("Short write to felix; buffer full?") continue } if err != nil { return err } - s.log.Debug("Wrote message to felix") + fw.log.Debug("Wrote message to felix") break } return nil } + +func InstallFelixPlugin() (err error) { + err = os.RemoveAll(config.FelixPluginDstPath) + if err != nil { + logrus.Warnf("Could not delete %s: %v", config.FelixPluginDstPath, err) + } + + in, err := os.Open(config.FelixPluginSrcPath) + if err != nil { + return errors.Wrap(err, "cannot open felix plugin to copy") + } + defer in.Close() + + out, err := os.OpenFile(config.FelixPluginDstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return errors.Wrap(err, "cannot open felix plugin to write") + } + defer func() { + cerr := out.Close() + if err == nil { + err = errors.Wrap(cerr, "cannot close felix plugin file") + } + }() + if _, err = io.Copy(out, in); err != nil { + return errors.Wrap(err, "cannot copy data") + } + err = out.Sync() + return errors.Wrapf(err, "could not sync felix plugin changes") +} diff --git a/calico-vpp-agent/watchers/net_watcher.go b/calico-vpp-agent/watchers/net_watcher.go index 0c58071ea..d185d20da 100644 --- a/calico-vpp-agent/watchers/net_watcher.go +++ b/calico-vpp-agent/watchers/net_watcher.go @@ -36,28 +36,12 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) -type VRF struct { - Tables [2]uint32 // one for ipv4, one for ipv6 -} - -type NetworkDefinition struct { - // VRF is the main table used for the corresponding physical network - VRF VRF - // PodVRF is the table used for the pods in the corresponding physical network - PodVRF VRF - Vni uint32 - PhysicalNetworkName string - Name string - Range string - NetAttachDefs string -} - type NetWatcher struct { log *logrus.Entry vpp *vpplink.VppLink client client.WithWatch stop chan struct{} - networkDefinitions map[string]*NetworkDefinition + networkDefinitions map[string]*common.NetworkDefinition nads map[string]string InSync chan interface{} nodeBGPSpec *common.LocalNodeSpec @@ -78,7 +62,7 @@ func NewNetWatcher(vpp *vpplink.VppLink, log *logrus.Entry) *NetWatcher { vpp: vpp, client: *kubernetesClient, stop: make(chan struct{}), - networkDefinitions: make(map[string]*NetworkDefinition), + networkDefinitions: make(map[string]*common.NetworkDefinition), nads: make(map[string]string), InSync: make(chan interface{}), } @@ -319,7 +303,7 @@ func (w *NetWatcher) OnNetDeleted(netName string) error { return nil } -func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRange string, phyNet string) (netDef *NetworkDefinition, err error) { +func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRange string, phyNet string) (netDef *common.NetworkDefinition, err error) { /* Create and Setup the per-network VRF */ if _, ok := w.networkDefinitions[networkName]; ok { return w.networkDefinitions[networkName], nil @@ -327,9 +311,9 @@ func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRan w.log.Infof("adding network %s", networkName) vrfID := common.VppManagerInfo.PhysicalNets[phyNet].VrfID podVrfID := common.VppManagerInfo.PhysicalNets[phyNet].PodVrfID - netDef = &NetworkDefinition{ - VRF: VRF{Tables: [2]uint32{vrfID, vrfID}}, - PodVRF: VRF{Tables: [2]uint32{podVrfID, podVrfID}}, + netDef = &common.NetworkDefinition{ + VRF: common.VRF{Tables: [2]uint32{vrfID, vrfID}}, + PodVRF: common.VRF{Tables: [2]uint32{podVrfID, podVrfID}}, Vni: uint32(networkVni), PhysicalNetworkName: phyNet, Name: networkName, @@ -338,7 +322,7 @@ func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRan return netDef, nil } -func (w *NetWatcher) DeleteNetwork(networkName string) (*NetworkDefinition, error) { +func (w *NetWatcher) DeleteNetwork(networkName string) (*common.NetworkDefinition, error) { if _, ok := w.networkDefinitions[networkName]; !ok { return nil, errors.Errorf("non-existent network deleted: %s", networkName) } diff --git a/calico-vpp-agent/watchers/peers_watcher.go b/calico-vpp-agent/watchers/peers_watcher.go index d72866cab..eda9ab8f6 100644 --- a/calico-vpp-agent/watchers/peers_watcher.go +++ b/calico-vpp-agent/watchers/peers_watcher.go @@ -68,7 +68,7 @@ type PeerWatcher struct { secretWatcher *secretWatcher nodeStatesByName map[string]common.LocalNodeSpec - peerWatcherEventChan chan common.CalicoVppEvent + peerWatcherEventChan chan any BGPConf *calicov3.BGPConfigurationSpec watcher watch.Interface currentWatchRevision string @@ -180,7 +180,11 @@ func (w *PeerWatcher) WatchBGPPeers(t *tomb.Tomb) error { default: w.log.Info("Peers updated, reevaluating peerings") } - case evt := <-w.peerWatcherEventChan: + case msg := <-w.peerWatcherEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { case common.PeerNodeStateChanged: @@ -541,7 +545,7 @@ func NewPeerWatcher(clientv3 calicov3cli.Interface, k8sclient *kubernetes.Client clientv3: clientv3, nodeStatesByName: make(map[string]common.LocalNodeSpec), log: log, - peerWatcherEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + peerWatcherEventChan: make(chan any, common.ChanSize), } w.secretWatcher, err = NewSecretWatcher(&w, k8sclient) if err != nil { diff --git a/calico-vpp-agent/watchers/uplink_route_watcher.go b/calico-vpp-agent/watchers/uplink_route_watcher.go index 8432cdacb..048642fd4 100644 --- a/calico-vpp-agent/watchers/uplink_route_watcher.go +++ b/calico-vpp-agent/watchers/uplink_route_watcher.go @@ -41,13 +41,13 @@ type RouteWatcher struct { addrNetlinkFailed chan struct{} addrUpdate chan struct{} closeLock sync.Mutex - eventChan chan common.CalicoVppEvent + eventChan chan any log *log.Entry } func NewRouteWatcher(log *log.Entry) *RouteWatcher { routeWatcher := &RouteWatcher{ - eventChan: make(chan common.CalicoVppEvent, common.ChanSize), + eventChan: make(chan any, common.ChanSize), log: log, } reg := common.RegisterHandler(routeWatcher.eventChan, "route watcher events") @@ -237,12 +237,16 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { } r.log.Warn("Route watcher stopped") return nil - case event := <-r.eventChan: + case msg := <-r.eventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch event.Type { case common.NetDeleted: - netDef, ok := event.Old.(*NetworkDefinition) + netDef, ok := event.Old.(*common.NetworkDefinition) if !ok { - r.log.Errorf("event.Old is not a (*NetworkDefinition) %v", event.Old) + r.log.Errorf("event.Old is not a (*common.NetworkDefinition) %v", event.Old) goto restart } key := netDef.Range @@ -259,9 +263,9 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { } } case common.NetAddedOrUpdated: - netDef, ok := event.New.(*NetworkDefinition) + netDef, ok := event.New.(*common.NetworkDefinition) if !ok { - r.log.Errorf("event.New is not a (*NetworkDefinition) %v", event.New) + r.log.Errorf("event.New is not a (*common.NetworkDefinition) %v", event.New) goto restart } key := netDef.Range diff --git a/config/config.go b/config/config.go index 247d8e95f..6dfecf86d 100644 --- a/config/config.go +++ b/config/config.go @@ -46,6 +46,9 @@ const ( CalicoVppPidFile = "/var/run/vpp/calico_vpp.pid" CalicoVppVersionFile = "/etc/calicovppversion" + FelixPluginSrcPath = "/bin/felix-api-proxy" + FelixPluginDstPath = "/var/lib/calico/felix-plugins/felix-api-proxy" + DefaultVXLANVni = 4096 DefaultVXLANPort = 4789 DefaultWireguardPort = 51820 From 5f5a43efcab8c7f0bf07143381f4a957fda6a548 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Wed, 22 Oct 2025 11:39:53 +0200 Subject: [PATCH 2/4] Split CNI into watcher/handler under felix This patch splits the CNI watcher and handlers in two pieces. The handling will be done in the main 'felix' goroutine, while the watching / grpc server will live under watchers/ and not store or access agent state. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 5 +- calico-vpp-agent/cni/cni_server.go | 510 ------------------ calico-vpp-agent/common/common.go | 1 - calico-vpp-agent/felix/cache/cache.go | 42 +- calico-vpp-agent/felix/cni/cni_handler.go | 301 +++++++++++ .../{ => felix}/cni/cni_node_test.go | 14 +- .../{ => felix}/cni/cni_pod_test.go | 113 ++-- calico-vpp-agent/felix/cni/model/events.go | 44 ++ .../{ => felix}/cni/model/pod_annotations.go | 0 .../{ => felix}/cni/model/pod_spec.go | 7 +- .../{ => felix}/cni/model/pod_status.go | 0 .../{ => felix}/cni/model/server_state.go | 0 .../{ => felix}/cni/netns_linux.go | 0 .../{ => felix}/cni/network_vpp.go | 32 +- .../{ => felix}/cni/network_vpp_hostports.go | 24 +- .../{ => felix}/cni/network_vpp_routes.go | 54 +- .../{ => felix}/cni/packet_helper.go | 0 .../{ => felix}/cni/podinterface/common.go | 29 +- .../{ => felix}/cni/podinterface/loopback.go | 16 +- .../{ => felix}/cni/podinterface/memif.go | 17 +- .../{ => felix}/cni/podinterface/tuntap.go | 33 +- .../{ => felix}/cni/podinterface/vcl.go | 17 +- calico-vpp-agent/felix/felix_server.go | 111 +++- calico-vpp-agent/felix/felixconfig.go | 1 + calico-vpp-agent/felix/ipam.go | 3 + calico-vpp-agent/prometheus/prometheus.go | 2 +- .../prometheus/prometheus_test.go | 2 +- calico-vpp-agent/routing/bgp_watcher.go | 2 +- calico-vpp-agent/services/service_handler.go | 5 +- calico-vpp-agent/services/service_server.go | 2 +- calico-vpp-agent/tests/mocks/ipam.go | 4 - calico-vpp-agent/testutils/testutils.go | 9 +- calico-vpp-agent/watchers/cni_grpc.go | 111 ++++ calico-vpp-agent/watchers/felix.go | 2 +- vpp-manager/vpp_runner.go | 2 +- 35 files changed, 784 insertions(+), 731 deletions(-) delete mode 100644 calico-vpp-agent/cni/cni_server.go create mode 100644 calico-vpp-agent/felix/cni/cni_handler.go rename calico-vpp-agent/{ => felix}/cni/cni_node_test.go (98%) rename calico-vpp-agent/{ => felix}/cni/cni_pod_test.go (90%) create mode 100644 calico-vpp-agent/felix/cni/model/events.go rename calico-vpp-agent/{ => felix}/cni/model/pod_annotations.go (100%) rename calico-vpp-agent/{ => felix}/cni/model/pod_spec.go (97%) rename calico-vpp-agent/{ => felix}/cni/model/pod_status.go (100%) rename calico-vpp-agent/{ => felix}/cni/model/server_state.go (100%) rename calico-vpp-agent/{ => felix}/cni/netns_linux.go (100%) rename calico-vpp-agent/{ => felix}/cni/network_vpp.go (90%) rename calico-vpp-agent/{ => felix}/cni/network_vpp_hostports.go (78%) rename calico-vpp-agent/{ => felix}/cni/network_vpp_routes.go (85%) rename calico-vpp-agent/{ => felix}/cni/packet_helper.go (100%) rename calico-vpp-agent/{ => felix}/cni/podinterface/common.go (88%) rename calico-vpp-agent/{ => felix}/cni/podinterface/loopback.go (85%) rename calico-vpp-agent/{ => felix}/cni/podinterface/memif.go (95%) rename calico-vpp-agent/{ => felix}/cni/podinterface/tuntap.go (92%) rename calico-vpp-agent/{ => felix}/cni/podinterface/vcl.go (85%) create mode 100644 calico-vpp-agent/watchers/cni_grpc.go diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index f8916daf2..8aa3d10b5 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -34,7 +34,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" @@ -154,12 +153,12 @@ func main() { localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) felixServer := felix.NewFelixServer(vpp, clientv3, log.WithFields(logrus.Fields{"component": "policy"})) felixWatcher := watchers.NewFelixWatcher(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "felix watcher"})) + cniServer := watchers.NewCNIServer(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "cni"})) err = watchers.InstallFelixPlugin() if err != nil { log.Fatalf("could not install felix plugin: %s", err) } connectivityServer := connectivity.NewConnectivityServer(vpp, felixServer, clientv3, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) - cniServer := cni.NewCNIServer(vpp, felixServer, log.WithFields(logrus.Fields{"component": "cni"})) /* Pubsub should now be registered */ @@ -222,7 +221,6 @@ func main() { serviceServer.SetOurBGPSpec(ourBGPSpec) localSIDWatcher.SetOurBGPSpec(ourBGPSpec) netWatcher.SetOurBGPSpec(ourBGPSpec) - cniServer.SetOurBGPSpec(ourBGPSpec) if *config.GetCalicoVppFeatureGates().MultinetEnabled { Go(netWatcher.WatchNetworks) @@ -236,7 +234,6 @@ func main() { } } - cniServer.SetFelixConfig(felixConfig) connectivityServer.SetFelixConfig(felixConfig) Go(routeWatcher.WatchRoutes) diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go deleted file mode 100644 index d0b94b063..000000000 --- a/calico-vpp-agent/cni/cni_server.go +++ /dev/null @@ -1,510 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cni - -import ( - "context" - gerrors "errors" - "fmt" - "net" - "os" - "sync" - "syscall" - - "github.com/pkg/errors" - calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" - felixConfig "github.com/projectcalico/calico/felix/config" - "github.com/projectcalico/calico/felix/proto" - "github.com/sirupsen/logrus" - "google.golang.org/grpc" - "gopkg.in/tomb.v2" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -type Server struct { - cniproto.UnimplementedCniDataplaneServer - log *logrus.Entry - vpp *vpplink.VppLink - - felixServerIpam common.FelixServerIpam - - grpcServer *grpc.Server - - podInterfaceMap map[string]model.LocalPodSpec - lock sync.Mutex /* protects Add/DelVppInterace/RescanState */ - cniEventChan chan any - - memifDriver *podinterface.MemifPodInterfaceDriver - tuntapDriver *podinterface.TunTapPodInterfaceDriver - vclDriver *podinterface.VclPodInterfaceDriver - loopbackDriver *podinterface.LoopbackPodInterfaceDriver - - availableBuffers uint64 - - RedirectToHostClassifyTableIndex uint32 - - networkDefinitions sync.Map - cniMultinetEventChan chan any - nodeBGPSpec *common.LocalNodeSpec -} - -func swIfIdxToIfName(idx uint32) string { - return fmt.Sprintf("vpp-tun-%d", idx) -} - -func (s *Server) SetFelixConfig(felixConfig *felixConfig.Config) { - s.tuntapDriver.SetFelixConfig(felixConfig) -} - -func (s *Server) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { - s.nodeBGPSpec = nodeBGPSpec -} - -func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { - /* We don't support request.GetDesiredHostInterfaceName() */ - podSpec, err := model.NewLocalPodSpecFromAdd(request, s.nodeBGPSpec) - if err != nil { - s.log.Errorf("Error parsing interface add request %v %v", request, err) - return &cniproto.AddReply{ - Successful: false, - ErrorMessage: err.Error(), - }, nil - } - if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) - if !ok { - return nil, fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) - } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("Value is not of type *common.NetworkDefinition") - } - _, route, err := net.ParseCIDR(networkDefinition.Range) - if err == nil { - podSpec.Routes = append(podSpec.Routes, *route) - } - } - } - if podSpec.NetnsName == "" { - s.log.Debugf("no netns passed, skipping") - return &cniproto.AddReply{ - Successful: true, - }, nil - } - - s.lock.Lock() - defer s.lock.Unlock() - - s.log.Infof("pod(add) spec=%s network=%s", podSpec.String(), request.DataplaneOptions["network_name"]) - - existingSpec, ok := s.podInterfaceMap[podSpec.Key()] - if ok { - s.log.Info("pod(add) found existing spec") - podSpec = &existingSpec - } - - swIfIndex, err := s.AddVppInterface(podSpec, true /* doHostSideConf */) - if err != nil { - s.log.Errorf("Interface add failed %s : %v", podSpec.String(), err) - return &cniproto.AddReply{ - Successful: false, - ErrorMessage: err.Error(), - }, nil - } - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { - err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex) - if err != nil { - return nil, err - } - } - - s.podInterfaceMap[podSpec.Key()] = *podSpec - err = model.PersistCniServerState( - model.NewCniServerState(s.podInterfaceMap), - config.CniServerStateFilename, - ) - if err != nil { - s.log.Errorf("CNI state persist errored %v", err) - } - s.log.Infof("pod(add) Done spec=%s", podSpec.String()) - // XXX: container MAC doesn't make sense with tun, we just pass back a constant one. - // How does calico / k8s use it? - // TODO: pass real mac for tap ? - return &cniproto.AddReply{ - Successful: true, - HostInterfaceName: swIfIdxToIfName(swIfIndex), - ContainerMac: "02:00:00:00:00:00", - }, nil -} - -func (s *Server) fetchNDataThreads() { - nDataThreads := common.FetchNDataThreads(s.vpp, s.log) - s.memifDriver.NDataThreads = nDataThreads - s.tuntapDriver.NDataThreads = nDataThreads -} - -func (s *Server) FetchBufferConfig() { - availableBuffers, _, _, err := s.vpp.GetBufferStats() - if err != nil { - s.log.WithError(err).Errorf("could not get available buffers") - } - s.availableBuffers = uint64(availableBuffers) -} - -func (s *Server) rescanState() { - s.FetchBufferConfig() - s.fetchNDataThreads() - - if *config.GetCalicoVppFeatureGates().VCLEnabled { - err := s.vclDriver.Init() - if err != nil { - /* it might already be enabled, do not return */ - s.log.Errorf("Error initializing VCL %v", err) - } - } - - cniServerState, err := model.LoadCniServerState(config.CniServerStateFilename) - if err != nil { - s.log.Errorf("Error getting pods from file %s, removing cache", err) - err := os.Remove(config.CniServerStateFilename) - if err != nil { - s.log.Errorf("Could not remove %s, %s", config.CniServerStateFilename, err) - } - // if the cniServerState file is corrupted, we remove it and give up. - return - } - - s.log.Infof("RescanState: re-creating all interfaces") - s.lock.Lock() - defer s.lock.Unlock() - for _, podSpec := range cniServerState.PodSpecs { - // we copy podSpec as a pointer to it will be sent over the event chan - podSpecCopy := podSpec.Copy() - _, err := s.AddVppInterface(&podSpecCopy, false /* doHostSideConf */) - switch err.(type) { - case PodNSNotFoundErr: - s.log.Infof("Interface restore but netns missing %s", podSpecCopy.String()) - case nil: - s.log.Infof("pod(re-add) podSpec=%s", podSpecCopy.String()) - s.podInterfaceMap[podSpec.Key()] = podSpecCopy - default: - s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err) - } - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" { - err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex) - if err != nil { - s.log.Error(err) - } - } - } - err = model.PersistCniServerState( - model.NewCniServerState(s.podInterfaceMap), - config.CniServerStateFilename, - ) - if err != nil { - s.log.Errorf("CNI state persist errored %v", err) - } -} - -func (s *Server) DelRedirectToHostOnInterface(swIfIndex uint32) error { - err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, false /*isAdd*/) - if err != nil { - return errors.Wrapf(err, "Error deleting classify input table from interface") - } else { - s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex) - return nil - } -} - -func (s *Server) AddRedirectToHostToInterface(swIfIndex uint32) error { - s.log.Infof("Setting classify input acl table %d on interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex) - err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, true) - if err != nil { - s.log.Warnf("Error setting classify input table: %s, retrying...", err) - return errors.Errorf("could not set input acl table %d for interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex) - } else { - s.log.Infof("set input acl table %d for interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex) - return nil - } -} - -func (s *Server) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) { - podSpecKey := model.LocalPodSpecKey(request.GetNetns(), request.GetInterfaceName()) - // Only try to delete the device if a namespace was passed in. - if request.GetNetns() == "" { - s.log.Debugf("no netns passed, skipping") - return &cniproto.DelReply{ - Successful: true, - }, nil - } - s.lock.Lock() - defer s.lock.Unlock() - - s.log.Infof("pod(del) key=%s", podSpecKey) - initialSpec, ok := s.podInterfaceMap[podSpecKey] - if !ok { - s.log.Warnf("Unknown pod to delete key=%s", podSpecKey) - } else { - s.log.Infof("pod(del) spec=%s", initialSpec.String()) - s.DelVppInterface(&initialSpec) - s.log.Infof("pod(del) Done! spec=%s", initialSpec.String()) - } - - delete(s.podInterfaceMap, podSpecKey) - err := model.PersistCniServerState( - model.NewCniServerState(s.podInterfaceMap), - config.CniServerStateFilename, - ) - if err != nil { - s.log.Errorf("CNI state persist errored %v", err) - } - - return &cniproto.DelReply{ - Successful: true, - }, nil -} - -// Serve runs the grpc server for the Calico CNI backend API -func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, log *logrus.Entry) *Server { - server := &Server{ - vpp: vpp, - log: log, - - felixServerIpam: felixServerIpam, - cniEventChan: make(chan any, common.ChanSize), - - grpcServer: grpc.NewServer(), - podInterfaceMap: make(map[string]model.LocalPodSpec), - tuntapDriver: podinterface.NewTunTapPodInterfaceDriver(vpp, log, felixServerIpam), - memifDriver: podinterface.NewMemifPodInterfaceDriver(vpp, log, felixServerIpam), - vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, log, felixServerIpam), - loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, log, felixServerIpam), - - cniMultinetEventChan: make(chan any, common.ChanSize), - } - reg := common.RegisterHandler(server.cniEventChan, "CNI server events") - reg.ExpectEvents( - common.FelixConfChanged, - common.IpamConfChanged, - ) - regM := common.RegisterHandler(server.cniMultinetEventChan, "CNI server Multinet events") - regM.ExpectEvents( - common.NetAddedOrUpdated, - common.NetDeleted, - common.NetsSynced, - ) - return server -} -func (s *Server) cniServerEventLoop(t *tomb.Tomb) error { -forloop: - for { - select { - case <-t.Dying(): - break forloop - case msg := <-s.cniEventChan: - evt, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch evt.Type { - case common.FelixConfChanged: - if new, _ := evt.New.(*felixConfig.Config); new != nil { - s.lock.Lock() - s.tuntapDriver.FelixConfigChanged(new, 0 /* ipipEncapRefCountDelta */, 0 /* vxlanEncapRefCountDelta */, s.podInterfaceMap) - s.lock.Unlock() - } - case common.IpamConfChanged: - old, _ := evt.Old.(*proto.IPAMPool) - new, _ := evt.New.(*proto.IPAMPool) - ipipEncapRefCountDelta := 0 - vxlanEncapRefCountDelta := 0 - if old != nil && calicov3.VXLANMode(old.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(old.VxlanMode) != "" { - vxlanEncapRefCountDelta-- - } - if old != nil && calicov3.IPIPMode(old.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(old.IpipMode) != "" { - ipipEncapRefCountDelta-- - } - if new != nil && calicov3.VXLANMode(new.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(new.VxlanMode) != "" { - vxlanEncapRefCountDelta++ - } - if new != nil && calicov3.IPIPMode(new.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(new.IpipMode) != "" { - ipipEncapRefCountDelta++ - } - - for _, podSpec := range s.podInterfaceMap { - for _, swIfIndex := range []uint32{podSpec.LoopbackSwIfIndex, podSpec.TunTapSwIfIndex, podSpec.MemifSwIfIndex} { - if swIfIndex != vpplink.InvalidID { - s.log.Infof("Enable/Disable interface[%d] SNAT", swIfIndex) - for _, ipFamily := range vpplink.IPFamilies { - err := s.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIP6, podSpec.NeedsSnat(s.felixServerIpam, ipFamily.IsIP6)) - if err != nil { - return errors.Wrapf(err, "Error enabling/disabling %s snat", ipFamily.Str) - } - } - } - } - } - s.lock.Lock() - s.tuntapDriver.FelixConfigChanged(nil /* felixConfig */, ipipEncapRefCountDelta, vxlanEncapRefCountDelta, s.podInterfaceMap) - s.lock.Unlock() - } - } - } - return nil -} - -func (s *Server) getMainInterface() *config.UplinkStatus { - for _, i := range common.VppManagerInfo.UplinkStatuses { - if i.IsMain { - return &i - } - } - return nil -} - -func (s *Server) createRedirectToHostRules() (uint32, error) { - var maxNumEntries uint32 - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 { - maxNumEntries = uint32(2 * len(config.GetCalicoVppInitialConfig().RedirectToHostRules)) - } else { - maxNumEntries = 1 - } - index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{ - Mask: types.DstThreeTupleMask, - NextTableIndex: types.InvalidID, - MaxNumEntries: maxNumEntries, - MissNextIndex: ^uint32(0), - }) - if err != nil { - return types.InvalidID, err - } - mainInterface := s.getMainInterface() - if mainInterface == nil { - return types.InvalidID, fmt.Errorf("no main interface found") - } - for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules { - mainInterfaceAddress := mainInterface.GetAddress(vpplink.IPFamilyFromIP(rule.IP)) - if mainInterfaceAddress == nil { - return types.InvalidID, fmt.Errorf("error installing rule %v no address found on uplink", rule) - } - err = s.vpp.AddSessionRedirect(&types.SessionRedirect{ - FiveTuple: types.NewDst3Tuple(rule.Proto, rule.IP, rule.Port), - TableIndex: index, - }, &types.RoutePath{ - Gw: mainInterfaceAddress.IP, - SwIfIndex: mainInterface.TapSwIfIndex, - }) - if err != nil { - return types.InvalidID, err - } - } - - return index, nil -} - -func (s *Server) ServeCNI(t *tomb.Tomb) error { - err := syscall.Unlink(config.CNIServerSocket) - if err != nil && !gerrors.Is(err, os.ErrNotExist) { - s.log.Warnf("unable to unlink cni server socket: %+v", err) - } - - socketListener, err := net.Listen("unix", config.CNIServerSocket) - if err != nil { - return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket) - } - - s.RedirectToHostClassifyTableIndex, err = s.createRedirectToHostRules() - if err != nil { - return err - } - cniproto.RegisterCniDataplaneServer(s.grpcServer, s) - - if *config.GetCalicoVppFeatureGates().MultinetEnabled { - netsSynced := make(chan bool) - go func() { - for { - select { - case <-t.Dying(): - s.log.Warn("Cni server asked to exit") - return - case msg := <-s.cniMultinetEventChan: - event, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch event.Type { - case common.NetsSynced: - netsSynced <- true - case common.NetAddedOrUpdated: - netDef, ok := event.New.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("event.New is not a *common.NetworkDefinition %v", event.New) - continue - } - s.networkDefinitions.Store(netDef.Name, netDef) - case common.NetDeleted: - netDef, ok := event.Old.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("event.Old is not a *common.NetworkDefinition %v", event.Old) - continue - } - s.networkDefinitions.Delete(netDef.Name) - } - } - } - }() - <-netsSynced - s.log.Infof("Networks synced") - } - s.rescanState() - - s.log.Infof("Serve() CNI") - - go func() { - err := s.grpcServer.Serve(socketListener) - if err != nil { - s.log.Fatalf("GrpcServer Server returned %s", err) - } - }() - - err = s.cniServerEventLoop(t) - if err != nil { - return err - } - - s.log.Infof("CNI Server returned") - - s.grpcServer.GracefulStop() - err = syscall.Unlink(config.CNIServerSocket) - if err != nil { - return err - } - - return nil -} - -// ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. -// The usage is mainly for testing purposes. -func (s *Server) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { - s.networkDefinitions.Store(networkDefinition.Name, networkDefinition) -} diff --git a/calico-vpp-agent/common/common.go b/calico-vpp-agent/common/common.go index 39c7e5146..861296835 100644 --- a/calico-vpp-agent/common/common.go +++ b/calico-vpp-agent/common/common.go @@ -53,7 +53,6 @@ const ( ) type FelixServerIpam interface { - IPNetNeedsSNAT(prefix *net.IPNet) bool GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool } diff --git a/calico-vpp-agent/felix/cache/cache.go b/calico-vpp-agent/felix/cache/cache.go index 80fa4ad1d..d39f41e8b 100644 --- a/calico-vpp-agent/felix/cache/cache.go +++ b/calico-vpp-agent/felix/cache/cache.go @@ -25,29 +25,43 @@ import ( calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type Cache struct { log *logrus.Entry - FelixConfig *felixConfig.Config - NodeByAddr map[string]common.LocalNodeSpec - Networks map[uint32]*common.NetworkDefinition - NetworkDefinitions map[string]*common.NetworkDefinition - IPPoolMap map[string]*proto.IPAMPool - NodeStatesByName map[string]*common.LocalNodeSpec - BGPConf *calicov3.BGPConfigurationSpec + FelixConfig *felixConfig.Config + NodeByAddr map[string]common.LocalNodeSpec + Networks map[uint32]*common.NetworkDefinition + NetworkDefinitions map[string]*common.NetworkDefinition + IPPoolMap map[string]*proto.IPAMPool + NodeStatesByName map[string]*common.LocalNodeSpec + BGPConf *calicov3.BGPConfigurationSpec + RedirectToHostClassifyTableIndex uint32 + VppAvailableBuffers uint64 + NumDataThreads int } func NewCache(log *logrus.Entry) *Cache { return &Cache{ - log: log, - NodeByAddr: make(map[string]common.LocalNodeSpec), - FelixConfig: felixConfig.New(), - Networks: make(map[uint32]*common.NetworkDefinition), - NetworkDefinitions: make(map[string]*common.NetworkDefinition), - IPPoolMap: make(map[string]*proto.IPAMPool), - NodeStatesByName: make(map[string]*common.LocalNodeSpec), + log: log, + NodeByAddr: make(map[string]common.LocalNodeSpec), + FelixConfig: felixConfig.New(), + Networks: make(map[uint32]*common.NetworkDefinition), + NetworkDefinitions: make(map[string]*common.NetworkDefinition), + IPPoolMap: make(map[string]*proto.IPAMPool), + RedirectToHostClassifyTableIndex: types.InvalidID, + NodeStatesByName: make(map[string]*common.LocalNodeSpec), + } +} + +func (cache *Cache) IPNetNeedsSNAT(prefix *net.IPNet) bool { + pool := cache.GetPrefixIPPool(prefix) + if pool == nil { + return false + } else { + return pool.Masquerade } } diff --git a/calico-vpp-agent/felix/cni/cni_handler.go b/calico-vpp-agent/felix/cni/cni_handler.go new file mode 100644 index 000000000..160ffd5a5 --- /dev/null +++ b/calico-vpp-agent/felix/cni/cni_handler.go @@ -0,0 +1,301 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cni + +import ( + "fmt" + "net" + "os" + + "github.com/pkg/errors" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type CNIHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + podInterfaceMap map[string]model.LocalPodSpec + + memifDriver *podinterface.MemifPodInterfaceDriver + tuntapDriver *podinterface.TunTapPodInterfaceDriver + vclDriver *podinterface.VclPodInterfaceDriver + loopbackDriver *podinterface.LoopbackPodInterfaceDriver +} + +// Serve runs the grpc server for the Calico CNI backend API +func NewCNIHandler(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *CNIHandler { + return &CNIHandler{ + vpp: vpp, + log: log, + cache: cache, + podInterfaceMap: make(map[string]model.LocalPodSpec), + tuntapDriver: podinterface.NewTunTapPodInterfaceDriver(vpp, cache, log), + memifDriver: podinterface.NewMemifPodInterfaceDriver(vpp, cache, log), + vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, cache, log), + loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, cache, log), + } +} + +func swIfIdxToIfName(idx uint32) string { + return fmt.Sprintf("vpp-tun-%d", idx) +} + +func (s *CNIHandler) OnNetAddedOrUpdated(old, new *common.NetworkDefinition) { + s.rescanState() +} + +func (s *CNIHandler) OnNetDeleted(old *common.NetworkDefinition) { + s.rescanState() +} + +func (s *CNIHandler) OnPodAdd(evt *model.CniPodAddEvent) error { + podSpec := evt.PodSpec + + if podSpec.NetworkName != "" { + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] + if !ok { + err := fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } else { + _, route, err := net.ParseCIDR(networkDefinition.Range) + if err == nil { + podSpec.Routes = append(podSpec.Routes, *route) + } + } + } + if podSpec.NetnsName == "" { + s.log.Debugf("no netns passed, skipping") + evt.Done <- &cniproto.AddReply{ + Successful: true, + } + return nil + } + + s.log.Infof("pod(add) spec=%s network=%s", podSpec.String(), podSpec.NetworkName) + + existingSpec, ok := s.podInterfaceMap[podSpec.Key()] + if ok { + s.log.Info("pod(add) found existing spec") + podSpec = &existingSpec + } + + swIfIndex, err := s.AddVppInterface(podSpec, true /* doHostSideConf */) + if err != nil { + s.log.Errorf("Interface add failed %s : %v", podSpec.String(), err) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { + err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex) + if err != nil { + s.log.Errorf("AddRedirectToHostToInterface failed %s : %v", podSpec.String(), err) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } + } + + s.podInterfaceMap[podSpec.Key()] = *podSpec + err = model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) + if err != nil { + s.log.Errorf("CNI state persist errored %v", err) + } + s.log.Infof("pod(add) Done spec=%s", podSpec.String()) + // XXX: container MAC doesn't make sense with tun, we just pass back a constant one. + // How does calico / k8s use it? + // TODO: pass real mac for tap ? + evt.Done <- &cniproto.AddReply{ + Successful: true, + HostInterfaceName: swIfIdxToIfName(swIfIndex), + ContainerMac: "02:00:00:00:00:00", + } + return nil +} + +func (s *CNIHandler) rescanState() { + if *config.GetCalicoVppFeatureGates().VCLEnabled { + err := s.vclDriver.Init() + if err != nil { + /* it might already be enabled, do not return */ + s.log.Errorf("Error initializing VCL %v", err) + } + } + + cniServerState, err := model.LoadCniServerState(config.CniServerStateFilename) + if err != nil { + s.log.Errorf("Error getting pods from file %s, removing cache", err) + err := os.Remove(config.CniServerStateFilename) + if err != nil { + s.log.Errorf("Could not remove %s, %s", config.CniServerStateFilename, err) + } + // if the cniServerState file is corrupted, we remove it and give up. + return + } + + s.log.Infof("RescanState: re-creating all interfaces") + for _, podSpec := range cniServerState.PodSpecs { + // we copy podSpec as a pointer to it will be sent over the event chan + podSpecCopy := podSpec.Copy() + _, err := s.AddVppInterface(&podSpecCopy, false /* doHostSideConf */) + switch err.(type) { + case PodNSNotFoundErr: + s.log.Infof("Interface restore but netns missing %s", podSpecCopy.String()) + case nil: + s.log.Infof("pod(re-add) podSpec=%s", podSpecCopy.String()) + s.podInterfaceMap[podSpec.Key()] = podSpecCopy + default: + s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err) + } + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" { + err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex) + if err != nil { + s.log.Error(err) + } + } + } + err = model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) + if err != nil { + s.log.Errorf("CNI state persist errored %v", err) + } +} + +func (s *CNIHandler) DelRedirectToHostOnInterface(swIfIndex uint32) error { + err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.cache.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, false /*isAdd*/) + if err != nil { + return errors.Wrapf(err, "Error deleting classify input table from interface") + } else { + s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + return nil + } +} + +func (s *CNIHandler) AddRedirectToHostToInterface(swIfIndex uint32) error { + s.log.Infof("Setting classify input acl table %d on interface %d", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.cache.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, true) + if err != nil { + s.log.Warnf("Error setting classify input table: %s, retrying...", err) + return errors.Errorf("could not set input acl table %d for interface %d", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + } else { + s.log.Infof("set input acl table %d for interface %d successfully", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + return nil + } +} + +func (s *CNIHandler) OnPodDelete(evt *model.CniPodDelEvent) { + s.log.Infof("pod(del) key=%s", evt.PodSpecKey) + initialSpec, ok := s.podInterfaceMap[evt.PodSpecKey] + if !ok { + s.log.Warnf("Unknown pod to delete key=%s", evt.PodSpecKey) + } else { + s.log.Infof("pod(del) spec=%s", initialSpec.String()) + s.DelVppInterface(&initialSpec) + s.log.Infof("pod(del) Done! spec=%s", initialSpec.String()) + } + + delete(s.podInterfaceMap, evt.PodSpecKey) + err := model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) + if err != nil { + s.log.Errorf("CNI state persist errored %v", err) + } + + evt.Done <- &cniproto.DelReply{Successful: true} +} + +func (s *CNIHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if new != nil { + s.tuntapDriver.FelixConfigChanged( + new, + 0, /* ipipEncapRefCountDelta */ + 0, /* vxlanEncapRefCountDelta */ + s.podInterfaceMap, + ) + } +} + +func (s *CNIHandler) OnIpamConfChanged(old, new *proto.IPAMPool) { + ipipEncapRefCountDelta := 0 + vxlanEncapRefCountDelta := 0 + if old != nil && calicov3.VXLANMode(old.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(old.VxlanMode) != "" { + vxlanEncapRefCountDelta-- + } + if old != nil && calicov3.IPIPMode(old.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(old.IpipMode) != "" { + ipipEncapRefCountDelta-- + } + if new != nil && calicov3.VXLANMode(new.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(new.VxlanMode) != "" { + vxlanEncapRefCountDelta++ + } + if new != nil && calicov3.IPIPMode(new.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(new.IpipMode) != "" { + ipipEncapRefCountDelta++ + } + + for _, podSpec := range s.podInterfaceMap { + for _, swIfIndex := range []uint32{podSpec.LoopbackSwIfIndex, podSpec.TunTapSwIfIndex, podSpec.MemifSwIfIndex} { + if swIfIndex != vpplink.InvalidID { + s.log.Infof("Enable/Disable interface[%d] SNAT", swIfIndex) + for _, ipFamily := range vpplink.IPFamilies { + err := s.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIP6, podSpec.NeedsSnat(s.cache, ipFamily.IsIP6)) + if err != nil { + s.log.WithError(err).Errorf("Error enabling/disabling %s snat", ipFamily.Str) + } + } + } + } + } + s.tuntapDriver.FelixConfigChanged(nil /* felixConfig */, ipipEncapRefCountDelta, vxlanEncapRefCountDelta, s.podInterfaceMap) +} + +func (s *CNIHandler) CNIHandlerInit() error { + s.rescanState() + return nil +} + +// ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. +// The usage is mainly for testing purposes. +func (s *CNIHandler) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { + s.cache.NetworkDefinitions[networkDefinition.Name] = networkDefinition +} diff --git a/calico-vpp-agent/cni/cni_node_test.go b/calico-vpp-agent/felix/cni/cni_node_test.go similarity index 98% rename from calico-vpp-agent/cni/cni_node_test.go rename to calico-vpp-agent/felix/cni/cni_node_test.go index 116dcf351..993e88118 100644 --- a/calico-vpp-agent/cni/cni_node_test.go +++ b/calico-vpp-agent/felix/cni/cni_node_test.go @@ -37,6 +37,7 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" @@ -106,6 +107,7 @@ var _ = Describe("Node-related functionality of CNI", func() { var ( log *logrus.Logger vpp *vpplink.VppLink + felixServer *felix.Server connectivityServer *connectivity.ConnectivityServer client *calico.CalicoClientStub ipamStub *mocks.IpamCacheStub @@ -133,11 +135,21 @@ var _ = Describe("Node-related functionality of CNI", func() { connectivityServer = connectivity.NewConnectivityServer(vpp, ipamStub, client, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{}) + felixServer = felix.NewFelixServer( + vpp, + client, + log.WithFields(logrus.Fields{"subcomponent": "connectivity"}), + ) if felixConfig == nil { felixConfig = &config.Config{} } connectivityServer.SetFelixConfig(felixConfig) - common.VppManagerInfo = &agentConf.VppManagerInfo{UplinkStatuses: map[string]agentConf.UplinkStatus{"eth0": {IsMain: true, SwIfIndex: 1}}} + felixServer.GetCache().FelixConfig = felixConfig + common.VppManagerInfo = &agentConf.VppManagerInfo{ + UplinkStatuses: map[string]agentConf.UplinkStatus{ + "eth0": {IsMain: true, SwIfIndex: 1}, + }, + } }) Describe("Addition of the node", func() { diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/felix/cni/cni_pod_test.go similarity index 90% rename from calico-vpp-agent/cni/cni_pod_test.go rename to calico-vpp-agent/felix/cni/cni_pod_test.go index a79d89073..a969c1514 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/felix/cni/cni_pod_test.go @@ -14,7 +14,6 @@ package cni_test import ( - "context" "fmt" "net" "os" @@ -27,12 +26,13 @@ import ( . "github.com/onsi/gomega" gs "github.com/onsi/gomega/gstruct" cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" - felixconfig "github.com/projectcalico/calico/felix/config" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -51,10 +51,10 @@ const ( var _ = Describe("Pod-related functionality of CNI", func() { var ( - log *logrus.Logger - vpp *vpplink.VppLink - cniServer *cni.Server - ipamStub *mocks.IpamCacheStub + log *logrus.Logger + vpp *vpplink.VppLink + cniHandler *cni.CNIHandler + testCache *cache.Cache ) BeforeEach(func() { @@ -65,16 +65,18 @@ var _ = Describe("Pod-related functionality of CNI", func() { nodeIP6String := net.ParseIP("2001:db8::68") testutils.StartVPP() vpp, _ = testutils.ConfigureVPP(log) + Expect(vpp.CnatSetSnatAddresses(nodeIP4String, nodeIP6String)).To(Succeed()) // setup connectivity server (functionality target of tests) - if ipamStub == nil { - ipamStub = mocks.NewIpamCacheStub() - } + testCache = cache.NewCache(log.WithFields(logrus.Fields{"component": "cache"})) + testCache.VppAvailableBuffers = 65536 // setup CNI server (functionality target of tests) common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) - cniServer = cni.NewCNIServer(vpp, ipamStub, log.WithFields(logrus.Fields{"component": "cni"})) - cniServer.SetFelixConfig(&felixconfig.Config{}) - cniServer.FetchBufferConfig() - vpp.CnatSetSnatAddresses(nodeIP4String, nodeIP6String) + cniHandler = cni.NewCNIHandler(vpp, testCache, log.WithFields(logrus.Fields{"component": "cni"})) + cfg := &config.CalicoVppInterfacesConfigType{ + DefaultPodIfSpec: &config.InterfaceSpec{}, + } + config.CalicoVppInterfaces = &cfg + cfg.DefaultPodIfSpec.Validate(nil) }) Describe("Addition of the pod", func() { @@ -97,7 +99,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding pod using CNI server") - newPod := &cniproto.AddRequest{ + podSpec, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: interfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, @@ -106,7 +108,8 @@ var _ = Describe("Pod-related functionality of CNI", func() { "cni.projectcalico.org/AllowedSourcePrefixes": "[\"172.16.104.7\", \"3.4.5.6\"]", }, }, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} os.Setenv("NODENAME", testutils.ThisNodeName) os.Setenv("CALICOVPP_CONFIG_TEMPLATE", "sss") @@ -117,7 +120,9 @@ var _ = Describe("Pod-related functionality of CNI", func() { } config.GetCalicoVppFeatureGates().IPSecEnabled = &config.False config.GetCalicoVppDebug().GSOEnabled = &config.True - reply, err := cniServer.Add(context.Background(), newPod) + evt := model.NewCniPodAddEvent(podSpec) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) @@ -131,7 +136,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { "for IP address or doesn't exist at all") By("Checking existence of interface tunnel at VPP's end") - ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPod) + ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, podSpec) By("Checking correct IP address of interface tunnel at VPP's end") testutils.AssertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) @@ -139,14 +144,14 @@ var _ = Describe("Pod-related functionality of CNI", func() { By("Checking correct MTU for tunnel interface at VPP's end") testutils.AssertTunnelInterfaceMTU(vpp, ifSwIfIndex) - testutils.RunInPod(newPod.Netns, func() { + testutils.RunInPod(podSpec.NetnsName, func() { By("Checking tun interface on pod side") _, err := netlink.LinkByName(interfaceName) Expect(err).ToNot(HaveOccurred(), "can't find tun interface in pod") }) By("Checking created pod RPF VRF") - RPFVRF := testutils.AssertRPFVRFExistence(vpp, interfaceName, newPod.Netns) + RPFVRF := testutils.AssertRPFVRFExistence(vpp, interfaceName, podSpec.NetnsName) By("Checking RPF routes are added") testutils.AssertRPFRoutes(vpp, RPFVRF, ifSwIfIndex, ipAddress) @@ -179,7 +184,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding pod using CNI server") - newPod := &cniproto.AddRequest{ + podSpec, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: interfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, @@ -190,21 +195,24 @@ var _ = Describe("Pod-related functionality of CNI", func() { memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPod) + evt := model.NewCniPodAddEvent(podSpec) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) By("Checking existence of main interface tunnel to pod (at VPP's end)") - ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPod) + ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, podSpec) By("Checking main tunnel's tun interface for common interface attributes") testutils.AssertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) testutils.AssertTunnelInterfaceMTU(vpp, ifSwIfIndex) - testutils.RunInPod(newPod.Netns, func() { + testutils.RunInPod(podSpec.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(interfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -212,7 +220,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { By("Checking secondary tunnel's memif interface for existence") memifSwIfIndex, err := vpp.SearchInterfaceWithTag( - testutils.InterfaceTagForLocalMemifTunnel(newPod.InterfaceName, newPod.Netns)) + testutils.InterfaceTagForLocalMemifTunnel(podSpec.InterfaceName, podSpec.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get memif interface at VPP's end") By("Checking secondary tunnel's memif interface for common interface attributes") @@ -235,7 +243,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { socket, err := vpp.MemifsocketByID(memifs[0].SocketID) Expect(err).ToNot(HaveOccurred(), "failed to get memif socket") Expect(socket.SocketFilename).To(Equal( - fmt.Sprintf("abstract:vpp/memif-%s,netns_name=%s", newPod.InterfaceName, newPod.Netns)), + fmt.Sprintf("abstract:vpp/memif-%s,netns_name=%s", podSpec.InterfaceName, podSpec.NetnsName)), "memif socket file is not configured correctly") By("Checking PBL (packet punting) to redirect some traffic into memif (secondary interface)") @@ -366,7 +374,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Name: networkName, Range: "10.1.1.0/24", // IP range for secondary network defined by multinet } - cniServer.ForceAddingNetworkDefinition(networkDefinition) + cniHandler.ForceAddingNetworkDefinition(networkDefinition) // setup PubSub handler to catch LocalPodAddressAdded events pubSubHandlerMock = mocks.NewPubSubHandlerMock(common.LocalPodAddressAdded) @@ -395,21 +403,24 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding Pod to primary network using CNI server") - newPodForPrimaryNetwork := &cniproto.AddRequest{ + newPodForPrimaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: mainInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, Workload: &cniproto.WorkloadIDs{}, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPodForPrimaryNetwork) + evt := model.NewCniPodAddEvent(newPodForPrimaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to primary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) By("Adding Pod to secondary(multinet) network using CNI server") secondaryIPAddress := testutils.FirstIPinIPRange(networkDefinition.Range).String() - newPodForSecondaryNetwork := &cniproto.AddRequest{ + newPodForSecondaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{ @@ -426,8 +437,11 @@ var _ = Describe("Pod-related functionality of CNI", func() { memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, - } - reply, err = cniServer.Add(context.Background(), newPodForSecondaryNetwork) + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") + evt = model.NewCniPodAddEvent(newPodForSecondaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply = <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to secondary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to secondary network failed due to: %s", reply.ErrorMessage)) @@ -443,7 +457,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { // secondarySwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPodForSecondaryNetwork) By("Checking secondary tunnel's memif interface for existence") memifSwIfIndex, err := vpp.SearchInterfaceWithTag( - testutils.InterfaceTagForLocalMemifTunnel(newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.Netns)) + testutils.InterfaceTagForLocalMemifTunnel(newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get memif interface at VPP's end") By("Checking secondary tunnel's memif interface for common interface attributes") @@ -466,10 +480,10 @@ var _ = Describe("Pod-related functionality of CNI", func() { socket, err := vpp.MemifsocketByID(memifs[0].SocketID) Expect(err).ToNot(HaveOccurred(), "failed to get memif socket") Expect(socket.SocketFilename).To(Equal( - fmt.Sprintf("abstract:%s,netns_name=%s", newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.Netns)), + fmt.Sprintf("abstract:%s,netns_name=%s", newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.NetnsName)), "memif socket file is not configured correctly") - testutils.RunInPod(newPodForSecondaryNetwork.Netns, func() { + testutils.RunInPod(newPodForSecondaryNetwork.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(mainInterfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -507,7 +521,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { )) By("Checking default route from pod-specific VRF to multinet network-specific pod-vrf") - podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.Netns, vpp) + podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.NetnsName, vpp) Expect(err).ToNot(HaveOccurred(), "can't find pod-specific VRFs") for idx, ipFamily := range vpplink.IPFamilies { podVrfID := podVrf4ID @@ -638,21 +652,25 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding Pod to primary network using CNI server") - newPodForPrimaryNetwork := &cniproto.AddRequest{ + newPodForPrimaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: mainInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, Workload: &cniproto.WorkloadIDs{}, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPodForPrimaryNetwork) + + evt := model.NewCniPodAddEvent(newPodForPrimaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to primary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) By("Adding Pod to secondary(multinet) network using CNI server") secondaryIPAddress := testutils.FirstIPinIPRange(networkDefinition.Range).String() - newPodForSecondaryNetwork := &cniproto.AddRequest{ + newPodForSecondaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{ @@ -662,8 +680,11 @@ var _ = Describe("Pod-related functionality of CNI", func() { DataplaneOptions: map[string]string{ testutils.DpoNetworkNameFieldName(): networkDefinition.Name, }, - } - reply, err = cniServer.Add(context.Background(), newPodForSecondaryNetwork) + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") + evt = model.NewCniPodAddEvent(newPodForSecondaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply = <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to secondary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to secondary network failed due to: %s", reply.ErrorMessage)) @@ -682,7 +703,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { testutils.AssertTunnelInterfaceIPAddress(vpp, secondarySwIfIndex, secondaryIPAddress) testutils.AssertTunnelInterfaceMTU(vpp, secondarySwIfIndex) - testutils.RunInPod(newPodForSecondaryNetwork.Netns, func() { + testutils.RunInPod(newPodForSecondaryNetwork.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(mainInterfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -720,7 +741,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { )) By("Checking default route from pod-specific VRF to multinet network-specific vrf") - podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.Netns, vpp) + podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.NetnsName, vpp) Expect(err).ToNot(HaveOccurred(), "can't find pod-specific VRFs") for idx, ipFamily := range vpplink.IPFamilies { podVrfID := podVrf4ID diff --git a/calico-vpp-agent/felix/cni/model/events.go b/calico-vpp-agent/felix/cni/model/events.go new file mode 100644 index 000000000..2b26915c8 --- /dev/null +++ b/calico-vpp-agent/felix/cni/model/events.go @@ -0,0 +1,44 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" +) + +type CniPodDelEvent struct { + PodSpecKey string + Done chan *cniproto.DelReply +} + +func NewCniPodDelEvent(podSpecKey string) *CniPodDelEvent { + return &CniPodDelEvent{ + PodSpecKey: podSpecKey, + Done: make(chan *cniproto.DelReply, 1), + } +} + +type CniPodAddEvent struct { + PodSpec *LocalPodSpec + Done chan *cniproto.AddReply +} + +func NewCniPodAddEvent(podSpec *LocalPodSpec) *CniPodAddEvent { + return &CniPodAddEvent{ + PodSpec: podSpec, + Done: make(chan *cniproto.AddReply, 1), + } +} diff --git a/calico-vpp-agent/cni/model/pod_annotations.go b/calico-vpp-agent/felix/cni/model/pod_annotations.go similarity index 100% rename from calico-vpp-agent/cni/model/pod_annotations.go rename to calico-vpp-agent/felix/cni/model/pod_annotations.go diff --git a/calico-vpp-agent/cni/model/pod_spec.go b/calico-vpp-agent/felix/cni/model/pod_spec.go similarity index 97% rename from calico-vpp-agent/cni/model/pod_spec.go rename to calico-vpp-agent/felix/cni/model/pod_spec.go index a964c8051..b6ddd7146 100644 --- a/calico-vpp-agent/cni/model/pod_spec.go +++ b/calico-vpp-agent/felix/cni/model/pod_spec.go @@ -25,6 +25,7 @@ import ( cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -78,7 +79,7 @@ type LocalPodSpec struct { NetworkName string `json:"networkName"` } -func NewLocalPodSpecFromAdd(request *cniproto.AddRequest, nodeBGPSpec *common.LocalNodeSpec) (*LocalPodSpec, error) { +func NewLocalPodSpecFromAdd(request *cniproto.AddRequest) (*LocalPodSpec, error) { podAnnotations, err := NewPodAnnotations( request.GetInterfaceName(), request.GetWorkload().GetAnnotations(), @@ -255,12 +256,12 @@ func (podSpec *LocalPodSpec) Hasv46() (hasv4 bool, hasv6 bool) { return hasv4, hasv6 } -func (podSpec *LocalPodSpec) NeedsSnat(felixServerIpam common.FelixServerIpam, isIP6 bool) bool { +func (podSpec *LocalPodSpec) NeedsSnat(cache *cache.Cache, isIP6 bool) bool { for _, containerIP := range podSpec.GetContainerIPs() { if containerIP.IP.To4() == nil != isIP6 { continue } - if felixServerIpam.IPNetNeedsSNAT(containerIP) { + if cache.IPNetNeedsSNAT(containerIP) { return true } } diff --git a/calico-vpp-agent/cni/model/pod_status.go b/calico-vpp-agent/felix/cni/model/pod_status.go similarity index 100% rename from calico-vpp-agent/cni/model/pod_status.go rename to calico-vpp-agent/felix/cni/model/pod_status.go diff --git a/calico-vpp-agent/cni/model/server_state.go b/calico-vpp-agent/felix/cni/model/server_state.go similarity index 100% rename from calico-vpp-agent/cni/model/server_state.go rename to calico-vpp-agent/felix/cni/model/server_state.go diff --git a/calico-vpp-agent/cni/netns_linux.go b/calico-vpp-agent/felix/cni/netns_linux.go similarity index 100% rename from calico-vpp-agent/cni/netns_linux.go rename to calico-vpp-agent/felix/cni/netns_linux.go diff --git a/calico-vpp-agent/cni/network_vpp.go b/calico-vpp-agent/felix/cni/network_vpp.go similarity index 90% rename from calico-vpp-agent/cni/network_vpp.go rename to calico-vpp-agent/felix/cni/network_vpp.go index eb8f89340..56f615ef8 100644 --- a/calico-vpp-agent/cni/network_vpp.go +++ b/calico-vpp-agent/felix/cni/network_vpp.go @@ -22,8 +22,8 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/pkg/errors" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -42,23 +42,23 @@ type NetworkPod struct { ContainerIP *net.IPNet } -func (s *Server) checkAvailableBuffers(podSpec *model.LocalPodSpec) error { +func (s *CNIHandler) checkAvailableBuffers(podSpec *model.LocalPodSpec) error { podBuffers := podSpec.GetBuffersNeeded() buffers := podBuffers existingPods := uint64(len(s.podInterfaceMap)) for _, existingPodSpec := range s.podInterfaceMap { buffers += existingPodSpec.GetBuffersNeeded() } - s.log.Infof("pod(add) checking available buffers, %d existing pods, request for this pod: %d, total request: %d / %d", existingPods, podBuffers, buffers, s.availableBuffers) - if buffers > s.availableBuffers { + s.log.Infof("pod(add) checking available buffers, %d existing pods, request for this pod: %d, total request: %d / %d", existingPods, podBuffers, buffers, s.cache.VppAvailableBuffers) + if buffers > s.cache.VppAvailableBuffers { return errors.Errorf("Cannot create interface: Out of buffers: available buffers = %d, buffers needed = %d. "+ "Increase buffers-per-numa in the VPP configuration or reduce CALICOVPP_TAP_RING_SIZE to allow more "+ - "pods to be scheduled. Limit the number of pods per node to prevent this error", s.availableBuffers, buffers) + "pods to be scheduled. Limit the number of pods per node to prevent this error", s.cache.VppAvailableBuffers, buffers) } return nil } -func (s *Server) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { +func (s *CNIHandler) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { podSpec.V4VrfID = types.InvalidID podSpec.V6VrfID = types.InvalidID @@ -92,7 +92,7 @@ func (s *Server) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { return false } -func (s *Server) removeConflictingContainers(newAddresses []net.IP, networkName string) { +func (s *CNIHandler) removeConflictingContainers(newAddresses []net.IP, networkName string) { addrMap := make(map[string]model.LocalPodSpec) for _, podSpec := range s.podInterfaceMap { for _, addr := range podSpec.ContainerIPs { @@ -124,7 +124,7 @@ func (s *Server) removeConflictingContainers(newAddresses []net.IP, networkName } // AddVppInterface performs the networking for the given config and IPAM result -func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { +func (s *CNIHandler) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { err = ns.IsNSorErr(podSpec.NetnsName) if err != nil { return vpplink.InvalidID, PodNSNotFoundErr{podSpec.NetnsName} @@ -132,7 +132,7 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo if podSpec.NetworkName != "" { s.log.Infof("Checking network exists") - _, ok := s.networkDefinitions.Load(podSpec.NetworkName) + _, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network %s does not exist", podSpec.NetworkName) return vpplink.InvalidID, errors.Errorf("network %s does not exist", podSpec.NetworkName) @@ -243,14 +243,10 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo } if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vni = networkDefinition.Vni } } @@ -292,7 +288,7 @@ err: } // CleanUpVPPNamespace deletes the devices in the network namespace. -func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DelVppInterface(podSpec *model.LocalPodSpec) { if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { err := s.DelRedirectToHostOnInterface(podSpec.TunTapSwIfIndex) if err != nil { @@ -315,14 +311,10 @@ func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { var vni uint32 deleteLocalPodAddress := true if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { deleteLocalPodAddress = false } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vni = networkDefinition.Vni } } diff --git a/calico-vpp-agent/cni/network_vpp_hostports.go b/calico-vpp-agent/felix/cni/network_vpp_hostports.go similarity index 78% rename from calico-vpp-agent/cni/network_vpp_hostports.go rename to calico-vpp-agent/felix/cni/network_vpp_hostports.go index aec0de83d..a22aafcfd 100644 --- a/calico-vpp-agent/cni/network_vpp_hostports.go +++ b/calico-vpp-agent/felix/cni/network_vpp_hostports.go @@ -18,36 +18,34 @@ package cni import ( "net" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) // getHostPortHostIP returns the hostIP for a given // hostIP strings and an IP family -func (s *Server) getHostPortHostIP(hostIP net.IP, isIP6 bool) net.IP { +func (s *CNIHandler) getHostPortHostIP(hostIP *net.IP, isIP6 bool) *net.IP { if hostIP != nil && !hostIP.IsUnspecified() { if (hostIP.To4() == nil) == isIP6 { return hostIP } - } else if s.nodeBGPSpec != nil { - if isIP6 && s.nodeBGPSpec.IPv6Address != nil { - return s.nodeBGPSpec.IPv6Address.IP - } else if !isIP6 && s.nodeBGPSpec.IPv4Address != nil { - return s.nodeBGPSpec.IPv4Address.IP - } } - return net.IP{} + if isIP6 { + return s.cache.GetNodeIP6() + } else { + return s.cache.GetNodeIP4() + } } -func (s *Server) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) error { +func (s *CNIHandler) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) error { for _, hostPort := range podSpec.HostPorts { for _, containerAddr := range podSpec.ContainerIPs { - hostIP := s.getHostPortHostIP(hostPort.HostIP, vpplink.IsIP6(containerAddr)) + hostIP := s.getHostPortHostIP(&hostPort.HostIP, vpplink.IsIP6(containerAddr)) if hostIP != nil && !hostIP.IsUnspecified() { entry := &types.CnatTranslateEntry{ Endpoint: types.CnatEndpoint{ - IP: hostIP, + IP: *hostIP, Port: hostPort.HostPort, }, Backends: []types.CnatEndpointTuple{{ @@ -77,7 +75,7 @@ func (s *Server) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.Cleanup return nil } -func (s *Server) DelHostPort(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DelHostPort(podSpec *model.LocalPodSpec) { initialSpec, ok := s.podInterfaceMap[podSpec.Key()] if ok { for hostport, entryIDs := range initialSpec.HostPortEntryIDs { diff --git a/calico-vpp-agent/cni/network_vpp_routes.go b/calico-vpp-agent/felix/cni/network_vpp_routes.go similarity index 85% rename from calico-vpp-agent/cni/network_vpp_routes.go rename to calico-vpp-agent/felix/cni/network_vpp_routes.go index 345e34045..d8ae66f78 100644 --- a/calico-vpp-agent/cni/network_vpp_routes.go +++ b/calico-vpp-agent/felix/cni/network_vpp_routes.go @@ -18,13 +18,13 @@ package cni import ( "github.com/pkg/errors" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool, inPodVrf bool) error { +func (s *CNIHandler) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool, inPodVrf bool) error { for _, containerIP := range podSpec.GetContainerIPs() { var table uint32 if podSpec.NetworkName != "" { @@ -32,14 +32,10 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C if vpplink.IsIP6(containerIP.IP) { idx = 1 } - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } table = networkDefinition.VRF.Tables[idx] } } else if inPodVrf { @@ -75,7 +71,7 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C return nil } -func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, inPodVrf bool, isL3 bool) { +func (s *CNIHandler) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, inPodVrf bool, isL3 bool) { for _, containerIP := range podSpec.GetContainerIPs() { var table uint32 if podSpec.NetworkName != "" { @@ -83,14 +79,10 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint if vpplink.IsIP6(containerIP.IP) { idx = 1 } - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } table = networkDefinition.VRF.Tables[idx] } } else if inPodVrf { @@ -123,7 +115,7 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint } } -func (s *Server) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { +func (s *CNIHandler) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { for _, containerIP := range podSpec.ContainerIPs { path := types.RoutePath{ SwIfIndex: swIfIndex, @@ -179,7 +171,7 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *v return nil } -func (s *Server) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, isL3 bool) { +func (s *CNIHandler) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, isL3 bool) { for _, pblIndex := range podSpec.PblIndexes { s.log.Infof("pod(del) PBL client[%d]", pblIndex) err := s.vpp.DelPblClient(pblIndex) @@ -204,7 +196,7 @@ func (s *Server) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIn } -func (s *Server) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, ipFamily := range vpplink.IPFamilies { vrfID, err := s.vpp.AllocateVRF(ipFamily.IsIP6, podSpec.GetVrfTag(ipFamily, "RPF")) podSpec.SetRPFVrfID(vrfID, ipFamily) @@ -218,7 +210,7 @@ func (s *Server) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cle return nil } -func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { /* Create and Setup the per-pod VRF */ for _, ipFamily := range vpplink.IPFamilies { vrfID, err := s.vpp.AllocateVRF(ipFamily.IsIP6, podSpec.GetVrfTag(ipFamily, "")) @@ -237,14 +229,10 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu if podSpec.NetworkName == "" { // no multi net vrfIndex = common.PodVRFIndex } else { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { return errors.Errorf("network not found %s", podSpec.NetworkName) } - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vrfIndex = networkDefinition.PodVRF.Tables[idx] } s.log.Infof("pod(add) VRF %d %s default route via VRF %d", vrfID, ipFamily.Str, vrfIndex) @@ -265,7 +253,7 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu return nil } -func (s *Server) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { s.log.Infof("pod(add) create pod RPF VRF") err = s.CreatePodRPFVRF(podSpec, stack) if err != nil { @@ -288,7 +276,7 @@ func (s *Server) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.C return nil } -func (s *Server) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { rpfVrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) // Always there (except multinet memif) @@ -337,7 +325,7 @@ func (s *Server) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu return nil } -func (s *Server) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { var err error for _, containerIP := range podSpec.GetContainerIPs() { rpfVrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) @@ -389,7 +377,7 @@ func (s *Server) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { } } -func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeletePodVRF(podSpec *model.LocalPodSpec) { var err error for idx, ipFamily := range vpplink.IPFamilies { vrfID := podSpec.GetVrfID(ipFamily) @@ -397,14 +385,10 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { if podSpec.NetworkName == "" { vrfIndex = common.PodVRFIndex } else { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vrfIndex = networkDefinition.PodVRF.Tables[idx] } } @@ -429,7 +413,7 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { } } -func (s *Server) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the main table route the container address to its VRF */ route := types.Route{ @@ -450,7 +434,7 @@ func (s *Server) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplin return nil } -func (s *Server) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the main table route the container address to its VRF */ route := types.Route{ @@ -468,7 +452,7 @@ func (s *Server) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { } } -func (s *Server) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { +func (s *CNIHandler) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the punt table (where all punted traffics ends), * route the container to the tun */ @@ -488,7 +472,7 @@ func (s *Server) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.Cle return nil } -func (s *Server) RemovePuntRoutes(podSpec *model.LocalPodSpec, swIfIndex uint32) { +func (s *CNIHandler) RemovePuntRoutes(podSpec *model.LocalPodSpec, swIfIndex uint32) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the punt table (where all punted traffics ends), route the container to the tun */ route := types.Route{ diff --git a/calico-vpp-agent/cni/packet_helper.go b/calico-vpp-agent/felix/cni/packet_helper.go similarity index 100% rename from calico-vpp-agent/cni/packet_helper.go rename to calico-vpp-agent/felix/cni/packet_helper.go diff --git a/calico-vpp-agent/cni/podinterface/common.go b/calico-vpp-agent/felix/cni/podinterface/common.go similarity index 88% rename from calico-vpp-agent/cni/podinterface/common.go rename to calico-vpp-agent/felix/cni/podinterface/common.go index 123d61688..ed3a09229 100644 --- a/calico-vpp-agent/cni/podinterface/common.go +++ b/calico-vpp-agent/felix/cni/podinterface/common.go @@ -19,25 +19,24 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type PodInterfaceDriverData struct { - log *logrus.Entry - vpp *vpplink.VppLink - Name string - NDataThreads int - felixServerIpam common.FelixServerIpam + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + Name string } func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTxQueues int) (err error) { i.log.WithFields(map[string]interface{}{ "swIfIndex": swIfIndex, - }).Debugf("Spreading %d TX queues on %d workers for pod interface: %v", numTxQueues, i.NDataThreads, i.Name) + }).Debugf("Spreading %d TX queues on %d workers for pod interface: %v", numTxQueues, i.cache.NumDataThreads, i.Name) // set first tx queue for main worker err = i.vpp.SetInterfaceTxPlacement(swIfIndex, 0 /* queue */, 0 /* worker */) @@ -45,9 +44,9 @@ func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTx return err } // share tx queues between the rest of workers - if i.NDataThreads > 0 { + if i.cache.NumDataThreads > 0 { for txq := 1; txq < numTxQueues; txq++ { - err = i.vpp.SetInterfaceTxPlacement(swIfIndex, txq /* queue */, (txq-1)%(i.NDataThreads)+1 /* worker */) + err = i.vpp.SetInterfaceTxPlacement(swIfIndex, txq /* queue */, (txq-1)%(i.cache.NumDataThreads)+1 /* worker */) if err != nil { return err } @@ -59,14 +58,14 @@ func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTx func (i *PodInterfaceDriverData) SpreadRxQueuesOnWorkers(swIfIndex uint32, numRxQueues int) { i.log.WithFields(map[string]interface{}{ "swIfIndex": swIfIndex, - }).Debugf("Spreading %d RX queues on %d workers for pod interface: %v", numRxQueues, i.NDataThreads, i.Name) + }).Debugf("Spreading %d RX queues on %d workers for pod interface: %v", numRxQueues, i.cache.NumDataThreads, i.Name) - if i.NDataThreads > 0 { + if i.cache.NumDataThreads > 0 { for queue := 0; queue < numRxQueues; queue++ { - worker := (int(swIfIndex)*numRxQueues + queue) % i.NDataThreads + worker := (int(swIfIndex)*numRxQueues + queue) % i.cache.NumDataThreads err := i.vpp.SetInterfaceRxPlacement(swIfIndex, queue, worker, false /* main */) if err != nil { - i.log.Warnf("failed to set if[%d] queue:%d worker:%d (tot workers %d): %v", swIfIndex, queue, worker, i.NDataThreads, err) + i.log.Warnf("failed to set if[%d] queue:%d worker:%d (tot workers %d): %v", swIfIndex, queue, worker, i.cache.NumDataThreads, err) } } } @@ -89,7 +88,7 @@ func (i *PodInterfaceDriverData) UndoPodIfNatConfiguration(swIfIndex uint32) { func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { for _, ipFamily := range vpplink.IPFamilies { - if podSpec.NeedsSnat(i.felixServerIpam, ipFamily.IsIP6) { + if podSpec.NeedsSnat(i.cache, ipFamily.IsIP6) { i.log.Infof("pod(add) Enable interface[%d] SNAT", swIfIndex) err = i.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIP6, true /*isEnable*/) if err != nil { diff --git a/calico-vpp-agent/cni/podinterface/loopback.go b/calico-vpp-agent/felix/cni/podinterface/loopback.go similarity index 85% rename from calico-vpp-agent/cni/podinterface/loopback.go rename to calico-vpp-agent/felix/cni/podinterface/loopback.go index 86adf083b..d16f7a489 100644 --- a/calico-vpp-agent/cni/podinterface/loopback.go +++ b/calico-vpp-agent/felix/cni/podinterface/loopback.go @@ -19,8 +19,9 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) @@ -28,16 +29,15 @@ type LoopbackPodInterfaceDriver struct { PodInterfaceDriverData } -func NewLoopbackPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry, felixServerIpam common.FelixServerIpam) *LoopbackPodInterfaceDriver { - i := &LoopbackPodInterfaceDriver{ +func NewLoopbackPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *LoopbackPodInterfaceDriver { + return &LoopbackPodInterfaceDriver{ PodInterfaceDriverData: PodInterfaceDriverData{ - felixServerIpam: felixServerIpam, + vpp: vpp, + log: log, + cache: cache, + Name: "loopback", }, } - i.vpp = vpp - i.log = log - i.Name = "loopback" - return i } func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { diff --git a/calico-vpp-agent/cni/podinterface/memif.go b/calico-vpp-agent/felix/cni/podinterface/memif.go similarity index 95% rename from calico-vpp-agent/cni/podinterface/memif.go rename to calico-vpp-agent/felix/cni/podinterface/memif.go index 0b77a243e..7c054a5a3 100644 --- a/calico-vpp-agent/cni/podinterface/memif.go +++ b/calico-vpp-agent/felix/cni/podinterface/memif.go @@ -23,8 +23,8 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -45,16 +45,15 @@ func (d *dummy) Attrs() *netlink.LinkAttrs { return &netlink.LinkAttrs{Name: d.name} } -func NewMemifPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry, felixServerIpam common.FelixServerIpam) *MemifPodInterfaceDriver { - i := &MemifPodInterfaceDriver{ +func NewMemifPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *MemifPodInterfaceDriver { + return &MemifPodInterfaceDriver{ PodInterfaceDriverData: PodInterfaceDriverData{ - felixServerIpam: felixServerIpam, + vpp: vpp, + log: log, + cache: cache, + Name: "memif", }, } - i.vpp = vpp - i.log = log - i.Name = "memif" - return i } func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, doHostSideConf bool) (err error) { diff --git a/calico-vpp-agent/cni/podinterface/tuntap.go b/calico-vpp-agent/felix/cni/podinterface/tuntap.go similarity index 92% rename from calico-vpp-agent/cni/podinterface/tuntap.go rename to calico-vpp-agent/felix/cni/podinterface/tuntap.go index fa8338e65..b73a01abd 100644 --- a/calico-vpp-agent/cni/podinterface/tuntap.go +++ b/calico-vpp-agent/felix/cni/podinterface/tuntap.go @@ -28,8 +28,9 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -37,21 +38,19 @@ import ( type TunTapPodInterfaceDriver struct { PodInterfaceDriverData - felixConfig *felixConfig.Config ipipEncapRefCounts int /* how many ippools with IPIP */ vxlanEncapRefCounts int /* how many ippools with VXLAN */ } -func NewTunTapPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry, felixServerIpam common.FelixServerIpam) *TunTapPodInterfaceDriver { - i := &TunTapPodInterfaceDriver{ +func NewTunTapPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *TunTapPodInterfaceDriver { + return &TunTapPodInterfaceDriver{ PodInterfaceDriverData: PodInterfaceDriverData{ - felixServerIpam: felixServerIpam, + vpp: vpp, + log: log, + cache: cache, + Name: "tun", }, } - i.vpp = vpp - i.log = log - i.Name = "tun" - return i } func reduceMtuIf(podMtu *int, tunnelMtu int, tunnelEnabled bool) { @@ -97,10 +96,6 @@ func (i *TunTapPodInterfaceDriver) computePodMtu(podSpecMtu int, fc *felixConfig return podMtu } -func (i *TunTapPodInterfaceDriver) SetFelixConfig(felixConfig *felixConfig.Config) { - i.felixConfig = felixConfig -} - /** * This is called when the felix config or ippool encap refcount change, * and update the linux mtu accordingly. @@ -108,12 +103,12 @@ func (i *TunTapPodInterfaceDriver) SetFelixConfig(felixConfig *felixConfig.Confi */ func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfig.Config, ipipEncapRefCountDelta int, vxlanEncapRefCountDelta int, podSpecs map[string]model.LocalPodSpec) { if newFelixConfig == nil { - newFelixConfig = i.felixConfig + newFelixConfig = i.cache.FelixConfig } - if i.felixConfig != nil { + if i.cache.FelixConfig != nil { for name, podSpec := range podSpecs { - oldMtu := i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0) - newMtu := i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts+ipipEncapRefCountDelta > 0, i.vxlanEncapRefCounts+vxlanEncapRefCountDelta > 0) + oldMtu := i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0) + newMtu := i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts+ipipEncapRefCountDelta > 0, i.vxlanEncapRefCounts+vxlanEncapRefCountDelta > 0) if oldMtu != newMtu { i.log.Infof("pod(upd) reconfiguring mtu=%d pod=%s", newMtu, name) err := ns.WithNetNSPath(podSpec.NetnsName, func(ns.NetNS) error { @@ -134,7 +129,7 @@ func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfi } } - i.felixConfig = newFelixConfig + i.cache.FelixConfig = newFelixConfig i.ipipEncapRefCounts = i.ipipEncapRefCounts + ipipEncapRefCountDelta i.vxlanEncapRefCounts = i.vxlanEncapRefCounts + vxlanEncapRefCountDelta } @@ -150,7 +145,7 @@ func (i *TunTapPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, }, HostNamespace: podSpec.NetnsName, Tag: podSpec.GetInterfaceTag(i.Name), - HostMtu: i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0), + HostMtu: i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0), } if *podSpec.IfSpec.IsL3 { diff --git a/calico-vpp-agent/cni/podinterface/vcl.go b/calico-vpp-agent/felix/cni/podinterface/vcl.go similarity index 85% rename from calico-vpp-agent/cni/podinterface/vcl.go rename to calico-vpp-agent/felix/cni/podinterface/vcl.go index 677ee157c..e3175a6a0 100644 --- a/calico-vpp-agent/cni/podinterface/vcl.go +++ b/calico-vpp-agent/felix/cni/podinterface/vcl.go @@ -20,8 +20,8 @@ import ( "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -35,16 +35,15 @@ func getPodAppNamespaceName(podSpec *model.LocalPodSpec) string { return fmt.Sprintf("app-ns-%s", podSpecKey) } -func NewVclPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry, felixServerIpam common.FelixServerIpam) *VclPodInterfaceDriver { - i := &VclPodInterfaceDriver{ +func NewVclPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *VclPodInterfaceDriver { + return &VclPodInterfaceDriver{ PodInterfaceDriverData: PodInterfaceDriverData{ - felixServerIpam: felixServerIpam, + vpp: vpp, + log: log, + cache: cache, + Name: "vcl", }, } - i.vpp = vpp - i.log = log - i.Name = "vcl" - return i } func (i *VclPodInterfaceDriver) Init() (err error) { diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index 0f125184e..c288dd691 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -28,12 +28,15 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" + "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) // Server holds all the data required to configure the policies defined by felix in VPP @@ -49,6 +52,7 @@ type Server struct { ippoolLock sync.RWMutex policiesHandler *policies.PoliciesHandler + cniHandler *cni.CNIHandler prometheusServer *prometheus.PrometheusServer } @@ -67,6 +71,7 @@ func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *l cache: cache, policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), + cniHandler: cni.NewCNIHandler(vpp, cache, log), prometheusServer: prometheus.NewPrometheusServer(vpp, log.WithFields(logrus.Fields{"component": "prometheus"})), } @@ -110,13 +115,81 @@ func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { return s.cache.GetPrefixIPPool(prefix) } -func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { - pool := s.GetPrefixIPPool(prefix) - if pool == nil { - return false +func (s *Server) getMainInterface() *config.UplinkStatus { + for _, i := range common.VppManagerInfo.UplinkStatuses { + if i.IsMain { + return &i + } + } + return nil +} + +func (s *Server) createRedirectToHostRules() error { + var maxNumEntries uint32 + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 { + maxNumEntries = uint32(2 * len(config.GetCalicoVppInitialConfig().RedirectToHostRules)) } else { - return pool.Masquerade + maxNumEntries = 1 + } + index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{ + Mask: types.DstThreeTupleMask, + NextTableIndex: types.InvalidID, + MaxNumEntries: maxNumEntries, + MissNextIndex: ^uint32(0), + }) + if err != nil { + return err + } + mainInterface := s.getMainInterface() + if mainInterface == nil { + return fmt.Errorf("no main interface found") + } + for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules { + mainInterfaceAddress := mainInterface.GetAddress(vpplink.IPFamilyFromIP(rule.IP)) + if mainInterfaceAddress == nil { + return fmt.Errorf("error installing rule %v no address found on uplink", rule) + } + err = s.vpp.AddSessionRedirect(&types.SessionRedirect{ + FiveTuple: types.NewDst3Tuple(rule.Proto, rule.IP, rule.Port), + TableIndex: index, + }, &types.RoutePath{ + Gw: mainInterfaceAddress.IP, + SwIfIndex: mainInterface.TapSwIfIndex, + }) + if err != nil { + return err + } + } + + s.cache.RedirectToHostClassifyTableIndex = index + return nil +} + +func (s *Server) fetchNumDataThreads() error { + nVppWorkers, err := s.vpp.GetNumVPPWorkers() + if err != nil { + return errors.Wrap(err, "Error getting number of VPP workers") } + nDataThreads := nVppWorkers + if config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread > 0 { + nDataThreads = nVppWorkers - config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread + if nDataThreads <= 0 { + s.log.Errorf("Couldn't fulfill request [crypto=%d total=%d]", config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread, nVppWorkers) + nDataThreads = nVppWorkers + } + s.log.Infof("Using ipsec workers [data=%d crypto=%d]", nDataThreads, nVppWorkers-nDataThreads) + } + s.cache.NumDataThreads = nDataThreads + return nil +} + +func (s *Server) fetchBufferConfig() error { + availableBuffers, _, _, err := s.vpp.GetBufferStats() + if err != nil { + return errors.Wrap(err, "could not get available buffers") + } + s.cache.VppAvailableBuffers = uint64(availableBuffers) + return nil } // Serve runs the felix server @@ -131,6 +204,22 @@ func (s *Server) ServeFelix(t *tomb.Tomb) error { if err != nil { return errors.Wrap(err, "Error in PoliciesHandlerInit") } + err = s.createRedirectToHostRules() + if err != nil { + return errors.Wrap(err, "Error in createRedirectToHostRules") + } + err = s.fetchNumDataThreads() + if err != nil { + return errors.Wrap(err, "Error in fetchNumDataThreads") + } + err = s.fetchBufferConfig() + if err != nil { + return errors.Wrap(err, "Error in fetchBufferConfig") + } + err = s.cniHandler.CNIHandlerInit() + if err != nil { + return errors.Wrap(err, "Error in CNIHandlerInit") + } for { select { case <-t.Dying(): @@ -201,6 +290,10 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { common.SendEvent(common.CalicoVppEvent{ Type: common.BGPConfChanged, }) + case *model.CniPodAddEvent: + err = s.cniHandler.OnPodAdd(evt) + case *model.CniPodDelEvent: + s.cniHandler.OnPodDelete(evt) case common.CalicoVppEvent: /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { @@ -209,8 +302,13 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { if !ok { return fmt.Errorf("evt.New is not a (*common.NetworkDefinition) %v", evt.New) } + old, ok := evt.Old.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.Old is not a (*common.NetworkDefinition) %v", evt.New) + } s.cache.NetworkDefinitions[new.Name] = new s.cache.Networks[new.Vni] = new + s.cniHandler.OnNetAddedOrUpdated(old, new) case common.NetDeleted: netDef, ok := evt.Old.(*common.NetworkDefinition) if !ok { @@ -218,6 +316,7 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { } delete(s.cache.NetworkDefinitions, netDef.Name) delete(s.cache.Networks, netDef.Vni) + s.cniHandler.OnNetDeleted(netDef) case common.PodAdded: podSpec, ok := evt.New.(*model.LocalPodSpec) if !ok { diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go index 73c9b8459..d03b35ae2 100644 --- a/calico-vpp-agent/felix/felixconfig.go +++ b/calico-vpp-agent/felix/felixconfig.go @@ -83,6 +83,7 @@ func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { return nil } + s.cniHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) return nil diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go index 1ab72e925..31b327f36 100644 --- a/calico-vpp-agent/felix/ipam.go +++ b/calico-vpp-agent/felix/ipam.go @@ -49,6 +49,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil || err2 != nil { return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) } + s.cniHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, Old: ipamPoolCopy(oldIpamPool), @@ -63,6 +64,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil { return errors.Wrap(err, "error handling ipam add") } + s.cniHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, New: ipamPoolCopy(newIpamPool), @@ -92,6 +94,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { Old: ipamPoolCopy(oldIpamPool), New: nil, }) + s.cniHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) } else { s.log.Warnf("Deleting unknown ippool") return nil diff --git a/calico-vpp-agent/prometheus/prometheus.go b/calico-vpp-agent/prometheus/prometheus.go index 5b9c2ef71..826461c2f 100644 --- a/calico-vpp-agent/prometheus/prometheus.go +++ b/calico-vpp-agent/prometheus/prometheus.go @@ -31,7 +31,7 @@ import ( "go.fd.io/govpp/adapter/statsclient" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) diff --git a/calico-vpp-agent/prometheus/prometheus_test.go b/calico-vpp-agent/prometheus/prometheus_test.go index a9266f5d7..30aa89f18 100644 --- a/calico-vpp-agent/prometheus/prometheus_test.go +++ b/calico-vpp-agent/prometheus/prometheus_test.go @@ -30,7 +30,7 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" agentConf "github.com/projectcalico/vpp-dataplane/v3/config" diff --git a/calico-vpp-agent/routing/bgp_watcher.go b/calico-vpp-agent/routing/bgp_watcher.go index 7a9b79947..97978c9aa 100644 --- a/calico-vpp-agent/routing/bgp_watcher.go +++ b/calico-vpp-agent/routing/bgp_watcher.go @@ -25,8 +25,8 @@ import ( "gopkg.in/tomb.v2" calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/ip_types" diff --git a/calico-vpp-agent/services/service_handler.go b/calico-vpp-agent/services/service_handler.go index eedf92948..845ac9db5 100644 --- a/calico-vpp-agent/services/service_handler.go +++ b/calico-vpp-agent/services/service_handler.go @@ -20,12 +20,11 @@ import ( v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" - discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/util/intstr" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" diff --git a/calico-vpp-agent/services/service_server.go b/calico-vpp-agent/services/service_server.go index 445408dfd..6fa541315 100644 --- a/calico-vpp-agent/services/service_server.go +++ b/calico-vpp-agent/services/service_server.go @@ -34,8 +34,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" diff --git a/calico-vpp-agent/tests/mocks/ipam.go b/calico-vpp-agent/tests/mocks/ipam.go index a570b06b7..0b4e4458e 100644 --- a/calico-vpp-agent/tests/mocks/ipam.go +++ b/calico-vpp-agent/tests/mocks/ipam.go @@ -69,10 +69,6 @@ func (s *IpamCacheStub) WaitReady() { panic("not implemented") } -func (s *IpamCacheStub) IPNetNeedsSNAT(prefix *net.IPNet) bool { - return false -} - func (s *IpamCacheStub) AddPrefixIPPool(prefix *net.IPNet, ipPool *proto.IPAMPoolUpdate) { s.ipPools[prefix.String()] = ipPool } diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 7207d08f7..70d8d1c5b 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -37,12 +37,11 @@ import ( "github.com/containernetworking/plugins/pkg/ns" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/calico/libcalico-go/lib/options" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/multinet-monitor/multinettypes" @@ -84,9 +83,9 @@ var ( VPPContainerName string ) -func AssertTunInterfaceExistence(vpp *vpplink.VppLink, newPod *cniproto.AddRequest) uint32 { +func AssertTunInterfaceExistence(vpp *vpplink.VppLink, podSpec *model.LocalPodSpec) uint32 { ifSwIfIndex, err := vpp.SearchInterfaceWithTag( - InterfaceTagForLocalTunTunnel(newPod.InterfaceName, newPod.Netns)) + InterfaceTagForLocalTunTunnel(podSpec.InterfaceName, podSpec.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get interface at VPP's end") Expect(ifSwIfIndex).ToNot(Equal(vpplink.InvalidSwIfIndex), "No interface at VPP's end is found") diff --git a/calico-vpp-agent/watchers/cni_grpc.go b/calico-vpp-agent/watchers/cni_grpc.go new file mode 100644 index 000000000..6fee70a6e --- /dev/null +++ b/calico-vpp-agent/watchers/cni_grpc.go @@ -0,0 +1,111 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watchers + +import ( + "context" + gerrors "errors" + "net" + "os" + "syscall" + + "github.com/pkg/errors" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "gopkg.in/tomb.v2" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type CNIServer struct { + cniproto.UnimplementedCniDataplaneServer + log *logrus.Entry + grpcServer *grpc.Server + eventChan chan any +} + +// Serve runs the grpc server for the Calico CNI backend API +func NewCNIServer(eventChan chan any, log *logrus.Entry) *CNIServer { + return &CNIServer{ + log: log, + grpcServer: grpc.NewServer(), + eventChan: eventChan, + } +} + +func (s *CNIServer) ServeCNI(t *tomb.Tomb) error { + err := syscall.Unlink(config.CNIServerSocket) + if err != nil && !gerrors.Is(err, os.ErrNotExist) { + s.log.Warnf("unable to unlink cni server socket: %+v", err) + } + + defer func() { + err = syscall.Unlink(config.CNIServerSocket) + if err != nil { + s.log.Errorf("error cleaning up CNIServerSocket %s", err) + } + }() + + socketListener, err := net.Listen("unix", config.CNIServerSocket) + if err != nil { + return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket) + } + + cniproto.RegisterCniDataplaneServer(s.grpcServer, s) + + s.log.Infof("Serving CNI grpc") + err = s.grpcServer.Serve(socketListener) + s.log.Infof("CNI Server returned") + return err +} + +func (s *CNIServer) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) { + podSpecKey := model.LocalPodSpecKey(request.GetNetns(), request.GetInterfaceName()) + // Only try to delete the device if a namespace was passed in. + if request.GetNetns() == "" { + s.log.Debugf("no netns passed, skipping") + return &cniproto.DelReply{ + Successful: true, + }, nil + } + evt := model.NewCniPodDelEvent(podSpecKey) + s.eventChan <- evt + + return <-evt.Done, nil +} + +func (s *CNIServer) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { + /* We don't support request.GetDesiredHostInterfaceName() */ + podSpec, err := model.NewLocalPodSpecFromAdd(request) + if err != nil { + s.log.Errorf("Error parsing interface add request %v %v", request, err) + return &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + }, nil + } + + evt := model.NewCniPodAddEvent(podSpec) + s.eventChan <- evt + + return <-evt.Done, nil +} + +func (s *CNIServer) GracefulStop() { + s.grpcServer.GracefulStop() +} diff --git a/calico-vpp-agent/watchers/felix.go b/calico-vpp-agent/watchers/felix.go index 327cd5464..c80f5d341 100644 --- a/calico-vpp-agent/watchers/felix.go +++ b/calico-vpp-agent/watchers/felix.go @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Cisco Systems Inc. +// Copyright (C) 2026 Cisco Systems Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go index b3caa42a8..9c7af291a 100644 --- a/vpp-manager/vpp_runner.go +++ b/vpp-manager/vpp_runner.go @@ -33,8 +33,8 @@ import ( calicoopts "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/vishvananda/netlink" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/hooks" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/uplink" From 61f234a917ef16bfc56c64735b9eda4d800041df Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Tue, 2 Sep 2025 15:00:24 +0200 Subject: [PATCH 3/4] Split Connectivity into watcher/handler under felix This patch moves the Connectivity handlers in the main felix loop to allow lockless access to the cache. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 10 +- calico-vpp-agent/common/common.go | 18 - calico-vpp-agent/common/pubsub.go | 3 - .../connectivity/connectivity_server.go | 453 ------------------ calico-vpp-agent/felix/cache/cache.go | 15 +- .../{ => felix}/connectivity/connectivity.go | 38 -- .../connectivity/connectivity_handler.go | 318 ++++++++++++ .../{ => felix}/connectivity/flat.go | 11 +- .../{ => felix}/connectivity/ipip.go | 29 +- .../{ => felix}/connectivity/ipsec.go | 40 +- .../{ => felix}/connectivity/srv6.go | 44 +- .../{ => felix}/connectivity/vxlan.go | 42 +- .../{ => felix}/connectivity/wireguard.go | 186 +++---- calico-vpp-agent/felix/felix_server.go | 85 +++- calico-vpp-agent/felix/felix_server_test.go | 6 - calico-vpp-agent/felix/felixconfig.go | 9 +- calico-vpp-agent/felix/ipam.go | 8 +- .../{cni/cni_node_test.go => node_test.go} | 134 ++---- .../{cni/cni_pod_test.go => pod_test.go} | 2 +- .../felix/policies/hostmetadata.go | 22 +- .../felix/policies/policies_handler.go | 2 + calico-vpp-agent/testutils/testutils.go | 12 +- 22 files changed, 676 insertions(+), 811 deletions(-) delete mode 100644 calico-vpp-agent/connectivity/connectivity_server.go rename calico-vpp-agent/{ => felix}/connectivity/connectivity.go (60%) create mode 100644 calico-vpp-agent/felix/connectivity/connectivity_handler.go rename calico-vpp-agent/{ => felix}/connectivity/flat.go (90%) rename calico-vpp-agent/{ => felix}/connectivity/ipip.go (88%) rename calico-vpp-agent/{ => felix}/connectivity/ipsec.go (92%) rename calico-vpp-agent/{ => felix}/connectivity/srv6.go (90%) rename calico-vpp-agent/{ => felix}/connectivity/vxlan.go (88%) rename calico-vpp-agent/{ => felix}/connectivity/wireguard.go (70%) rename calico-vpp-agent/felix/{cni/cni_node_test.go => node_test.go} (89%) rename calico-vpp-agent/felix/{cni/cni_pod_test.go => pod_test.go} (99%) diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 8aa3d10b5..4f59c6f66 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -26,7 +26,6 @@ import ( apipb "github.com/osrg/gobgp/v3/api" bgpserver "github.com/osrg/gobgp/v3/pkg/server" "github.com/pkg/errors" - felixconfig "github.com/projectcalico/calico/felix/config" calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -35,7 +34,6 @@ import ( "k8s.io/client-go/rest" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/health" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/routing" @@ -158,7 +156,6 @@ func main() { if err != nil { log.Fatalf("could not install felix plugin: %s", err) } - connectivityServer := connectivity.NewConnectivityServer(vpp, felixServer, clientv3, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) /* Pubsub should now be registered */ @@ -185,14 +182,13 @@ func main() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - var felixConfig *felixconfig.Config var ourBGPSpec *common.LocalNodeSpec felixConfigReceived := false bgpSpecReceived := false for !felixConfigReceived || !bgpSpecReceived { select { - case felixConfig = <-felixServer.FelixConfigChan: + case <-felixServer.GotFelixConfig(): felixConfigReceived = true log.Info("FelixConfig received from calico pod") case ourBGPSpec = <-felixServer.GotOurNodeBGPchan(): @@ -216,7 +212,6 @@ func main() { log.Info("Felix configuration received") prefixWatcher.SetOurBGPSpec(ourBGPSpec) - connectivityServer.SetOurBGPSpec(ourBGPSpec) routingServer.SetOurBGPSpec(ourBGPSpec) serviceServer.SetOurBGPSpec(ourBGPSpec) localSIDWatcher.SetOurBGPSpec(ourBGPSpec) @@ -234,15 +229,12 @@ func main() { } } - connectivityServer.SetFelixConfig(felixConfig) - Go(routeWatcher.WatchRoutes) Go(linkWatcher.WatchLinks) Go(bgpConfigurationWatcher.WatchBGPConfiguration) Go(prefixWatcher.WatchPrefix) Go(peerWatcher.WatchBGPPeers) Go(bgpFilterWatcher.WatchBGPFilters) - Go(connectivityServer.ServeConnectivity) Go(routingServer.ServeRouting) Go(serviceServer.ServeService) Go(cniServer.ServeCNI) diff --git a/calico-vpp-agent/common/common.go b/calico-vpp-agent/common/common.go index 861296835..6f6724d40 100644 --- a/calico-vpp-agent/common/common.go +++ b/calico-vpp-agent/common/common.go @@ -557,24 +557,6 @@ func FormatBGPConfiguration(conf *calicov3.BGPConfigurationSpec) string { ) } -func FetchNDataThreads(vpp *vpplink.VppLink, log *logrus.Entry) int { - nVppWorkers, err := vpp.GetNumVPPWorkers() - if err != nil { - log.Panicf("Error getting number of VPP workers: %v", err) - } - nDataThreads := nVppWorkers - if config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread > 0 { - nDataThreads = nVppWorkers - config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread - if nDataThreads <= 0 { - log.Errorf("Couldn't fulfill request [crypto=%d total=%d]", config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread, nVppWorkers) - nDataThreads = nVppWorkers - } - log.Infof("Using ipsec workers [data=%d crypto=%d]", nDataThreads, nVppWorkers-nDataThreads) - - } - return nDataThreads -} - func CompareIPList(newIPList, oldIPList []net.IP) (added []net.IP, deleted []net.IP, changed bool) { oldIPListMap := make(map[string]bool) newIPListMap := make(map[string]bool) diff --git a/calico-vpp-agent/common/pubsub.go b/calico-vpp-agent/common/pubsub.go index 1ceb72484..2ac7b253e 100644 --- a/calico-vpp-agent/common/pubsub.go +++ b/calico-vpp-agent/common/pubsub.go @@ -27,7 +27,6 @@ const ( ChanSize = 500 PeerNodeStateChanged CalicoVppEventType = "PeerNodeStateChanged" - FelixConfChanged CalicoVppEventType = "FelixConfChanged" IpamConfChanged CalicoVppEventType = "IpamConfChanged" BGPConfChanged CalicoVppEventType = "BGPConfChanged" @@ -66,8 +65,6 @@ const ( IpamPoolUpdate CalicoVppEventType = "IpamPoolUpdate" IpamPoolRemove CalicoVppEventType = "IpamPoolRemove" - - WireguardPublicKeyChanged CalicoVppEventType = "WireguardPublicKeyChanged" ) var ( diff --git a/calico-vpp-agent/connectivity/connectivity_server.go b/calico-vpp-agent/connectivity/connectivity_server.go deleted file mode 100644 index ded4adb17..000000000 --- a/calico-vpp-agent/connectivity/connectivity_server.go +++ /dev/null @@ -1,453 +0,0 @@ -// Copyright (C) 2021 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package connectivity - -import ( - "fmt" - "net" - - "github.com/pkg/errors" - felixConfig "github.com/projectcalico/calico/felix/config" - "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" - calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" -) - -type ConnectivityServer struct { - log *logrus.Entry - - providers map[string]ConnectivityProvider - connectivityMap map[string]common.NodeConnectivity - felixServerIpam common.FelixServerIpam - Clientv3 calicov3cli.Interface - nodeBGPSpec *common.LocalNodeSpec - vpp *vpplink.VppLink - - felixConfig *felixConfig.Config - nodeByAddr map[string]common.LocalNodeSpec - - connectivityEventChan chan any - - networks map[uint32]common.NetworkDefinition -} - -type change uint8 - -const ( - AddChange change = 0 - DeleteChange change = 1 -) - -func (s *ConnectivityServer) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { - s.nodeBGPSpec = nodeBGPSpec -} - -func (s *ConnectivityServer) SetFelixConfig(felixConfig *felixConfig.Config) { - s.felixConfig = felixConfig -} - -func NewConnectivityServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, - clientv3 calicov3cli.Interface, log *logrus.Entry) *ConnectivityServer { - server := ConnectivityServer{ - log: log, - vpp: vpp, - felixServerIpam: felixServerIpam, - Clientv3: clientv3, - connectivityMap: make(map[string]common.NodeConnectivity), - connectivityEventChan: make(chan any, common.ChanSize), - nodeByAddr: make(map[string]common.LocalNodeSpec), - networks: make(map[uint32]common.NetworkDefinition), - } - - reg := common.RegisterHandler(server.connectivityEventChan, "connectivity server events") - reg.ExpectEvents( - common.NetAddedOrUpdated, - common.NetDeleted, - common.ConnectivityAdded, - common.ConnectivityDeleted, - common.PeerNodeStateChanged, - common.FelixConfChanged, - common.IpamConfChanged, - common.SRv6PolicyAdded, - common.SRv6PolicyDeleted, - common.WireguardPublicKeyChanged, - ) - - nDataThreads := common.FetchNDataThreads(vpp, log) - providerData := NewConnectivityProviderData(server.vpp, &server, log) - - server.providers = make(map[string]ConnectivityProvider) - server.providers[FLAT] = NewFlatL3Provider(providerData) - server.providers[IPIP] = NewIPIPProvider(providerData) - server.providers[IPSEC] = NewIPsecProvider(providerData, nDataThreads) - server.providers[VXLAN] = NewVXLanProvider(providerData) - server.providers[WIREGUARD] = NewWireguardProvider(providerData) - server.providers[SRv6] = NewSRv6Provider(providerData) - - return &server -} - -func (s *ConnectivityServer) GetNodeByIP(addr net.IP) *common.LocalNodeSpec { - ns, found := s.nodeByAddr[addr.String()] - if !found { - return nil - } - return &ns -} - -func (s *ConnectivityServer) GetNodeIPs() (ip4 *net.IP, ip6 *net.IP) { - ip4, ip6 = common.GetBGPSpecAddresses(s.nodeBGPSpec) - return ip4, ip6 -} - -func (s *ConnectivityServer) GetNodeIPNet(isv6 bool) *net.IPNet { - if s.nodeBGPSpec == nil { - return nil - } - ip4, ip6 := s.nodeBGPSpec.IPv4Address, s.nodeBGPSpec.IPv6Address - if isv6 { - return ip6 - } else { - return ip4 - } -} - -func (s *ConnectivityServer) updateAllIPConnectivity() { - for _, cn := range s.connectivityMap { - err := s.UpdateIPConnectivity(&cn, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while re-updating connectivity %s", err) - } - } -} - -func (s *ConnectivityServer) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) { - if node.IPv4Address != nil { - err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd) - if err != nil { - s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err) - } - } - if node.IPv6Address != nil { - err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd) - if err != nil { - s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err) - } - } -} - -func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error { - /** - * There might be leftover state in VPP in case we restarted - * so first check what is present */ - for _, provider := range s.providers { - provider.RescanState() - } - for { - select { - case <-t.Dying(): - s.log.Warn("Connectivity Server asked to stop") - return nil - case msg := <-s.connectivityEventChan: - /* Note: we will only receive events we ask for when registering the chan */ - evt, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch evt.Type { - case common.NetAddedOrUpdated: - new, ok := evt.New.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("evt.New is not a *common.NetworkDefinition %v", evt.New) - } - s.networks[new.Vni] = *new - case common.NetDeleted: - old, ok := evt.Old.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("evt.Old is not a *common.NetworkDefinition %v", evt.Old) - } - delete(s.networks, old.Vni) - case common.ConnectivityAdded: - new, ok := evt.New.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) - } - err := s.UpdateIPConnectivity(new, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while adding connectivity %s", err) - } - case common.ConnectivityDeleted: - old, ok := evt.Old.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) - } - err := s.UpdateIPConnectivity(old, true /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while deleting connectivity %s", err) - } - case common.WireguardPublicKeyChanged: - old, ok := evt.Old.(*common.NodeWireguardPublicKey) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeWireguardPublicKey %v", evt.Old) - } - new, ok := evt.New.(*common.NodeWireguardPublicKey) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeWireguardPublicKey %v", evt.New) - } - wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) - if !ok { - panic("Type is not WireguardProvider") - } - wgProvider.nodesToWGPublicKey[new.Name] = new.WireguardPublicKey - change := common.GetStringChangeType(old.WireguardPublicKey, new.WireguardPublicKey) - if change != common.ChangeSame { - s.log.Infof("connectivity(upd) WireguardPublicKey Changed (%s) %s->%s", old.Name, old.WireguardPublicKey, new.WireguardPublicKey) - s.updateAllIPConnectivity() - } - case common.PeerNodeStateChanged: - old, ok := evt.Old.(*common.LocalNodeSpec) - if !ok { - s.log.Errorf("evt.Old is not a *common.LocalNodeSpec %v", evt.Old) - } - new, ok := evt.New.(*common.LocalNodeSpec) - if !ok { - s.log.Errorf("evt.New is not a *common.LocalNodeSpec %v", evt.New) - } - if old != nil { - if old.IPv4Address != nil { - delete(s.nodeByAddr, old.IPv4Address.IP.String()) - } - if old.IPv6Address != nil { - delete(s.nodeByAddr, old.IPv6Address.IP.String()) - } - s.configureRemoteNodeSnat(old, false /* isAdd */) - } - if new != nil { - if new.IPv4Address != nil { - s.nodeByAddr[new.IPv4Address.IP.String()] = *new - } - if new.IPv6Address != nil { - s.nodeByAddr[new.IPv6Address.IP.String()] = *new - } - s.configureRemoteNodeSnat(new, true /* isAdd */) - } - case common.FelixConfChanged: - old, ok := evt.Old.(*felixConfig.Config) - if !ok { - s.log.Errorf("evt.Old is not a *felixConfig.Config %v", evt.Old) - } - new, ok := evt.New.(*felixConfig.Config) - if !ok { - s.log.Errorf("evt.Old is not a *felixConfig.Config %v", evt.New) - } - if new == nil || old == nil { - /* First/last update, do nothing more */ - continue - } - s.felixConfig = new - if old.WireguardEnabled != new.WireguardEnabled { - s.log.Infof("connectivity(upd) WireguardEnabled Changed %t->%t", old.WireguardEnabled, new.WireguardEnabled) - s.providers[WIREGUARD].EnableDisable(new.WireguardEnabled) - s.updateAllIPConnectivity() - } else if old.WireguardListeningPort != new.WireguardListeningPort { - s.log.Warnf("connectivity(upd) WireguardListeningPort Changed [NOT IMPLEMENTED]") - } - case common.IpamConfChanged: - s.log.Infof("connectivity(upd) ipamConf Changed") - s.updateAllIPConnectivity() - case common.SRv6PolicyAdded: - new, ok := evt.New.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) - } - err := s.UpdateSRv6Policy(new, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while adding SRv6 Policy %s", err) - } - case common.SRv6PolicyDeleted: - old, ok := evt.Old.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) - } - err := s.UpdateSRv6Policy(old, true /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while deleting SRv6 Policy %s", err) - } - } - } - } -} - -func (s *ConnectivityServer) UpdateSRv6Policy(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { - s.log.Infof("updateSRv6Policy") - providerType := SRv6 - if IsWithdraw { - err = s.providers[providerType].DelConnectivity(cn) - } else { - err = s.providers[providerType].AddConnectivity(cn) - } - return err -} - -func (s *ConnectivityServer) getProviderType(cn *common.NodeConnectivity) (string, error) { - // use vxlan tunnel if secondary network, no need for ippool - if cn.Vni != 0 { - return VXLAN, nil - } - ipPool := s.felixServerIpam.GetPrefixIPPool(&cn.Dst) - s.log.Debugf("IPPool for route %s: %+v", cn.String(), ipPool) - if *config.GetCalicoVppFeatureGates().SRv6Enabled { - return SRv6, nil - } - if ipPool == nil { - return FLAT, nil - } - if ipPool.IpipMode == encap.Always { - if s.providers[IPSEC].Enabled(cn) { - return IPSEC, nil - } else if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } else { - return IPIP, nil - } - } - nodeIPNet := s.GetNodeIPNet(vpplink.IsIP6(cn.Dst.IP)) - if ipPool.IpipMode == encap.CrossSubnet { - if nodeIPNet == nil { - return FLAT, fmt.Errorf("missing node IPnet") - } - if !nodeIPNet.Contains(cn.NextHop) { - if s.providers[IPSEC].Enabled(cn) { - return IPSEC, nil - } else if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } else { - return IPIP, nil - } - } - } - if ipPool.VxlanMode == encap.Always { - if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } - return VXLAN, nil - } - if ipPool.VxlanMode == encap.CrossSubnet { - if nodeIPNet == nil { - return FLAT, fmt.Errorf("missing node IPnet") - } - if !nodeIPNet.Contains(cn.NextHop) { - if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } - return VXLAN, nil - } - } - return FLAT, nil -} - -func (s *ConnectivityServer) UpdateIPConnectivity(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { - var providerType string - if IsWithdraw { - oldCn, found := s.connectivityMap[cn.String()] - if !found { - providerType, err = s.getProviderType(cn) - if err != nil { - return errors.Wrap(err, "getting provider failed") - } - s.log.Infof("connectivity(del) Didnt find provider in map, trying providerType=%s", providerType) - } else { - providerType = oldCn.ResolvedProvider - delete(s.connectivityMap, oldCn.String()) - s.log.Infof("connectivity(del) path providerType=%s cn=%s", providerType, oldCn.String()) - } - return s.providers[providerType].DelConnectivity(cn) - } else { - providerType, err = s.getProviderType(cn) - if err != nil { - return errors.Wrap(err, "getting provider failed") - } - oldCn, found := s.connectivityMap[cn.String()] - if found { - oldProviderType := oldCn.ResolvedProvider - if oldProviderType != providerType { - s.log.Infof("connectivity(upd) provider Change providerType=%s->%s cn=%s", oldProviderType, providerType, cn.String()) - err := s.providers[oldProviderType].DelConnectivity(cn) - if err != nil { - s.log.Errorf("Error del connectivity when changing provider %s->%s : %s", oldProviderType, providerType, err) - } - cn.ResolvedProvider = providerType - s.connectivityMap[cn.String()] = *cn - return s.providers[providerType].AddConnectivity(cn) - } else { - s.log.Infof("connectivity(same) path providerType=%s cn=%s", providerType, cn.String()) - return s.providers[providerType].AddConnectivity(cn) - } - } else { - s.log.Infof("connectivity(add) path providerType=%s cn=%s", providerType, cn.String()) - cn.ResolvedProvider = providerType - s.connectivityMap[cn.String()] = *cn - return s.providers[providerType].AddConnectivity(cn) - } - } -} - -// ForceRescanState forces to rescan VPP state (ConnectivityProvider.RescanState()) for initialized -// ConnectivityProvider of given type. -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceRescanState(providerType string) (err error) { - provider, found := s.providers[providerType] - if !found { - return fmt.Errorf("can't find connectivity provider of type %s", providerType) - } - provider.RescanState() - return nil -} - -// ForceProviderEnableDisable force to enable/disable specific connectivity provider. -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceProviderEnableDisable(providerType string, enable bool) (err error) { - provider, found := s.providers[providerType] - if !found { - return fmt.Errorf("can't find connectivity provider of type %s", providerType) - } - provider.EnableDisable(enable) - return nil -} - -// TODO get rid (if possible) of all this "Force" methods by refactor the test code -// (run the ConnectivityServer.ServeConnectivity(...) function and send into it events with common.SendEvent(...)) - -// ForceNodeAddition will add other node information as provided by calico configuration -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceNodeAddition(newNode common.LocalNodeSpec, newNodeIP net.IP) { - s.nodeByAddr[newNodeIP.String()] = newNode -} - -// ForceWGPublicKeyAddition will add other node information as provided by calico configuration -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceWGPublicKeyAddition(newNode string, wgPublicKey string) { - wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) - if !ok { - panic("Type is not WireguardProvider") - } - wgProvider.nodesToWGPublicKey[newNode] = wgPublicKey -} diff --git a/calico-vpp-agent/felix/cache/cache.go b/calico-vpp-agent/felix/cache/cache.go index d39f41e8b..0d6ab7753 100644 --- a/calico-vpp-agent/felix/cache/cache.go +++ b/calico-vpp-agent/felix/cache/cache.go @@ -32,7 +32,7 @@ type Cache struct { log *logrus.Entry FelixConfig *felixConfig.Config - NodeByAddr map[string]common.LocalNodeSpec + NodeByAddr map[string]*common.LocalNodeSpec Networks map[uint32]*common.NetworkDefinition NetworkDefinitions map[string]*common.NetworkDefinition IPPoolMap map[string]*proto.IPAMPool @@ -46,7 +46,7 @@ type Cache struct { func NewCache(log *logrus.Entry) *Cache { return &Cache{ log: log, - NodeByAddr: make(map[string]common.LocalNodeSpec), + NodeByAddr: make(map[string]*common.LocalNodeSpec), FelixConfig: felixConfig.New(), Networks: make(map[uint32]*common.NetworkDefinition), NetworkDefinitions: make(map[string]*common.NetworkDefinition), @@ -110,3 +110,14 @@ func (cache *Cache) GetNodeIP6() *net.IP { } return nil } + +func (cache *Cache) GetNodeIPNet(isv6 bool) *net.IPNet { + if spec, found := cache.NodeStatesByName[*config.NodeName]; found { + if isv6 { + return spec.IPv6Address + } else { + return spec.IPv4Address + } + } + return nil +} diff --git a/calico-vpp-agent/connectivity/connectivity.go b/calico-vpp-agent/felix/connectivity/connectivity.go similarity index 60% rename from calico-vpp-agent/connectivity/connectivity.go rename to calico-vpp-agent/felix/connectivity/connectivity.go index 09511a4e2..277aad45a 100644 --- a/calico-vpp-agent/connectivity/connectivity.go +++ b/calico-vpp-agent/felix/connectivity/connectivity.go @@ -17,14 +17,7 @@ package connectivity import ( - "net" - - felixConfig "github.com/projectcalico/calico/felix/config" - calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) const ( @@ -36,12 +29,6 @@ const ( SRv6 = "srv6" ) -type ConnectivityProviderData struct { - vpp *vpplink.VppLink - log *logrus.Entry - server *ConnectivityServer -} - // ConnectivityProvider configures VPP to have proper connectivity to other K8s nodes. // Different implementations can connect VPP with VPP in other K8s node by using different networking // technologies (VXLAN, SRv6,...). @@ -54,28 +41,3 @@ type ConnectivityProvider interface { Enabled(cn *common.NodeConnectivity) bool EnableDisable(isEnable bool) } - -func (p *ConnectivityProviderData) GetNodeByIP(addr net.IP) *common.LocalNodeSpec { - return p.server.GetNodeByIP(addr) -} -func (p *ConnectivityProviderData) GetNodeIPs() (*net.IP, *net.IP) { - return p.server.GetNodeIPs() -} -func (p *ConnectivityProviderData) Clientv3() calicov3cli.Interface { - return p.server.Clientv3 -} -func (p *ConnectivityProviderData) GetFelixConfig() *felixConfig.Config { - return p.server.felixConfig -} - -func NewConnectivityProviderData( - vpp *vpplink.VppLink, - server *ConnectivityServer, - log *logrus.Entry, -) *ConnectivityProviderData { - return &ConnectivityProviderData{ - vpp: vpp, - log: log, - server: server, - } -} diff --git a/calico-vpp-agent/felix/connectivity/connectivity_handler.go b/calico-vpp-agent/felix/connectivity/connectivity_handler.go new file mode 100644 index 000000000..2b70d9d57 --- /dev/null +++ b/calico-vpp-agent/felix/connectivity/connectivity_handler.go @@ -0,0 +1,318 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectivity + +import ( + "fmt" + "net" + "sync" + + "github.com/pkg/errors" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" +) + +type ConnectivityHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + providers map[string]ConnectivityProvider + connectivityMap map[string]common.NodeConnectivity + nodeByWGPublicKey map[string]string + + connectivityHandlerInitOnce sync.Once +} + +func NewConnectivityHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calicov3cli.Interface, log *logrus.Entry) *ConnectivityHandler { + return &ConnectivityHandler{ + log: log, + vpp: vpp, + cache: cache, + connectivityMap: make(map[string]common.NodeConnectivity), + providers: map[string]ConnectivityProvider{ + FLAT: NewFlatL3Provider(vpp, log), + IPIP: NewIPIPProvider(vpp, cache, log), + IPSEC: NewIPsecProvider(vpp, cache, log), + VXLAN: NewVXLanProvider(vpp, cache, log), + WIREGUARD: NewWireguardProvider(vpp, clientv3, cache, log), + SRv6: NewSRv6Provider(vpp, clientv3, cache, log), + }, + nodeByWGPublicKey: make(map[string]string), + } +} + +type change uint8 + +const ( + AddChange change = 0 + DeleteChange change = 1 +) + +func (s *ConnectivityHandler) UpdateAllIPConnectivity() { + s.log.Infof("connectivity(upd) ipamConf Changed") + for _, cn := range s.connectivityMap { + err := s.UpdateIPConnectivity(&cn, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while re-updating connectivity %s", err) + } + } +} + +func (s *ConnectivityHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if new == nil || old == nil { + // First/last update, do nothing more + return + } + if old.WireguardEnabled != new.WireguardEnabled { + s.log.Infof("connectivity(upd) WireguardEnabled Changed %t->%t", old.WireguardEnabled, new.WireguardEnabled) + s.providers[WIREGUARD].EnableDisable(new.WireguardEnabled) + s.UpdateAllIPConnectivity() + } else if old.WireguardListeningPort != new.WireguardListeningPort { + s.log.Warnf("connectivity(upd) WireguardListeningPort Changed [NOT IMPLEMENTED]") + } +} + +func (s *ConnectivityHandler) OnIpamConfChanged(old, new *proto.IPAMPool) { + s.UpdateAllIPConnectivity() +} + +func (s *ConnectivityHandler) OnHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Update) (err error) { + localNodeSpec, err := common.NewLocalNodeSpec(msg) + if err != nil { + return errors.Wrapf(err, "OnHostMetadataV4V6Update errored") + } + if localNodeSpec.Name == *config.NodeName && + (localNodeSpec.IPv4Address != nil || localNodeSpec.IPv6Address != nil) { + s.connectivityHandlerInitOnce.Do(func() { + // this is needed as connectivity does not support support + // starting without knowing node IPs + // TODO: we should properly implement the node address update + err = s.connectivityHandlerInit() + if err != nil { + s.log.WithError(err).Errorf("Error in connectivityHandlerInit") + } + }) + } + return nil +} + +func (s *ConnectivityHandler) UpdateSRv6Policy(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { + s.log.Infof("updateSRv6Policy") + providerType := SRv6 + if IsWithdraw { + err = s.providers[providerType].DelConnectivity(cn) + } else { + err = s.providers[providerType].AddConnectivity(cn) + } + return err +} + +func (s *ConnectivityHandler) getProviderType(cn *common.NodeConnectivity) (string, error) { + // use vxlan tunnel if secondary network, no need for ippool + if cn.Vni != 0 { + return VXLAN, nil + } + ipPool := s.cache.GetPrefixIPPool(&cn.Dst) + s.log.Debugf("IPPool for route %s: %+v", cn.String(), ipPool) + if isSRv6Enabled() { + return SRv6, nil + } + if ipPool == nil { + return FLAT, nil + } + if ipPool.IpipMode == encap.Always { + if s.providers[IPSEC].Enabled(cn) { + return IPSEC, nil + } else if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } else { + return IPIP, nil + } + } + nodeIPNet := s.cache.GetNodeIPNet(vpplink.IsIP6(cn.Dst.IP)) + if ipPool.IpipMode == encap.CrossSubnet { + if nodeIPNet == nil { + return FLAT, fmt.Errorf("missing node IPnet") + } + if !nodeIPNet.Contains(cn.NextHop) { + if s.providers[IPSEC].Enabled(cn) { + return IPSEC, nil + } else if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } else { + return IPIP, nil + } + } + } + if ipPool.VxlanMode == encap.Always { + if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } + return VXLAN, nil + } + if ipPool.VxlanMode == encap.CrossSubnet { + if nodeIPNet == nil { + return FLAT, fmt.Errorf("missing node IPnet") + } + if !nodeIPNet.Contains(cn.NextHop) { + if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } + return VXLAN, nil + } + } + return FLAT, nil +} + +func (s *ConnectivityHandler) UpdateIPConnectivity(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { + var providerType string + if IsWithdraw { + oldCn, found := s.connectivityMap[cn.String()] + if !found { + providerType, err = s.getProviderType(cn) + if err != nil { + return errors.Wrap(err, "getting provider failed") + } + s.log.Infof("connectivity(del) Didnt find provider in map, trying providerType=%s", providerType) + } else { + providerType = oldCn.ResolvedProvider + delete(s.connectivityMap, oldCn.String()) + s.log.Infof("connectivity(del) path providerType=%s cn=%s", providerType, oldCn.String()) + } + return s.providers[providerType].DelConnectivity(cn) + } else { + providerType, err = s.getProviderType(cn) + if err != nil { + return errors.Wrap(err, "getting provider failed") + } + oldCn, found := s.connectivityMap[cn.String()] + if found { + oldProviderType := oldCn.ResolvedProvider + if oldProviderType != providerType { + s.log.Infof("connectivity(upd) provider Change providerType=%s->%s cn=%s", oldProviderType, providerType, cn.String()) + err := s.providers[oldProviderType].DelConnectivity(cn) + if err != nil { + s.log.Errorf("Error del connectivity when changing provider %s->%s : %s", oldProviderType, providerType, err) + } + cn.ResolvedProvider = providerType + s.connectivityMap[cn.String()] = *cn + return s.providers[providerType].AddConnectivity(cn) + } else { + s.log.Infof("connectivity(same) path providerType=%s cn=%s", providerType, cn.String()) + return s.providers[providerType].AddConnectivity(cn) + } + } else { + s.log.Infof("connectivity(add) path providerType=%s cn=%s", providerType, cn.String()) + cn.ResolvedProvider = providerType + s.connectivityMap[cn.String()] = *cn + return s.providers[providerType].AddConnectivity(cn) + } + } +} + +// ForceRescanState forces to rescan VPP state (ConnectivityProvider.RescanState()) for initialized +// ConnectivityProvider of given type. +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceRescanState(providerType string) (err error) { + provider, found := s.providers[providerType] + if !found { + return fmt.Errorf("can't find connectivity provider of type %s", providerType) + } + provider.RescanState() + return nil +} + +// ForceProviderEnableDisable force to enable/disable specific connectivity provider. +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceProviderEnableDisable(providerType string, enable bool) (err error) { + provider, found := s.providers[providerType] + if !found { + return fmt.Errorf("can't find connectivity provider of type %s", providerType) + } + provider.EnableDisable(enable) + return nil +} + +// TODO get rid (if possible) of all this "Force" methods by refactor the test code +// (run the Server.ServeConnectivity(...) function and send into it events with common.SendEvent(...)) + +// ForceNodeAddition will add other node information as provided by calico configuration +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceNodeAddition(newNode common.LocalNodeSpec, newNodeIP net.IP) { + s.cache.NodeByAddr[newNodeIP.String()] = &newNode +} + +// ForceWGPublicKeyAddition will add other node information as provided by calico configuration +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceWGPublicKeyAddition(newNode string, wgPublicKey string) { + wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) + if !ok { + panic("Type is not WireguardProvider") + } + wgProvider.NodesToWGPublicKey[newNode] = wgPublicKey +} + +func (s *ConnectivityHandler) OnWireguardEndpointUpdate(msg *proto.WireguardEndpointUpdate) (err error) { + s.log.Infof("Received wireguard public key %+v", msg) + var old *common.NodeWireguardPublicKey + _, ok := s.nodeByWGPublicKey[msg.Hostname] + if ok { + old = &common.NodeWireguardPublicKey{ + Name: msg.Hostname, + WireguardPublicKey: s.nodeByWGPublicKey[msg.Hostname], + } + } else { + old = &common.NodeWireguardPublicKey{Name: msg.Hostname} + } + new := &common.NodeWireguardPublicKey{ + Name: msg.Hostname, + WireguardPublicKey: msg.PublicKey, + } + + wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) + if !ok { + panic("Type is not WireguardProvider") + } + wgProvider.NodesToWGPublicKey[new.Name] = new.WireguardPublicKey + change := common.GetStringChangeType(old.WireguardPublicKey, new.WireguardPublicKey) + if change != common.ChangeSame { + s.log.Infof("connectivity(upd) WireguardPublicKey Changed (%s) %s->%s", old.Name, old.WireguardPublicKey, new.WireguardPublicKey) + s.UpdateAllIPConnectivity() + } + return nil +} + +func (s *ConnectivityHandler) OnWireguardEndpointRemove(msg *proto.WireguardEndpointRemove) (err error) { + return nil +} + +func (s *ConnectivityHandler) connectivityHandlerInit() error { + // There might be leftover state in VPP in case we + // restarted so first check what is present + for _, provider := range s.providers { + provider.RescanState() + } + return nil +} diff --git a/calico-vpp-agent/connectivity/flat.go b/calico-vpp-agent/felix/connectivity/flat.go similarity index 90% rename from calico-vpp-agent/connectivity/flat.go rename to calico-vpp-agent/felix/connectivity/flat.go index ccd8d4b23..53233deeb 100644 --- a/calico-vpp-agent/connectivity/flat.go +++ b/calico-vpp-agent/felix/connectivity/flat.go @@ -19,6 +19,7 @@ import ( "net" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -26,7 +27,8 @@ import ( ) type FlatL3Provider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry } func getRoutePaths(addr net.IP) []types.RoutePath { @@ -48,8 +50,11 @@ func (p *FlatL3Provider) Enabled(cn *common.NodeConnectivity) bool { return true } -func NewFlatL3Provider(d *ConnectivityProviderData) *FlatL3Provider { - return &FlatL3Provider{d} +func NewFlatL3Provider(vpp *vpplink.VppLink, log *logrus.Entry) *FlatL3Provider { + return &FlatL3Provider{ + vpp: vpp, + log: log, + } } func (p *FlatL3Provider) AddConnectivity(cn *common.NodeConnectivity) error { diff --git a/calico-vpp-agent/connectivity/ipip.go b/calico-vpp-agent/felix/connectivity/ipip.go similarity index 88% rename from calico-vpp-agent/connectivity/ipip.go rename to calico-vpp-agent/felix/connectivity/ipip.go index 8fd270471..f05009229 100644 --- a/calico-vpp-agent/connectivity/ipip.go +++ b/calico-vpp-agent/felix/connectivity/ipip.go @@ -21,20 +21,30 @@ import ( "github.com/pkg/errors" vpptypes "github.com/calico-vpp/vpplink/api/v0" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type IpipProvider struct { - *ConnectivityProviderData ipipIfs map[string]*vpptypes.IPIPTunnel ipipRoutes map[uint32]map[string]bool + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache } -func NewIPIPProvider(d *ConnectivityProviderData) *IpipProvider { - return &IpipProvider{d, make(map[string]*vpptypes.IPIPTunnel), make(map[uint32]map[string]bool)} +func NewIPIPProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *IpipProvider { + return &IpipProvider{ + vpp: vpp, + log: log, + cache: cache, + ipipIfs: make(map[string]*vpptypes.IPIPTunnel), + ipipRoutes: make(map[uint32]map[string]bool), + } } func (p *IpipProvider) EnableDisable(isEnable bool) { @@ -52,9 +62,9 @@ func (p *IpipProvider) RescanState() { p.log.Errorf("Error listing ipip tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() for _, tunnel := range tunnels { - if (ip4 != nil && tunnel.Src.Equal(*ip4)) || (ip6 != nil && tunnel.Src.Equal(*ip6)) { + if (p.cache.GetNodeIP4() != nil && tunnel.Src.Equal(*p.cache.GetNodeIP4())) || + (p.cache.GetNodeIP6() != nil && tunnel.Src.Equal(*p.cache.GetNodeIP6())) { p.log.Infof("Found existing tunnel: %s", tunnel) p.ipipIfs[tunnel.Dst.String()] = tunnel } @@ -98,11 +108,10 @@ func (p *IpipProvider) AddConnectivity(cn *common.NodeConnectivity) error { tunnel = &vpptypes.IPIPTunnel{ Dst: cn.NextHop, } - ip4, ip6 := p.server.GetNodeIPs() - if vpplink.IsIP6(cn.NextHop) && ip6 != nil { - tunnel.Src = *ip6 - } else if !vpplink.IsIP6(cn.NextHop) && ip4 != nil { - tunnel.Src = *ip4 + if vpplink.IsIP6(cn.NextHop) && p.cache.GetNodeIP6() != nil { + tunnel.Src = *p.cache.GetNodeIP6() + } else if !vpplink.IsIP6(cn.NextHop) && p.cache.GetNodeIP4() != nil { + tunnel.Src = *p.cache.GetNodeIP4() } else { return fmt.Errorf("missing node address") } diff --git a/calico-vpp-agent/connectivity/ipsec.go b/calico-vpp-agent/felix/connectivity/ipsec.go similarity index 92% rename from calico-vpp-agent/connectivity/ipsec.go rename to calico-vpp-agent/felix/connectivity/ipsec.go index c6e4ffc81..1908ae9de 100644 --- a/calico-vpp-agent/connectivity/ipsec.go +++ b/calico-vpp-agent/felix/connectivity/ipsec.go @@ -24,8 +24,10 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -54,10 +56,11 @@ func (tunnel *IpsecTunnel) IsInitiator() bool { } type IpsecProvider struct { - *ConnectivityProviderData - ipsecIfs map[string][]IpsecTunnel - ipsecRoutes map[string]map[string]bool - nonCryptoThreads int + ipsecIfs map[string][]IpsecTunnel + ipsecRoutes map[string]map[string]bool + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache } func (p *IpsecProvider) EnableDisable(isEnable bool) { @@ -81,9 +84,8 @@ func (p *IpsecProvider) RescanState() { for _, profile := range profiles { pmap[profile.Name] = true } - ip4, ip6 := p.server.GetNodeIPs() for _, tunnel := range tunnels { - if (ip4 != nil && tunnel.Src.Equal(*ip4)) || (ip6 != nil && tunnel.Src.Equal(*ip6)) { + if (p.cache.GetNodeIP4() != nil && tunnel.Src.Equal(*p.cache.GetNodeIP4())) || (p.cache.GetNodeIP6() != nil && tunnel.Src.Equal(*p.cache.GetNodeIP6())) { ipsecTunnel := NewIpsecTunnel(tunnel) if _, found := pmap[ipsecTunnel.Profile()]; found { p.ipsecIfs[ipsecTunnel.Dst.String()] = append(p.ipsecIfs[ipsecTunnel.Dst.String()], *ipsecTunnel) @@ -122,10 +124,10 @@ func (p *IpsecProvider) RescanState() { p.log.Errorf("SetIPsecAsyncMode error %s", err) } - p.log.Infof("Using async workers for ipsec, nonCryptoThreads=%d", p.nonCryptoThreads) - // setting first p.nonCryptoThreads threads to not be used for Crypto calculation (-> other packet processing) + p.log.Infof("Using async workers for ipsec, NumDataThreads=%d", p.cache.NumDataThreads) + // setting first p.cache.NumDataThreads threads to not be used for Crypto calculation (-> other packet processing) // and let the remaining threads handle crypto operations - for i := 0; i < p.nonCryptoThreads; i++ { + for i := 0; i < p.cache.NumDataThreads; i++ { err = p.vpp.SetCryptoWorker(uint32(i), false) if err != nil { p.log.Errorf("SetCryptoWorker error %s", err) @@ -134,12 +136,13 @@ func (p *IpsecProvider) RescanState() { } } -func NewIPsecProvider(d *ConnectivityProviderData, nonCryptoThreads int) *IpsecProvider { +func NewIPsecProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *IpsecProvider { return &IpsecProvider{ - ConnectivityProviderData: d, - ipsecIfs: make(map[string][]IpsecTunnel), - ipsecRoutes: make(map[string]map[string]bool), - nonCryptoThreads: nonCryptoThreads, + vpp: vpp, + log: log, + cache: cache, + ipsecIfs: make(map[string][]IpsecTunnel), + ipsecRoutes: make(map[string]map[string]bool), } } @@ -329,8 +332,8 @@ func (p *IpsecProvider) forceOtherNodeIP4(addr net.IP) (ip4 net.IP, err error) { if !vpplink.IsIP6(addr) { return addr, nil } - otherNode := p.GetNodeByIP(addr) - if otherNode == nil { + otherNode, found := p.cache.NodeByAddr[addr.String()] + if !found { return nil, fmt.Errorf("didnt find an ip4 for ip %s", addr.String()) } var nodeIP net.IP @@ -351,8 +354,7 @@ func (p *IpsecProvider) AddConnectivity(cn *common.NodeConnectivity) (err error) return errors.Wrap(err, "Ipsec v6 config failed") } /* IP6 is not yet supported by ikev2 */ - nodeIP4, _ := p.server.GetNodeIPs() - if nodeIP4 == nil { + if p.cache.GetNodeIP4() == nil { return fmt.Errorf("no ip4 node address found") } @@ -360,7 +362,7 @@ func (p *IpsecProvider) AddConnectivity(cn *common.NodeConnectivity) (err error) _, found := p.ipsecIfs[cn.NextHop.String()] if !found { - tunnelSpecs := p.getIPSECTunnelSpecs(nodeIP4, &cn.NextHop) + tunnelSpecs := p.getIPSECTunnelSpecs(p.cache.GetNodeIP4(), &cn.NextHop) for _, tunnelSpec := range tunnelSpecs { err = p.createIPSECTunnel(&tunnelSpec, *config.IPSecIkev2Psk, stack) if err != nil { diff --git a/calico-vpp-agent/connectivity/srv6.go b/calico-vpp-agent/felix/connectivity/srv6.go similarity index 90% rename from calico-vpp-agent/connectivity/srv6.go rename to calico-vpp-agent/felix/connectivity/srv6.go index f15a81bf2..47062bdb7 100644 --- a/calico-vpp-agent/connectivity/srv6.go +++ b/calico-vpp-agent/felix/connectivity/srv6.go @@ -6,9 +6,12 @@ import ( "net" "github.com/pkg/errors" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -32,8 +35,10 @@ type NodeToPolicies struct { // SRv6Provider is node connectivity provider that uses segment routing over IPv6 (SRv6) to connect the nodes // For more info about SRv6, see https://datatracker.ietf.org/doc/html/rfc8986. type SRv6Provider struct { - *ConnectivityProviderData - + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache + clientv3 calicov3cli.Interface // nodePrefixes is internal data holder for information from common.NodeConnectivity data // from common.ConnectivityAdded event nodePrefixes map[string]*NodeToPrefixes @@ -46,9 +51,23 @@ type SRv6Provider struct { localSidIPPool net.IPNet } -func NewSRv6Provider(d *ConnectivityProviderData) *SRv6Provider { - p := &SRv6Provider{d, make(map[string]*NodeToPrefixes), make(map[string]*NodeToPolicies), net.IPNet{}, net.IPNet{}} - if *config.GetCalicoVppFeatureGates().SRv6Enabled { +func isSRv6Enabled() bool { + enabled := config.GetCalicoVppFeatureGates().SRv6Enabled + return enabled != nil && *enabled +} + +func NewSRv6Provider(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, cache *cache.Cache, log *logrus.Entry) *SRv6Provider { + p := &SRv6Provider{ + vpp: vpp, + log: log, + cache: cache, + clientv3: clientv3, + nodePrefixes: make(map[string]*NodeToPrefixes), + nodePolices: make(map[string]*NodeToPolicies), + policyIPPool: net.IPNet{}, + localSidIPPool: net.IPNet{}, + } + if isSRv6Enabled() { p.localSidIPPool = cnet.MustParseNetwork(config.GetCalicoVppSrv6().LocalsidPool).IPNet p.policyIPPool = cnet.MustParseNetwork(config.GetCalicoVppSrv6().PolicyPool).IPNet } @@ -65,7 +84,7 @@ func (p *SRv6Provider) EnableDisable(isEnable bool) { } func (p *SRv6Provider) Enabled(cn *common.NodeConnectivity) bool { - return *config.GetCalicoVppFeatureGates().SRv6Enabled + return isSRv6Enabled() } // RescanState recreates(if missing in VPP) the static parts of the SRv6 tunneling on this node: @@ -74,7 +93,7 @@ func (p *SRv6Provider) Enabled(cn *common.NodeConnectivity) bool { func (p *SRv6Provider) RescanState() { p.log.Infof("SRv6Provider RescanState") - if !*config.GetCalicoVppFeatureGates().SRv6Enabled { + if !isSRv6Enabled() { return } @@ -269,15 +288,14 @@ func (p *SRv6Provider) getPolicyNode(nodeip string, behavior types.SrBehavior) ( func (p *SRv6Provider) setEncapSource() (err error) { p.log.Infof("SRv6Provider setEncapSource") - _, nodeIP6 := p.GetNodeIPs() - if nodeIP6 == nil { + if p.cache.GetNodeIP6() == nil { return fmt.Errorf("no ip6 found for node") } - if err = p.vpp.SetEncapSource(*nodeIP6); err != nil { + if err = p.vpp.SetEncapSource(*p.cache.GetNodeIP6()); err != nil { p.log.Errorf("SRv6Provider setEncapSource: %v", err) return errors.Wrapf(err, "SRv6Provider setEncapSource") } - p.log.Debugf("SRv6Provider setEncapSource with IP6 %s", nodeIP6.String()) + p.log.Debugf("SRv6Provider setEncapSource with IP6 %s", p.cache.GetNodeIP6().String()) return err } @@ -349,14 +367,14 @@ func (p *SRv6Provider) setEndDT(typeDT int) (newLocalSid *types.SrLocalsid, err } func (p *SRv6Provider) getSidFromPool(poolName string) (newSidAddr ip_types.IP6Address, err error) { - ippool, err := p.Clientv3().IPPools().Get(context.Background(), poolName, options.GetOptions{}) + ippool, err := p.clientv3.IPPools().Get(context.Background(), poolName, options.GetOptions{}) if err != nil || ippool == nil { p.log.Infof("SRv6Provider Error assigning ip LocalSid") return newSidAddr, errors.Wrapf(err, "SRv6Provider Error getSidFromPool") } poolIPNet := []cnet.IPNet{cnet.MustParseNetwork(ippool.Spec.CIDR)} - _, newSids, err := p.Clientv3().IPAM().AutoAssign(context.Background(), ipam.AutoAssignArgs{ + _, newSids, err := p.clientv3.IPAM().AutoAssign(context.Background(), ipam.AutoAssignArgs{ Num6: 1, IPv6Pools: poolIPNet, IntendedUse: "Tunnel", diff --git a/calico-vpp-agent/connectivity/vxlan.go b/calico-vpp-agent/felix/connectivity/vxlan.go similarity index 88% rename from calico-vpp-agent/connectivity/vxlan.go rename to calico-vpp-agent/felix/connectivity/vxlan.go index d7e1eb218..644dec44e 100644 --- a/calico-vpp-agent/connectivity/vxlan.go +++ b/calico-vpp-agent/felix/connectivity/vxlan.go @@ -21,23 +21,33 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type VXLanProvider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache vxlanIfs map[string]vpptypes.VXLanTunnel vxlanRoutes map[uint32]map[string]bool ip4NodeIndex uint32 ip6NodeIndex uint32 } -func NewVXLanProvider(d *ConnectivityProviderData) *VXLanProvider { - return &VXLanProvider{d, make(map[string]vpptypes.VXLanTunnel), make(map[uint32]map[string]bool), 0, 0} +func NewVXLanProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *VXLanProvider { + return &VXLanProvider{ + vpp: vpp, + log: log, + cache: cache, + vxlanIfs: make(map[string]vpptypes.VXLanTunnel), + vxlanRoutes: make(map[uint32]map[string]bool), + } } func (p *VXLanProvider) EnableDisable(isEnable bool) { @@ -72,9 +82,8 @@ func (p *VXLanProvider) RescanState() { if err != nil { p.log.Errorf("Error listing VXLan tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() for _, tunnel := range tunnels { - if (ip4 != nil && tunnel.SrcAddress.Equal(*ip4)) || (ip6 != nil && tunnel.SrcAddress.Equal(*ip6)) { + if (p.cache.GetNodeIP4() != nil && tunnel.SrcAddress.Equal(*p.cache.GetNodeIP4())) || (p.cache.GetNodeIP6() != nil && tunnel.SrcAddress.Equal(*p.cache.GetNodeIP6())) { if tunnel.Vni == p.getVXLANVNI() && tunnel.DstPort == p.getVXLANPort() && tunnel.SrcPort == p.getVXLANPort() { p.log.Infof("Found existing tunnel: %s", tunnel.String()) p.vxlanIfs[tunnel.DstAddress.String()+"-"+fmt.Sprint(tunnel.Vni)] = tunnel @@ -107,7 +116,7 @@ func (p *VXLanProvider) RescanState() { } func (p *VXLanProvider) getVXLANVNI() uint32 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.VXLANVNI == 0 { return uint32(config.DefaultVXLANVni) } @@ -115,7 +124,7 @@ func (p *VXLanProvider) getVXLANVNI() uint32 { } func (p *VXLanProvider) getVXLANPort() uint16 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.VXLANPort == 0 { return config.DefaultVXLANPort } @@ -124,7 +133,7 @@ func (p *VXLanProvider) getVXLANPort() uint16 { func (p *VXLanProvider) getEncapVrfID(cn *common.NodeConnectivity) uint32 { if cn.Vni != 0 { - net, ok := p.server.networks[cn.Vni] + net, ok := p.cache.Networks[cn.Vni] if ok { return net.VRF.Tables[0] } @@ -133,11 +142,10 @@ func (p *VXLanProvider) getEncapVrfID(cn *common.NodeConnectivity) uint32 { } func (p *VXLanProvider) getNodeIPForConnectivity(cn *common.NodeConnectivity) (nodeIP net.IP, err error) { - ip4, ip6 := p.server.GetNodeIPs() - if vpplink.IsIP6(cn.NextHop) && ip6 != nil { - return *ip6, nil - } else if !vpplink.IsIP6(cn.NextHop) && ip4 != nil { - return *ip4, nil + if vpplink.IsIP6(cn.NextHop) && p.cache.GetNodeIP6() != nil { + return *p.cache.GetNodeIP6(), nil + } else if !vpplink.IsIP6(cn.NextHop) && p.cache.GetNodeIP4() != nil { + return *p.cache.GetNodeIP4(), nil } else { return nodeIP, fmt.Errorf("missing node address") } @@ -223,7 +231,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { }) if cn.Vni != 0 { for idx, ipFamily := range vpplink.IPFamilies { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[idx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[idx] p.log.Infof("connectivity(add) set vxlan interface %d in vrf %d", tunnel.SwIfIndex, vrfIndex) err := p.vpp.SetInterfaceVRF(tunnel.SwIfIndex, vrfIndex, ipFamily.IsIP6) if err != nil { @@ -234,7 +242,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { p.log.Infof("connectivity(add) set vxlan interface unnumbered") var uplinkToUse uint32 for _, intf := range common.VppManagerInfo.UplinkStatuses { - if intf.PhysicalNetworkName == p.server.networks[cn.Vni].PhysicalNetworkName { + if intf.PhysicalNetworkName == p.cache.Networks[cn.Vni].PhysicalNetworkName { uplinkToUse = intf.SwIfIndex break } @@ -252,7 +260,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { if cn.Vni == 0 { p.log.Infof("connectivity(add) vxlan route dst=%s via swIfIndex=%d", cn.Dst.IP.String(), tunnel.SwIfIndex) } else { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] p.log.Infof("connectivity(add) vxlan route dst=%s via swIfIndex %d in VRF %d (VNI:%d)", cn.Dst.IP.String(), tunnel.SwIfIndex, vrfIndex, cn.Vni) table = vrfIndex @@ -290,7 +298,7 @@ func (p *VXLanProvider) DelConnectivity(cn *common.NodeConnectivity) error { }}, } } else { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] p.log.Infof("connectivity(del) VXLan cn=%s swIfIndex=%d in VRF %d (VNI:%d)", cn.String(), tunnel.SwIfIndex, vrfIndex, cn.Vni) routeToDelete = &types.Route{ Dst: &cn.Dst, diff --git a/calico-vpp-agent/connectivity/wireguard.go b/calico-vpp-agent/felix/connectivity/wireguard.go similarity index 70% rename from calico-vpp-agent/connectivity/wireguard.go rename to calico-vpp-agent/felix/connectivity/wireguard.go index 859143487..4f8d76b24 100644 --- a/calico-vpp-agent/connectivity/wireguard.go +++ b/calico-vpp-agent/felix/connectivity/wireguard.go @@ -23,41 +23,53 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type WireguardProvider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache + clientv3 calicov3cli.Interface wireguardTunnels map[string]*vpptypes.WireguardTunnel wireguardPeers map[string]vpptypes.WireguardPeer - nodesToWGPublicKey map[string]string + NodesToWGPublicKey map[string]string } -func NewWireguardProvider(d *ConnectivityProviderData) *WireguardProvider { +func NewWireguardProvider(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, cache *cache.Cache, log *logrus.Entry) *WireguardProvider { return &WireguardProvider{ - ConnectivityProviderData: d, - wireguardTunnels: make(map[string]*vpptypes.WireguardTunnel), - wireguardPeers: make(map[string]vpptypes.WireguardPeer), - nodesToWGPublicKey: make(map[string]string), + vpp: vpp, + log: log, + cache: cache, + clientv3: clientv3, + wireguardTunnels: make(map[string]*vpptypes.WireguardTunnel), + wireguardPeers: make(map[string]vpptypes.WireguardPeer), + NodesToWGPublicKey: make(map[string]string), } } func (p *WireguardProvider) Enabled(cn *common.NodeConnectivity) bool { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if !felixConfig.WireguardEnabled { return false } - node := p.GetNodeByIP(cn.NextHop) - return p.nodesToWGPublicKey[node.Name] != "" + if node, found := p.cache.NodeByAddr[cn.NextHop.String()]; found { + return p.NodesToWGPublicKey[node.Name] != "" + } + return false } func (p *WireguardProvider) getWireguardPort() uint16 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.WireguardListeningPort == 0 { return uint16(config.DefaultWireguardPort) } @@ -65,28 +77,31 @@ func (p *WireguardProvider) getWireguardPort() uint16 { } func (p *WireguardProvider) getNodePublicKey(cn *common.NodeConnectivity) ([]byte, error) { - node := p.GetNodeByIP(cn.NextHop) - if p.nodesToWGPublicKey[node.Name] == "" { + node, found := p.cache.NodeByAddr[cn.NextHop.String()] + if !found { + return nil, fmt.Errorf("node=%s not found", cn.NextHop.String()) + } + if p.NodesToWGPublicKey[node.Name] == "" { return nil, fmt.Errorf("no public key for node=%s", node.Name) } - p.log.Infof("connectivity(add) Wireguard nodeName=%s pubKey=%s", node.Name, p.nodesToWGPublicKey[node.Name]) - key, err := base64.StdEncoding.DecodeString(p.nodesToWGPublicKey[node.Name]) + p.log.Infof("connectivity(add) Wireguard nodeName=%s pubKey=%s", node.Name, p.NodesToWGPublicKey[node.Name]) + key, err := base64.StdEncoding.DecodeString(p.NodesToWGPublicKey[node.Name]) if err != nil { - return nil, errors.Wrapf(err, "Error decoding wireguard public key %s", p.nodesToWGPublicKey[node.Name]) + return nil, errors.Wrapf(err, "Error decoding wireguard public key %s", p.NodesToWGPublicKey[node.Name]) } return key, nil } func (p *WireguardProvider) publishWireguardPublicKey(pubKey string) error { // Ref: felix/daemon/daemon.go:1056 - node, err := p.Clientv3().Nodes().Get(context.Background(), *config.NodeName, options.GetOptions{}) + node, err := p.clientv3.Nodes().Get(context.Background(), *config.NodeName, options.GetOptions{}) if err != nil { return errors.Wrapf(err, "Error getting node config") } p.log.Infof("connectivity(add) Wireguard publishing nodeName=%s pubKey=%s", *config.NodeName, pubKey) node.Status.WireguardPublicKey = pubKey - _, err = p.Clientv3().Nodes().Update(context.Background(), node, options.SetOptions{}) + _, err = p.clientv3.Nodes().Update(context.Background(), node, options.SetOptions{}) if err != nil { return errors.Wrapf(err, "Error updating node config") } @@ -102,13 +117,12 @@ func (p *WireguardProvider) RescanState() { if err != nil { p.log.Errorf("Error listing wireguard tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() for _, tunnel := range tunnels { - if ip4 != nil && tunnel.Addr.Equal(*ip4) { + if p.cache.GetNodeIP4() != nil && tunnel.Addr.Equal(*p.cache.GetNodeIP4()) { p.log.Infof("Found existing v4 tunnel: %s", tunnel) p.wireguardTunnels["ip4"] = tunnel } - if ip6 != nil && tunnel.Addr.Equal(*ip6) { + if p.cache.GetNodeIP6() != nil && tunnel.Addr.Equal(*p.cache.GetNodeIP6()) { p.log.Infof("Found existing v6 tunnel: %s", tunnel) p.wireguardTunnels["ip6"] = tunnel } @@ -135,11 +149,16 @@ func (p *WireguardProvider) errorCleanup(tunnel *vpptypes.WireguardTunnel) { func (p *WireguardProvider) EnableDisable(isEnable bool) { if isEnable { if len(p.wireguardTunnels) == 0 { - err := p.createWireguardTunnels() + err := p.createWireguardTunnels(p.cache.GetNodeIP4(), "ip4") if err != nil { p.log.Errorf("Wireguard: Error creating v4 tunnel %s", err) return } + err = p.createWireguardTunnels(p.cache.GetNodeIP6(), "ip6") + if err != nil { + p.log.Errorf("Wireguard: Error creating v6 tunnel %s", err) + return + } } for _, tun := range p.wireguardTunnels { @@ -160,83 +179,70 @@ func (p *WireguardProvider) EnableDisable(isEnable bool) { } } -func (p *WireguardProvider) createWireguardTunnels() error { - - var nodeIP4, nodeIP6 net.IP - ip4, ip6 := p.server.GetNodeIPs() - if ip6 != nil { - nodeIP6 = *ip6 +func (p *WireguardProvider) createWireguardTunnels(nodeIP *net.IP, ipFamily string) error { + if nodeIP == nil { + return nil + } + p.log.Debugf("Adding wireguard Tunnel to VPP") + tunnel := &vpptypes.WireguardTunnel{ + Addr: *nodeIP, + Port: p.getWireguardPort(), } - if ip4 != nil { - nodeIP4 = *ip4 + var swIfIndex uint32 + var err error + if len(p.wireguardTunnels) != 0 { // we already have one, use same public key + for _, tun := range p.wireguardTunnels { + tunnel.PrivateKey = tun.PrivateKey + break + } + swIfIndex, err = p.vpp.AddWireguardTunnel(tunnel, false /* generateKey */) } else { - return fmt.Errorf("missing node address") - } - nodeIPs := map[string]net.IP{"ip4": nodeIP4, "ip6": nodeIP6} - for ipfamily, nodeIP := range nodeIPs { - if nodeIP != nil { - p.log.Debugf("Adding wireguard Tunnel to VPP") - tunnel := &vpptypes.WireguardTunnel{ - Addr: nodeIP, - Port: p.getWireguardPort(), - } - var swIfIndex uint32 - var err error - if len(p.wireguardTunnels) != 0 { // we already have one, use same public key - for _, tun := range p.wireguardTunnels { - tunnel.PrivateKey = tun.PrivateKey - break - } - swIfIndex, err = p.vpp.AddWireguardTunnel(tunnel, false /* generateKey */) - } else { - swIfIndex, err = p.vpp.AddWireguardTunnel(tunnel, true /* generateKey */) - } + swIfIndex, err = p.vpp.AddWireguardTunnel(tunnel, true /* generateKey */) + } - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error creating wireguard tunnel") - } - // fetch public key of created tunnel - createdTunnel, err := p.vpp.GetWireguardTunnel(swIfIndex) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error fetching wireguard tunnel after creation") - } - tunnel.PublicKey = createdTunnel.PublicKey - tunnel.PrivateKey = createdTunnel.PrivateKey + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error creating wireguard tunnel") + } + // fetch public key of created tunnel + createdTunnel, err := p.vpp.GetWireguardTunnel(swIfIndex) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error fetching wireguard tunnel after creation") + } + tunnel.PublicKey = createdTunnel.PublicKey + tunnel.PrivateKey = createdTunnel.PrivateKey - err = p.vpp.InterfaceSetUnnumbered(swIfIndex, common.VppManagerInfo.GetMainSwIfIndex()) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error setting wireguard tunnel unnumbered") - } + err = p.vpp.InterfaceSetUnnumbered(swIfIndex, common.VppManagerInfo.GetMainSwIfIndex()) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error setting wireguard tunnel unnumbered") + } - err = p.vpp.EnableGSOFeature(swIfIndex) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error enabling gso for wireguard interface") - } + err = p.vpp.EnableGSOFeature(swIfIndex) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error enabling gso for wireguard interface") + } - err = p.vpp.CnatEnableFeatures(swIfIndex) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error enabling nat for wireguard interface") - } + err = p.vpp.CnatEnableFeatures(swIfIndex) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error enabling nat for wireguard interface") + } - err = p.vpp.InterfaceAdminUp(swIfIndex) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error setting wireguard interface up") - } + err = p.vpp.InterfaceAdminUp(swIfIndex) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error setting wireguard interface up") + } - common.SendEvent(common.CalicoVppEvent{ - Type: common.TunnelAdded, - New: swIfIndex, - }) + common.SendEvent(common.CalicoVppEvent{ + Type: common.TunnelAdded, + New: swIfIndex, + }) - p.wireguardTunnels[ipfamily] = tunnel - } - } + p.wireguardTunnels[ipFamily] = tunnel p.log.Infof("connectivity(add) Wireguard Done tunnel=%s", p.wireguardTunnels) return nil } diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index c288dd691..adafbe48d 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -17,12 +17,9 @@ package felix import ( "fmt" - "net" - "sync" "github.com/pkg/errors" calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - felixConfig "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/proto" calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/sirupsen/logrus" @@ -32,6 +29,7 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -47,12 +45,9 @@ type Server struct { felixServerEventChan chan any - felixConfigReceived bool - FelixConfigChan chan *felixConfig.Config - - ippoolLock sync.RWMutex - policiesHandler *policies.PoliciesHandler - cniHandler *cni.CNIHandler + policiesHandler *policies.PoliciesHandler + cniHandler *cni.CNIHandler + connectivityHandler *connectivity.ConnectivityHandler prometheusServer *prometheus.PrometheusServer } @@ -61,17 +56,14 @@ type Server struct { func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *logrus.Entry) *Server { cache := cache.NewCache(log) server := &Server{ - log: log, - vpp: vpp, - + log: log, + vpp: vpp, felixServerEventChan: make(chan any, common.ChanSize), - felixConfigReceived: false, - FelixConfigChan: make(chan *felixConfig.Config), - - cache: cache, - policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), - cniHandler: cni.NewCNIHandler(vpp, cache, log), + cache: cache, + policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), + cniHandler: cni.NewCNIHandler(vpp, cache, log), + connectivityHandler: connectivity.NewConnectivityHandler(vpp, cache, clientv3, log.WithFields(logrus.Fields{"component": "connectivity"})), prometheusServer: prometheus.NewPrometheusServer(vpp, log.WithFields(logrus.Fields{"component": "prometheus"})), } @@ -101,6 +93,10 @@ func (s *Server) GotOurNodeBGPchan() chan *common.LocalNodeSpec { return s.policiesHandler.GotOurNodeBGPchan } +func (s *Server) GotFelixConfig() chan any { + return s.policiesHandler.GotFelixConfig +} + func (s *Server) GetCache() *cache.Cache { return s.cache } @@ -109,12 +105,6 @@ func (s *Server) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { s.cache.BGPConf = bgpConf } -func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { - s.ippoolLock.RLock() - defer s.ippoolLock.RUnlock() - return s.cache.GetPrefixIPPool(prefix) -} - func (s *Server) getMainInterface() *config.UplinkStatus { for _, i := range common.VppManagerInfo.UplinkStatuses { if i.IsMain { @@ -271,6 +261,13 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { s.log.Debugf("Ignoring HostMetadataRemove") case *proto.HostMetadataV4V6Update: err = s.policiesHandler.OnHostMetadataV4V6Update(evt) + if err != nil { + return err + } + err = s.connectivityHandler.OnHostMetadataV4V6Update(evt) + if err != nil { + return err + } case *proto.HostMetadataV4V6Remove: err = s.policiesHandler.OnHostMetadataV4V6Remove(evt) case *proto.IPAMPoolUpdate: @@ -290,6 +287,10 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { common.SendEvent(common.CalicoVppEvent{ Type: common.BGPConfChanged, }) + case *proto.WireguardEndpointUpdate: + err = s.connectivityHandler.OnWireguardEndpointUpdate(evt) + case *proto.WireguardEndpointRemove: + err = s.connectivityHandler.OnWireguardEndpointRemove(evt) case *model.CniPodAddEvent: err = s.cniHandler.OnPodAdd(evt) case *model.CniPodDelEvent: @@ -357,6 +358,42 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) } s.policiesHandler.OnTunnelDelete(swIfIndex) + case common.ConnectivityAdded: + new, ok := evt.New.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) + } + err := s.connectivityHandler.UpdateIPConnectivity(new, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while adding connectivity %s", err) + } + case common.ConnectivityDeleted: + old, ok := evt.Old.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) + } + err := s.connectivityHandler.UpdateIPConnectivity(old, true /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while deleting connectivity %s", err) + } + case common.SRv6PolicyAdded: + new, ok := evt.New.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) + } + err := s.connectivityHandler.UpdateSRv6Policy(new, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while adding SRv6 Policy %s", err) + } + case common.SRv6PolicyDeleted: + old, ok := evt.Old.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) + } + err := s.connectivityHandler.UpdateSRv6Policy(old, true /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while deleting SRv6 Policy %s", err) + } default: s.log.Warnf("Unhandled CalicoVppEvent.Type: %s", evt.Type) } diff --git a/calico-vpp-agent/felix/felix_server_test.go b/calico-vpp-agent/felix/felix_server_test.go index ee198a1b0..67f86e8d0 100644 --- a/calico-vpp-agent/felix/felix_server_test.go +++ b/calico-vpp-agent/felix/felix_server_test.go @@ -575,9 +575,6 @@ var _ = Describe("Felix functionality", func() { }) It("should error out when state is not connected", func() { policiesHandler.OnFelixSocketStateChanged(&common.FelixSocketStateChanged{NewState: common.StateDisconnected}) - go func() { - <-felixServer.FelixConfigChan - }() _ = felixServer.handleConfigUpdate( &proto.ConfigUpdate{ Config: configs, @@ -588,9 +585,6 @@ var _ = Describe("Felix functionality", func() { It("should update felix config", func() { By("adding new felix config, that changes endpointToHostAction and removes failsafe rules") policiesHandler.OnFelixSocketStateChanged(&common.FelixSocketStateChanged{NewState: common.StateConnected}) - go func() { - <-felixServer.FelixConfigChan - }() err := felixServer.handleConfigUpdate( &proto.ConfigUpdate{ Config: configs, diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go index d03b35ae2..916e5c7ac 100644 --- a/calico-vpp-agent/felix/felixconfig.go +++ b/calico-vpp-agent/felix/felixconfig.go @@ -71,18 +71,11 @@ func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { s.cache.FelixConfig.RawValues(), ) - // Note: This function will be called each time the Felix config changes. - // If we start handling config settings that require agent restart, - // we'll need to add a mechanism for that - if !s.felixConfigReceived { - s.felixConfigReceived = true - s.FelixConfigChan <- s.cache.FelixConfig - } - if !changed { return nil } + s.connectivityHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.cniHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go index 31b327f36..1570d19f9 100644 --- a/calico-vpp-agent/felix/ipam.go +++ b/calico-vpp-agent/felix/ipam.go @@ -30,8 +30,6 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { s.log.Debugf("Empty pool") return nil } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() newIpamPool := msg.GetPool() oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] @@ -49,6 +47,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil || err2 != nil { return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) } + s.connectivityHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) s.cniHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, @@ -64,6 +63,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil { return errors.Wrap(err, "error handling ipam add") } + s.connectivityHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) s.cniHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, @@ -78,8 +78,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { s.log.Debugf("Empty pool") return nil } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] if found { delete(s.cache.IPPoolMap, msg.GetId()) @@ -94,6 +93,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { Old: ipamPoolCopy(oldIpamPool), New: nil, }) + s.connectivityHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) s.cniHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) } else { s.log.Warnf("Deleting unknown ippool") diff --git a/calico-vpp-agent/felix/cni/cni_node_test.go b/calico-vpp-agent/felix/node_test.go similarity index 89% rename from calico-vpp-agent/felix/cni/cni_node_test.go rename to calico-vpp-agent/felix/node_test.go index 993e88118..27608db38 100644 --- a/calico-vpp-agent/felix/cni/cni_node_test.go +++ b/calico-vpp-agent/felix/node_test.go @@ -11,16 +11,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cni_test +package felix import ( "context" "encoding/base64" "fmt" "net" - "os" "strings" - "testing" vpptypes "github.com/calico-vpp/vpplink/api/v0" . "github.com/onsi/ginkgo" @@ -36,8 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" @@ -46,47 +43,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -// Names of integration tests arguments -const ( - VppImageArgName = "VPP_IMAGE" - VppBinaryArgName = "VPP_BINARY" - VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" -) - -// TestCniIntegration runs all the ginkgo integration test inside CNI package -func TestCniIntegration(t *testing.T) { - // skip test if test run is not integration test run (prevent accidental run of integration tests using go test ./...) - _, isIntegrationTestRun := os.LookupEnv(VppImageArgName) - if !isIntegrationTestRun { - t.Skip("skipping CNI integration tests (set INTEGRATION_TEST env variable to run these tests)") - } - - // integrate gomega and ginkgo -> register all CNI integration tests - RegisterFailHandler(Fail) - RunSpecs(t, "CNI Integration Suite") -} - -var _ = BeforeSuite(func() { - // extract common input for CNI integration tests - var found bool - testutils.VppImage, found = os.LookupEnv(VppImageArgName) - if !found { - Expect(testutils.VppImage).ToNot(BeEmpty(), fmt.Sprintf("Please specify docker image containing "+ - "VPP binary using %s environment variable.", VppImageArgName)) - } - testutils.VppBinary, found = os.LookupEnv(VppBinaryArgName) - if !found { - Expect(testutils.VppBinary).ToNot(BeEmpty(), fmt.Sprintf("Please specify VPP binary (full path) "+ - "inside docker image %s using %s environment variable.", testutils.VppImage, VppBinaryArgName)) - } - - vppContainerExtraArgsList, found := os.LookupEnv(VppContainerExtraArgsName) - if found { - testutils.VppContainerExtraArgs = append(testutils.VppContainerExtraArgs, strings.Split(vppContainerExtraArgsList, ",")...) - } - -}) - // Common setup constants const ( ThisNodeName = "node1" @@ -105,19 +61,19 @@ const ( var _ = Describe("Node-related functionality of CNI", func() { var ( - log *logrus.Logger - vpp *vpplink.VppLink - felixServer *felix.Server - connectivityServer *connectivity.ConnectivityServer - client *calico.CalicoClientStub - ipamStub *mocks.IpamCacheStub - pubSubHandlerMock *mocks.PubSubHandlerMock - felixConfig *config.Config - uplinkSwIfIndex uint32 + log *logrus.Logger + vpp *vpplink.VppLink + felixServer *Server + client *calico.CalicoClientStub + ipamPoolUpdates []*proto.IPAMPoolUpdate + pubSubHandlerMock *mocks.PubSubHandlerMock + felixConfig *config.Config + uplinkSwIfIndex uint32 ) BeforeEach(func() { log = logrus.New() client = calico.NewCalicoClientStub() + ipamPoolUpdates = nil common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) agentConf.GetCalicoVppFeatureGates().SRv6Enabled = &agentConf.False }) @@ -129,13 +85,7 @@ var _ = Describe("Node-related functionality of CNI", func() { vpp, uplinkSwIfIndex = testutils.ConfigureVPP(log) // setup connectivity server (functionality target of tests) - if ipamStub == nil { - ipamStub = mocks.NewIpamCacheStub() - } - connectivityServer = connectivity.NewConnectivityServer(vpp, ipamStub, client, - log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) - connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{}) - felixServer = felix.NewFelixServer( + felixServer = NewFelixServer( vpp, client, log.WithFields(logrus.Fields{"subcomponent": "connectivity"}), @@ -143,8 +93,10 @@ var _ = Describe("Node-related functionality of CNI", func() { if felixConfig == nil { felixConfig = &config.Config{} } - connectivityServer.SetFelixConfig(felixConfig) felixServer.GetCache().FelixConfig = felixConfig + for _, poolUpdate := range ipamPoolUpdates { + felixServer.GetCache().IPPoolMap[poolUpdate.Id] = poolUpdate.Pool + } common.VppManagerInfo = &agentConf.VppManagerInfo{ UplinkStatuses: map[string]agentConf.UplinkStatus{ "eth0": {IsMain: true, SwIfIndex: 1}, @@ -156,7 +108,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Context("With FLAT connectivity", func() { It("should only configure correct routes in VPP", func() { By("Adding node") - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.FLAT, @@ -188,8 +140,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Context("With IPSEC connectivity", func() { BeforeEach(func() { // add node pool for IPSec (uses IPIP tunnels) - ipamStub = mocks.NewIpamCacheStub() - ipamStub.AddPrefixIPPool(testutils.IPNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ + ipamPoolUpdates = append(ipamPoolUpdates, &proto.IPAMPoolUpdate{ Id: fmt.Sprintf("custom-test-pool-for-ipsec-%s", AddedNodeIP+"/24"), Pool: &proto.IPAMPool{ Cidr: AddedNodeIP + "/24", @@ -224,7 +175,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // Note: not testing setting of IPsecAsyncMode and threads dedicated to IPSec (CryptoWorkers) // inside RescanState() function call By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) // FIXME The concept of Destination and NextHop in common.NodeConnectivity is not well defined // (is the Destination the IP of added node, or it subnet or totally unrelated network? Is // the nexthop the IP of added node or could it be IP of some intermediate router that is @@ -232,7 +183,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // Need to either define it well(unify it?) and fix connectivity providers(and tests) or leave it // in connectivity provider implementation and check each test for semantics used in given // connectivity provider - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(AddedNodeIP), // next hop == other node IP (for IPSec impl) ResolvedProvider: connectivity.IPSEC, @@ -330,8 +281,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Context("With VXLAN connectivity", func() { BeforeEach(func() { // add node pool for VXLAN - ipamStub = mocks.NewIpamCacheStub() - ipamStub.AddPrefixIPPool(testutils.IPNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ + ipamPoolUpdates = append(ipamPoolUpdates, &proto.IPAMPoolUpdate{ Id: fmt.Sprintf("custom-test-pool-for-vxlan-%s", AddedNodeIP+"/24"), Pool: &proto.IPAMPool{ Cidr: AddedNodeIP + "/24", @@ -350,7 +300,7 @@ var _ = Describe("Node-related functionality of CNI", func() { It("should have vxlan tunnel and route forwarding to it", func() { By("Initialize VXLAN and add static VXLAN configuration") - err := connectivityServer.ForceRescanState(connectivity.VXLAN) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.VXLAN) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly create ???") @@ -359,8 +309,8 @@ var _ = Describe("Node-related functionality of CNI", func() { testutils.AssertNextNodeLink("vxlan6-input", "ip6-input", vpp) By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.VXLAN, @@ -435,8 +385,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Context("With IP-IP connectivity", func() { BeforeEach(func() { // add node pool for IPIP - ipamStub = mocks.NewIpamCacheStub() - ipamStub.AddPrefixIPPool(testutils.IPNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ + ipamPoolUpdates = append(ipamPoolUpdates, &proto.IPAMPoolUpdate{ Id: fmt.Sprintf("custom-test-pool-for-ipip-%s", AddedNodeIP+"/24"), Pool: &proto.IPAMPool{ Cidr: AddedNodeIP + "/24", @@ -455,8 +404,8 @@ var _ = Describe("Node-related functionality of CNI", func() { It("should have IP-IP tunnel and route forwarding to it", func() { By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.IPIP, @@ -527,6 +476,15 @@ var _ = Describe("Node-related functionality of CNI", func() { }) Context("With Wireguard connectivity", func() { BeforeEach(func() { + // add node pool for WireGuard (uses IPIP pool selection with WireGuard enabled) + ipamPoolUpdates = append(ipamPoolUpdates, &proto.IPAMPoolUpdate{ + Id: fmt.Sprintf("custom-test-pool-for-wireguard-%s", AddedNodeIP+"/24"), + Pool: &proto.IPAMPool{ + Cidr: AddedNodeIP + "/24", + IpipMode: encap.Always, + }, + }) + // setup felix config for Wireguard configuration felixConfig.WireguardEnabled = true felixConfig.WireguardListeningPort = 11111 @@ -557,16 +515,16 @@ var _ = Describe("Node-related functionality of CNI", func() { It("must configure wireguard tunnel with one peer and routes to it", func() { By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.ForceProviderEnableDisable(connectivity.WIREGUARD, true) // creates the tunnel (this is normally called by Felix config change event handler) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.ForceProviderEnableDisable(connectivity.WIREGUARD, true) // creates the tunnel (this is normally called by Felix config change event handler) Expect(err).ToNot(HaveOccurred(), "could not call ForceProviderEnableDisable") addedNodePublicKey := "public-key-for-added-node" // max 32 characters due to VPP binapi - connectivityServer.ForceNodeAddition(common.LocalNodeSpec{ + felixServer.connectivityHandler.ForceNodeAddition(common.LocalNodeSpec{ Name: AddedNodeName, }, net.ParseIP(AddedNodeIP)) - connectivityServer.ForceWGPublicKeyAddition(AddedNodeName, base64.StdEncoding.EncodeToString([]byte(addedNodePublicKey))) - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + felixServer.connectivityHandler.ForceWGPublicKeyAddition(AddedNodeName, base64.StdEncoding.EncodeToString([]byte(addedNodePublicKey))) + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(AddedNodeIP), // wireguard impl uses nexthop as node IP ResolvedProvider: connectivity.WIREGUARD, @@ -700,7 +658,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // Note: localsids as tunnel endpoints are not bound to any particular tunnel, they can // exists without tunnel or server one or more tunnels at the same time // -> they are not dependent on anything from NodeConnection event and are created before event loop - err := connectivityServer.ForceRescanState(connectivity.SRv6) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.SRv6) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly create SRv6 tunnel endpoints(LocalSids) for this node") @@ -741,8 +699,8 @@ var _ = Describe("Node-related functionality of CNI", func() { By("Setting and checking encapsulation source for SRv6") // Note: encapsulation source sets source IP for traffic when exiting tunnel(=decapsulating) - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.ForceRescanState(connectivity.SRv6) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.SRv6) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly set encapsulation source IP for this node") // Note: no specialized binary api for getting SR encap source address -> using VPP's VPE binary API @@ -752,7 +710,7 @@ var _ = Describe("Node-related functionality of CNI", func() { "sr encapsulation source address is misconfigured") By("Adding node (the tunnel end node IP destination)") - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *tunnelEndNodeIPNet, NextHop: net.ParseIP(AddedNodeIPv6), ResolvedProvider: connectivity.SRv6, @@ -766,7 +724,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // policy/tunnel info(1 segment long SRv6 tunnel) leading to that new localsid (to tunnel-end // node). Then it uses BGP to inform this node (the tunnel start node) about it. The BGP watcher // catches it and sends event to connectivity server on this node and that results in call below. - err = connectivityServer.UpdateSRv6Policy(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateSRv6Policy(&common.NodeConnectivity{ Dst: net.IPNet{}, NextHop: net.ParseIP(AddedNodeIPv6), ResolvedProvider: "", @@ -793,7 +751,7 @@ var _ = Describe("Node-related functionality of CNI", func() { "configuration (steering and policy)") By("Adding Srv6 traffic routing") - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *tunnelEndLocalSid, NextHop: net.ParseIP(GatewayIPv6), ResolvedProvider: connectivity.SRv6, diff --git a/calico-vpp-agent/felix/cni/cni_pod_test.go b/calico-vpp-agent/felix/pod_test.go similarity index 99% rename from calico-vpp-agent/felix/cni/cni_pod_test.go rename to calico-vpp-agent/felix/pod_test.go index a969c1514..63249360f 100644 --- a/calico-vpp-agent/felix/cni/cni_pod_test.go +++ b/calico-vpp-agent/felix/pod_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cni_test +package felix import ( "fmt" diff --git a/calico-vpp-agent/felix/policies/hostmetadata.go b/calico-vpp-agent/felix/policies/hostmetadata.go index 50d9857dc..48ae1ff7b 100644 --- a/calico-vpp-agent/felix/policies/hostmetadata.go +++ b/calico-vpp-agent/felix/policies/hostmetadata.go @@ -94,7 +94,21 @@ func (s *PoliciesHandler) OnHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Up } } + if old != nil { + if old.IPv4Address != nil { + delete(s.cache.NodeByAddr, old.IPv4Address.IP.String()) + } + if old.IPv6Address != nil { + delete(s.cache.NodeByAddr, old.IPv6Address.IP.String()) + } + } s.cache.NodeStatesByName[localNodeSpec.Name] = localNodeSpec + if localNodeSpec.IPv4Address != nil { + s.cache.NodeByAddr[localNodeSpec.IPv4Address.IP.String()] = localNodeSpec + } + if localNodeSpec.IPv6Address != nil { + s.cache.NodeByAddr[localNodeSpec.IPv6Address.IP.String()] = localNodeSpec + } return nil } @@ -112,8 +126,14 @@ func (s *PoliciesHandler) OnHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Re // restart if our BGP config changed return NodeWatcherRestartError{} } - s.configureRemoteNodeSnat(old, false /* isAdd */) + delete(s.cache.NodeStatesByName, msg.Hostname) + if old.IPv4Address != nil { + delete(s.cache.NodeByAddr, old.IPv4Address.IP.String()) + } + if old.IPv6Address != nil { + delete(s.cache.NodeByAddr, old.IPv6Address.IP.String()) + } return nil } diff --git a/calico-vpp-agent/felix/policies/policies_handler.go b/calico-vpp-agent/felix/policies/policies_handler.go index 88ad0eb54..e659ab8cb 100644 --- a/calico-vpp-agent/felix/policies/policies_handler.go +++ b/calico-vpp-agent/felix/policies/policies_handler.go @@ -66,6 +66,7 @@ type PoliciesHandler struct { GotOurNodeBGPchan chan *common.LocalNodeSpec GotOurNodeBGPchanOnce sync.Once + GotFelixConfig chan any } func NewPoliciesHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calicov3cli.Interface, log *logrus.Entry) *PoliciesHandler { @@ -81,6 +82,7 @@ func NewPoliciesHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calic state: common.StateDisconnected, GotOurNodeBGPchan: make(chan *common.LocalNodeSpec), + GotFelixConfig: make(chan any), } } diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 70d8d1c5b..68669414a 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -39,7 +39,7 @@ import ( apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" @@ -481,15 +481,19 @@ func AssertNextNodeLink(node, linkedNextNode string, vpp *vpplink.VppLink) int { return nextNodeIndex } -func ConfigureBGPNodeIPAddresses(connectivityServer *connectivity.ConnectivityServer) { +func ConfigureBGPNodeIPAddresses(cache *cache.Cache) { ip4, ip4net, _ := net.ParseCIDR(ThisNodeIP + "/24") ip4net.IP = ip4 ip6, ip6net, _ := net.ParseCIDR(ThisNodeIPv6 + "/128") ip6net.IP = ip6 - connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{ + nodeSpec := &common.LocalNodeSpec{ + Name: *config.NodeName, IPv4Address: ip4net, IPv6Address: ip6net, - }) + } + cache.NodeByAddr[ip4.String()] = nodeSpec + cache.NodeByAddr[ip6.String()] = nodeSpec + cache.NodeStatesByName[*config.NodeName] = nodeSpec } // AddIPPoolForCalicoClient is convenience function for adding IPPool to mocked Calico IPAM Stub used From 012b664fe32f6e172fd6a3b6df2b18e8b5574d19 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Tue, 2 Sep 2025 16:50:58 +0200 Subject: [PATCH 4/4] Split Services into watcher/handler under felix This patch splits services in two components, - a watcher that handles the informer fetching services and endpoints from the k8s API. - a handler that takes care of programming VPP with the NAT rules, within the context of the felix server's single goroutine. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Also cleaned up unused code from single-thread agent refactor. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 6 +- calico-vpp-agent/common/common.go | 17 + calico-vpp-agent/common/pubsub.go | 4 - calico-vpp-agent/felix/cni/cni_handler.go | 2 +- .../connectivity/connectivity_handler.go | 4 +- calico-vpp-agent/felix/felix_server.go | 16 +- calico-vpp-agent/felix/node_test.go | 6 +- calico-vpp-agent/felix/pod_test.go | 16 +- .../felix/policies/policies_handler.go | 4 +- .../{ => felix}/services/service.go | 0 .../felix/services/service_handler.go | 644 ++++++++++++++++++ .../{ => felix}/services/services_test.go | 88 ++- calico-vpp-agent/services/service_handler.go | 342 ---------- calico-vpp-agent/services/service_server.go | 549 --------------- calico-vpp-agent/tests/mocks/ipam.go | 74 -- calico-vpp-agent/testutils/testutils.go | 3 +- calico-vpp-agent/watchers/net_watcher.go | 3 - calico-vpp-agent/watchers/service_watcher.go | 267 ++++++++ config/config.go | 2 +- 19 files changed, 1015 insertions(+), 1032 deletions(-) rename calico-vpp-agent/{ => felix}/services/service.go (100%) create mode 100644 calico-vpp-agent/felix/services/service_handler.go rename calico-vpp-agent/{ => felix}/services/services_test.go (85%) delete mode 100644 calico-vpp-agent/services/service_handler.go delete mode 100644 calico-vpp-agent/services/service_server.go delete mode 100644 calico-vpp-agent/tests/mocks/ipam.go create mode 100644 calico-vpp-agent/watchers/service_watcher.go diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 4f59c6f66..47d39f5b2 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -37,7 +37,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/health" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/routing" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/services" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" ) @@ -147,11 +146,12 @@ func main() { bgpFilterWatcher := watchers.NewBGPFilterWatcher(clientv3, k8sclient, log.WithFields(logrus.Fields{"subcomponent": "BGPFilter-watcher"})) netWatcher := watchers.NewNetWatcher(vpp, log.WithFields(logrus.Fields{"component": "net-watcher"})) routingServer := routing.NewRoutingServer(vpp, bgpServer, log.WithFields(logrus.Fields{"component": "routing"})) - serviceServer := services.NewServiceServer(vpp, k8sclient, log.WithFields(logrus.Fields{"component": "services"})) localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) felixServer := felix.NewFelixServer(vpp, clientv3, log.WithFields(logrus.Fields{"component": "policy"})) felixWatcher := watchers.NewFelixWatcher(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "felix watcher"})) cniServer := watchers.NewCNIServer(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "cni"})) + serviceServer := watchers.NewServiceServer(felixServer.GetFelixServerEventChan(), k8sclient, log.WithFields(logrus.Fields{"component": "services"})) + err = watchers.InstallFelixPlugin() if err != nil { log.Fatalf("could not install felix plugin: %s", err) @@ -166,7 +166,6 @@ func main() { peerWatcher.SetBGPConf(bgpConf) routingServer.SetBGPConf(bgpConf) - serviceServer.SetBGPConf(bgpConf) felixServer.SetBGPConf(bgpConf) Go(felixServer.ServeFelix) @@ -213,7 +212,6 @@ func main() { prefixWatcher.SetOurBGPSpec(ourBGPSpec) routingServer.SetOurBGPSpec(ourBGPSpec) - serviceServer.SetOurBGPSpec(ourBGPSpec) localSIDWatcher.SetOurBGPSpec(ourBGPSpec) netWatcher.SetOurBGPSpec(ourBGPSpec) diff --git a/calico-vpp-agent/common/common.go b/calico-vpp-agent/common/common.go index 6f6724d40..3b6afe0bf 100644 --- a/calico-vpp-agent/common/common.go +++ b/calico-vpp-agent/common/common.go @@ -36,6 +36,9 @@ import ( "github.com/projectcalico/calico/felix/proto" apb "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/timestamppb" + v1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -56,6 +59,20 @@ type FelixServerIpam interface { GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool } +type ServiceAndEndpoints struct { + Service *v1.Service + EndpointSlices map[string]*discoveryv1.EndpointSlice +} + +type ServiceEndpointsUpdate struct { + Old *ServiceAndEndpoints + New *ServiceAndEndpoints +} + +type ServiceEndpointsDelete struct { + Meta *metav1.ObjectMeta +} + type LocalNodeSpec struct { ASNumber *numorstring.ASNumber Labels map[string]string diff --git a/calico-vpp-agent/common/pubsub.go b/calico-vpp-agent/common/pubsub.go index 2ac7b253e..f1af491ba 100644 --- a/calico-vpp-agent/common/pubsub.go +++ b/calico-vpp-agent/common/pubsub.go @@ -61,10 +61,6 @@ const ( NetAddedOrUpdated CalicoVppEventType = "NetAddedOrUpdated" NetDeleted CalicoVppEventType = "NetDeleted" - NetsSynced CalicoVppEventType = "NetsSynced" - - IpamPoolUpdate CalicoVppEventType = "IpamPoolUpdate" - IpamPoolRemove CalicoVppEventType = "IpamPoolRemove" ) var ( diff --git a/calico-vpp-agent/felix/cni/cni_handler.go b/calico-vpp-agent/felix/cni/cni_handler.go index 160ffd5a5..28f9116dc 100644 --- a/calico-vpp-agent/felix/cni/cni_handler.go +++ b/calico-vpp-agent/felix/cni/cni_handler.go @@ -294,7 +294,7 @@ func (s *CNIHandler) CNIHandlerInit() error { return nil } -// ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. +// ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI handler. // The usage is mainly for testing purposes. func (s *CNIHandler) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { s.cache.NetworkDefinitions[networkDefinition.Name] = networkDefinition diff --git a/calico-vpp-agent/felix/connectivity/connectivity_handler.go b/calico-vpp-agent/felix/connectivity/connectivity_handler.go index 2b70d9d57..ca8706b0e 100644 --- a/calico-vpp-agent/felix/connectivity/connectivity_handler.go +++ b/calico-vpp-agent/felix/connectivity/connectivity_handler.go @@ -255,8 +255,8 @@ func (s *ConnectivityHandler) ForceProviderEnableDisable(providerType string, en return nil } -// TODO get rid (if possible) of all this "Force" methods by refactor the test code -// (run the Server.ServeConnectivity(...) function and send into it events with common.SendEvent(...)) +// TODO get rid (if possible) of all these "Force" methods by refactoring tests +// to send connectivity events through the Felix event channel. // ForceNodeAddition will add other node information as provided by calico configuration // The usage is mainly for testing purposes. diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index adafbe48d..7d3fcc18d 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -31,6 +31,7 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/services" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -48,6 +49,7 @@ type Server struct { policiesHandler *policies.PoliciesHandler cniHandler *cni.CNIHandler connectivityHandler *connectivity.ConnectivityHandler + serviceHandler *services.ServiceHandler prometheusServer *prometheus.PrometheusServer } @@ -56,14 +58,16 @@ type Server struct { func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *logrus.Entry) *Server { cache := cache.NewCache(log) server := &Server{ - log: log, - vpp: vpp, + log: log, + vpp: vpp, + felixServerEventChan: make(chan any, common.ChanSize), cache: cache, policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), cniHandler: cni.NewCNIHandler(vpp, cache, log), connectivityHandler: connectivity.NewConnectivityHandler(vpp, cache, clientv3, log.WithFields(logrus.Fields{"component": "connectivity"})), + serviceHandler: services.NewServiceHandler(vpp, cache, log), prometheusServer: prometheus.NewPrometheusServer(vpp, log.WithFields(logrus.Fields{"component": "prometheus"})), } @@ -210,6 +214,10 @@ func (s *Server) ServeFelix(t *tomb.Tomb) error { if err != nil { return errors.Wrap(err, "Error in CNIHandlerInit") } + err = s.serviceHandler.ServiceHandlerInit() + if err != nil { + return errors.Wrap(err, "Error in ServiceHandlerInit") + } for { select { case <-t.Dying(): @@ -227,6 +235,10 @@ func (s *Server) ServeFelix(t *tomb.Tomb) error { func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { s.log.Debugf("Got message from felix: %#v", msg) switch evt := msg.(type) { + case *common.ServiceEndpointsUpdate: + s.serviceHandler.OnServiceEndpointsUpdate(evt) + case *common.ServiceEndpointsDelete: + s.serviceHandler.OnServiceEndpointsDelete(evt) case *proto.ConfigUpdate: err = s.handleConfigUpdate(evt) case *proto.InSync: diff --git a/calico-vpp-agent/felix/node_test.go b/calico-vpp-agent/felix/node_test.go index 27608db38..3ec25b605 100644 --- a/calico-vpp-agent/felix/node_test.go +++ b/calico-vpp-agent/felix/node_test.go @@ -84,7 +84,7 @@ var _ = Describe("Node-related functionality of CNI", func() { testutils.StartVPP() vpp, uplinkSwIfIndex = testutils.ConfigureVPP(log) - // setup connectivity server (functionality target of tests) + // setup Felix server with connectivity handler (functionality target of tests) felixServer = NewFelixServer( vpp, client, @@ -563,7 +563,7 @@ var _ = Describe("Node-related functionality of CNI", func() { })) By("checking remembering of public key for wireguard tunnel in calico configuration") - // Note: public/private key is created by VPP (connectivity server sends empty public/private + // Note: public/private key is created by VPP (connectivity handler sends empty public/private // keys but retrieves it back properly filled) thisNode, err := client.Nodes().Get(context.Background(), *agentConf.NodeName, options.GetOptions{}) Expect(err).ToNot(HaveOccurred(), @@ -723,7 +723,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // The tunnel-end node watches localsids on its node. On new localsid detection it creates // policy/tunnel info(1 segment long SRv6 tunnel) leading to that new localsid (to tunnel-end // node). Then it uses BGP to inform this node (the tunnel start node) about it. The BGP watcher - // catches it and sends event to connectivity server on this node and that results in call below. + // catches it and sends an event to the Felix connectivity handler on this node. err = felixServer.connectivityHandler.UpdateSRv6Policy(&common.NodeConnectivity{ Dst: net.IPNet{}, NextHop: net.ParseIP(AddedNodeIPv6), diff --git a/calico-vpp-agent/felix/pod_test.go b/calico-vpp-agent/felix/pod_test.go index 63249360f..3dbefeb62 100644 --- a/calico-vpp-agent/felix/pod_test.go +++ b/calico-vpp-agent/felix/pod_test.go @@ -66,10 +66,10 @@ var _ = Describe("Pod-related functionality of CNI", func() { testutils.StartVPP() vpp, _ = testutils.ConfigureVPP(log) Expect(vpp.CnatSetSnatAddresses(nodeIP4String, nodeIP6String)).To(Succeed()) - // setup connectivity server (functionality target of tests) + // setup shared Felix cache for CNI handler tests testCache = cache.NewCache(log.WithFields(logrus.Fields{"component": "cache"})) testCache.VppAvailableBuffers = 65536 - // setup CNI server (functionality target of tests) + // setup CNI handler (functionality target of tests) common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) cniHandler = cni.NewCNIHandler(vpp, testCache, log.WithFields(logrus.Fields{"component": "cni"})) cfg := &config.CalicoVppInterfacesConfigType{ @@ -98,7 +98,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Expect(err).Should(BeNil(), "Failed to get pod mock container's PID string") containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") - By("Adding pod using CNI server") + By("Adding pod using CNI handler") podSpec, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: interfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host @@ -183,7 +183,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Expect(err).Should(BeNil(), "Failed to get pod mock container's PID string") containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") - By("Adding pod using CNI server") + By("Adding pod using CNI handler") podSpec, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: interfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host @@ -402,7 +402,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Expect(err).Should(BeNil(), "Failed to get pod mock container's PID string") containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") - By("Adding Pod to primary network using CNI server") + By("Adding Pod to primary network using CNI handler") newPodForPrimaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: mainInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host @@ -418,7 +418,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) - By("Adding Pod to secondary(multinet) network using CNI server") + By("Adding Pod to secondary(multinet) network using CNI handler") secondaryIPAddress := testutils.FirstIPinIPRange(networkDefinition.Range).String() newPodForSecondaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, @@ -651,7 +651,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Expect(err).Should(BeNil(), "Failed to get pod mock container's PID string") containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") - By("Adding Pod to primary network using CNI server") + By("Adding Pod to primary network using CNI handler") newPodForPrimaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: mainInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host @@ -668,7 +668,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) - By("Adding Pod to secondary(multinet) network using CNI server") + By("Adding Pod to secondary(multinet) network using CNI handler") secondaryIPAddress := testutils.FirstIPinIPRange(networkDefinition.Range).String() newPodForSecondaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, diff --git a/calico-vpp-agent/felix/policies/policies_handler.go b/calico-vpp-agent/felix/policies/policies_handler.go index e659ab8cb..cbbea8315 100644 --- a/calico-vpp-agent/felix/policies/policies_handler.go +++ b/calico-vpp-agent/felix/policies/policies_handler.go @@ -112,7 +112,7 @@ func (s *PoliciesHandler) OnInSync(msg *proto.InSync) (err error) { return s.ApplyPendingState() } -// workloadAdded is called by the CNI server when a container interface is created, +// OnWorkloadAdded is called by the CNI handler when a container interface is created, // either during startup when reconnecting the interfaces, or when a new pod is created func (s *PoliciesHandler) OnWorkloadAdded(id *WorkloadEndpointID, swIfIndex uint32, ifName string, containerIPs []*net.IPNet) { // TODO: Send WorkloadEndpointStatusUpdate to felix @@ -161,7 +161,7 @@ func (s *PoliciesHandler) OnWorkloadAdded(id *WorkloadEndpointID, swIfIndex uint } } -// WorkloadRemoved is called by the CNI server when the interface of a pod is deleted +// OnWorkloadRemoved is called by the CNI handler when the interface of a pod is deleted func (s *PoliciesHandler) OnWorkloadRemoved(id *WorkloadEndpointID, containerIPs []*net.IPNet) { // TODO: Send WorkloadEndpointStatusRemove to felix diff --git a/calico-vpp-agent/services/service.go b/calico-vpp-agent/felix/services/service.go similarity index 100% rename from calico-vpp-agent/services/service.go rename to calico-vpp-agent/felix/services/service.go diff --git a/calico-vpp-agent/felix/services/service_handler.go b/calico-vpp-agent/felix/services/service_handler.go new file mode 100644 index 000000000..b525a1471 --- /dev/null +++ b/calico-vpp-agent/felix/services/service_handler.go @@ -0,0 +1,644 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package services + +import ( + "net" + "strconv" + "strings" + + v1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + + "github.com/pkg/errors" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +/** + * Service descriptions from the API are resolved into + * slices of LocalService, this allows to diffs between + * previous & expected state in VPP + */ +type LocalService struct { + Entries []types.CnatTranslateEntry + SpecificRoutes []net.IP + ServiceID string +} + +/** + * Store VPP's state in a map [CnatTranslateEntry.Key()]->ServiceState + */ +type ServiceState struct { + OwnerServiceID string /* serviceID(service.ObjectMeta) of the service that created this entry */ + VppID uint32 /* cnat translation ID in VPP */ +} + +type ServiceHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + serviceStateMap map[string]ServiceState +} + +func NewServiceHandler(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *ServiceHandler { + return &ServiceHandler{ + vpp: vpp, + log: log, + cache: cache, + serviceStateMap: make(map[string]ServiceState), + } +} + +func (s *ServiceHandler) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { + s.cache.BGPConf = bgpConf +} + +func (s *ServiceHandler) configureSnat() (err error) { + err = s.vpp.CnatSetSnatAddresses(s.getNodeIP(false /* isv6 */), s.getNodeIP(true /* isv6 */)) + if err != nil { + s.log.Errorf("Failed to configure SNAT addresses %v", err) + } + var nodeSpec *common.LocalNodeSpec + if spec, found := s.cache.NodeStatesByName[*config.NodeName]; found { + nodeSpec = spec + } + nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(nodeSpec) + if nodeIP6 != nil { + err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP6)) + if err != nil { + s.log.Errorf("Failed to add SNAT %s %v", common.FullyQualified(*nodeIP6), err) + } + } + if nodeIP4 != nil { + err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP4)) + if err != nil { + s.log.Errorf("Failed to add SNAT %s %v", common.FullyQualified(*nodeIP4), err) + } + } + for _, serviceCIDR := range *config.ServiceCIDRs { + err = s.vpp.CnatAddSnatPrefix(serviceCIDR) + if err != nil { + s.log.Errorf("Failed to Add Service CIDR %s %v", serviceCIDR, err) + } + } + err = s.vpp.SetK8sSnatPolicy() + if err != nil { + return errors.Wrap(err, "Error configuring cnat source policy") + } + for _, uplink := range common.VppManagerInfo.UplinkStatuses { + err = s.vpp.RegisterPodInterface(uplink.TapSwIfIndex) + if err != nil { + return errors.Wrap(err, "error configuring vpptap0 as pod intf") + } + + err = s.vpp.RegisterHostInterface(uplink.TapSwIfIndex) + if err != nil { + return errors.Wrap(err, "error configuring vpptap0 as host intf") + } + } + return nil +} + +func (s *ServiceHandler) getServiceIPs() ([]*net.IPNet, []*net.IPNet, []*net.IPNet) { + if s.cache.BGPConf == nil { + return nil, nil, nil + } + var serviceClusterIPNets []*net.IPNet + var serviceExternalIPNets []*net.IPNet + var serviceLBIPNets []*net.IPNet + for _, serviceClusterIP := range s.cache.BGPConf.ServiceClusterIPs { + _, netIP, err := net.ParseCIDR(serviceClusterIP.CIDR) + if err != nil { + s.log.Error(err) + continue + } + serviceClusterIPNets = append(serviceClusterIPNets, netIP) + } + for _, serviceExternalIP := range s.cache.BGPConf.ServiceExternalIPs { + _, netIP, err := net.ParseCIDR(serviceExternalIP.CIDR) + if err != nil { + s.log.Error(err) + continue + } + serviceExternalIPNets = append(serviceExternalIPNets, netIP) + } + for _, serviceLBIP := range s.cache.BGPConf.ServiceLoadBalancerIPs { + _, netIP, err := net.ParseCIDR(serviceLBIP.CIDR) + if err != nil { + s.log.Error(err) + continue + } + serviceLBIPNets = append(serviceLBIPNets, netIP) + } + + return serviceClusterIPNets, serviceExternalIPNets, serviceLBIPNets +} + +func (s *ServiceHandler) ServiceHandlerInit() error { + err := s.configureSnat() + if err != nil { + s.log.Errorf("Failed to configure SNAT: %v", err) + } + serviceClusterIPNets, serviceExternalIPNets, serviceLBIPNets := s.getServiceIPs() + for _, serviceIPNet := range append(serviceClusterIPNets, append(serviceExternalIPNets, serviceLBIPNets...)...) { + common.SendEvent(common.CalicoVppEvent{ + Type: common.LocalPodAddressAdded, + New: cni.NetworkPod{ContainerIP: serviceIPNet, NetworkVni: 0}, + }) + } + + err = s.vpp.CnatPurge() + if err != nil { + return err + } + return nil +} + +func getCnatBackendDstPort(servicePort *v1.ServicePort, endpointPort *discoveryv1.EndpointPort) uint16 { + targetPort := servicePort.TargetPort + if targetPort.Type == intstr.Int { + if targetPort.IntVal == 0 { + // Unset targetport + return uint16(servicePort.Port) + } + return uint16(targetPort.IntVal) + } + if endpointPort.Port != nil { + return uint16(*endpointPort.Port) + } + return 0 +} + +func getServicePortProto(proto v1.Protocol) types.IPProto { + switch proto { + case v1.ProtocolUDP: + return types.UDP + case v1.ProtocolSCTP: + return types.SCTP + case v1.ProtocolTCP: + return types.TCP + default: + return types.TCP + } +} + +func isEndpointAddressLocal(endpoint *discoveryv1.Endpoint) bool { + if endpoint != nil && endpoint.NodeName != nil && *endpoint.NodeName != *config.NodeName { + return false + } + return true +} + +func getCnatLBType(lbType lbType) types.CnatLbType { + if lbType == lbTypeMaglev || lbType == lbTypeMaglevDSR { + return types.MaglevLB + } + return types.DefaultLB +} + +func getCnatVipDstPort(servicePort *v1.ServicePort, isNodePort bool) uint16 { + if isNodePort { + return uint16(servicePort.NodePort) + } + return uint16(servicePort.Port) +} + +func endpointPortMatchesServicePort(servicePort *v1.ServicePort, endpointPort *discoveryv1.EndpointPort) bool { + if endpointPort.Name == nil { + return servicePort.Name == "" + } + return servicePort.Name == *endpointPort.Name +} + +func (s *ServiceHandler) buildCnatEntryForServicePort(servicePort *v1.ServicePort, epSlices []*discoveryv1.EndpointSlice, serviceIP net.IP, isNodePort bool, svcInfo serviceInfo, isLocalOnly bool) *types.CnatTranslateEntry { + backends := make([]types.CnatEndpointTuple, 0) + for _, epSlice := range epSlices { + for _, endpoint := range epSlice.Endpoints { + for _, endpointPort := range epSlice.Ports { + if !endpointPortMatchesServicePort(servicePort, &endpointPort) { + continue + } + if endpointPort.Port == nil && servicePort.TargetPort.Type != intstr.Int { + s.log.Warnf("null ports not supported for port %s", servicePort.Name) + continue + } + for _, endpointAddress := range endpoint.Addresses { + var flags uint8 + if !isEndpointAddressLocal(&endpoint) && isLocalOnly { + continue + } + if !isEndpointAddressLocal(&endpoint) && svcInfo.lbType == lbTypeMaglevDSR && !isNodePort { + flags |= types.CnatNoNat + } + ip := net.ParseIP(endpointAddress) + if ip == nil { + continue + } + if (ip.To4() != nil) != (serviceIP.To4() != nil) { + continue + } + backend := types.CnatEndpointTuple{ + DstEndpoint: types.CnatEndpoint{ + Port: getCnatBackendDstPort(servicePort, &endpointPort), + IP: ip, + }, + Flags: flags, + } + if isNodePort && !isEndpointAddressLocal(&endpoint) { + backend.SrcEndpoint.IP = serviceIP + } + backends = append(backends, backend) + } + break + } + } + } + + return &types.CnatTranslateEntry{ + Proto: getServicePortProto(servicePort.Protocol), + Endpoint: types.CnatEndpoint{ + Port: getCnatVipDstPort(servicePort, isNodePort), + IP: serviceIP, + }, + Backends: backends, + IsRealIP: isNodePort, + LbType: getCnatLBType(svcInfo.lbType), + HashConfig: svcInfo.hashConfig, + } +} + +func (s *ServiceHandler) OnServiceEndpointsUpdate(evt *common.ServiceEndpointsUpdate) { + s.handleServiceEndpointEvent(s.getLocalService(evt.New), s.getLocalService(evt.Old)) +} + +func (s *ServiceHandler) handleServiceEndpointEvent(service *LocalService, oldService *LocalService) { + if added, same, deleted, changed := compareEntryLists(service, oldService); changed { + s.deleteServiceEntries(deleted, oldService) + s.sameServiceEntries(same, service) + s.addServiceEntries(added, service) + } + if added, deleted, changed := compareSpecificRoutes(service, oldService); changed { + s.advertiseSpecificRoute(added, deleted) + } +} + +func (s *ServiceHandler) getNodeIP(isv6 bool) net.IP { + var nodeSpec *common.LocalNodeSpec + if spec, found := s.cache.NodeStatesByName[*config.NodeName]; found { + nodeSpec = spec + } + nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(nodeSpec) + if isv6 { + if nodeIP6 != nil { + return *nodeIP6 + } + } else { + if nodeIP4 != nil { + return *nodeIP4 + } + } + return net.IP{} +} + +func ExternalIsLocalOnly(service *v1.Service) bool { + return service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal +} + +func InternalIsLocalOnly(service *v1.Service) bool { + return service.Spec.InternalTrafficPolicy != nil && *service.Spec.InternalTrafficPolicy == v1.ServiceInternalTrafficPolicyLocal +} + +func ServiceID(meta *metav1.ObjectMeta) string { + return meta.Namespace + "/" + meta.Name +} + +func (s *ServiceHandler) getLocalService(serviceAndEndpoints *common.ServiceAndEndpoints) *LocalService { + if serviceAndEndpoints == nil { + return nil + } + return s.GetLocalService(serviceAndEndpoints.Service, serviceAndEndpoints.EndpointSlices) +} + +func (s *ServiceHandler) GetLocalService(service *v1.Service, epSlicesMap map[string]*discoveryv1.EndpointSlice) (localService *LocalService) { + if service == nil { + return nil + } + epSlices := make([]*discoveryv1.EndpointSlice, 0, len(epSlicesMap)) + for _, epSlice := range epSlicesMap { + epSlices = append(epSlices, epSlice) + } + + localService = &LocalService{ + Entries: make([]types.CnatTranslateEntry, 0), + SpecificRoutes: make([]net.IP, 0), + ServiceID: ServiceID(&service.ObjectMeta), + } + + serviceSpec := s.ParseServiceAnnotations(service.Annotations, service.Name) + clusterIPStrings := service.Spec.ClusterIPs + if len(clusterIPStrings) == 0 && service.Spec.ClusterIP != "" { + clusterIPStrings = []string{service.Spec.ClusterIP} + } + + var clusterIPs []net.IP + var nodeIPs []net.IP + for _, clusterIPString := range clusterIPStrings { + clusterIP := net.ParseIP(clusterIPString) + if clusterIP == nil { + continue + } + clusterIPs = append(clusterIPs, clusterIP) + nodeIPs = append(nodeIPs, s.getNodeIP(vpplink.IsIP6(clusterIP))) + } + + for _, servicePort := range service.Spec.Ports { + for _, clusterIP := range clusterIPs { + if !clusterIP.IsUnspecified() && len(clusterIP) > 0 { + entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, clusterIP, false /* isNodePort */, *serviceSpec, InternalIsLocalOnly(service)) + localService.Entries = append(localService.Entries, *entry) + } + } + + for _, eip := range service.Spec.ExternalIPs { + extIP := net.ParseIP(eip) + if extIP != nil && !extIP.IsUnspecified() && len(extIP) > 0 { + entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, extIP, false /* isNodePort */, *serviceSpec, ExternalIsLocalOnly(service)) + localService.Entries = append(localService.Entries, *entry) + if ExternalIsLocalOnly(service) && len(entry.Backends) > 0 { + localService.SpecificRoutes = append(localService.SpecificRoutes, extIP) + } + } + } + + for _, ingress := range service.Status.LoadBalancer.Ingress { + ingressIP := net.ParseIP(ingress.IP) + if ingressIP != nil && !ingressIP.IsUnspecified() && len(ingressIP) > 0 { + entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, ingressIP, false /* isNodePort */, *serviceSpec, ExternalIsLocalOnly(service)) + localService.Entries = append(localService.Entries, *entry) + if ExternalIsLocalOnly(service) && len(entry.Backends) > 0 { + localService.SpecificRoutes = append(localService.SpecificRoutes, ingressIP) + } + } + } + + if service.Spec.Type == v1.ServiceTypeNodePort { + for _, nodeIP := range nodeIPs { + if !nodeIP.IsUnspecified() && len(nodeIP) > 0 { + entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, nodeIP, true /* isNodePort */, *serviceSpec, false) + localService.Entries = append(localService.Entries, *entry) + } + } + } + + if service.Spec.Type == v1.ServiceTypeLoadBalancer && service.Spec.AllocateLoadBalancerNodePorts != nil && *service.Spec.AllocateLoadBalancerNodePorts { + for _, nodeIP := range nodeIPs { + if !nodeIP.IsUnspecified() && len(nodeIP) > 0 { + entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, nodeIP, true /* isNodePort */, *serviceSpec, false) + localService.Entries = append(localService.Entries, *entry) + } + } + } + } + return localService +} + +func (s *ServiceHandler) ParseServiceAnnotations(annotations map[string]string, name string) *serviceInfo { + var err []error + svc := &serviceInfo{} + for key, value := range annotations { + switch key { + case config.LBTypeAnnotation: + switch strings.ToLower(value) { + case "ecmp": + svc.lbType = lbTypeECMP + case "maglev": + svc.lbType = lbTypeMaglev + case "maglevdsr": + svc.lbType = lbTypeMaglevDSR + default: + svc.lbType = lbTypeECMP // default value + err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) + } + case config.HashConfigAnnotation: + hashConfigList := strings.Split(strings.TrimSpace(value), ",") + for _, hc := range hashConfigList { + switch strings.TrimSpace(strings.ToLower(hc)) { + case "srcport": + svc.hashConfig |= types.FlowHashSrcPort + case "dstport": + svc.hashConfig |= types.FlowHashDstPort + case "srcaddr": + svc.hashConfig |= types.FlowHashSrcIP + case "dstaddr": + svc.hashConfig |= types.FlowHashDstIP + case "iproto": + svc.hashConfig |= types.FlowHashProto + case "reverse": + svc.hashConfig |= types.FlowHashReverse + case "symmetric": + svc.hashConfig |= types.FlowHashSymetric + default: + err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) + } + } + case config.KeepOriginalPacketAnnotation: + var err1 error + svc.keepOriginalPacket, err1 = strconv.ParseBool(value) + if err1 != nil { + err = append(err, errors.Wrapf(err1, "Unknown value %s for key %s", value, key)) + } + default: + continue + } + if len(err) != 0 { + s.log.Errorf("Error parsing annotations for service %s: %s", name, err) + } + } + return svc +} + +func (s *ServiceHandler) isAddressExternalServiceIP(IPAddress net.IP) bool { + _, serviceExternalIPNets, serviceLBIPNets := s.getServiceIPs() + for _, serviceIPNet := range append(serviceExternalIPNets, serviceLBIPNets...) { + if serviceIPNet.Contains(IPAddress) { + return true + } + } + return false +} + +func (s *ServiceHandler) advertiseSpecificRoute(added []net.IP, deleted []net.IP) { + for _, specificRoute := range deleted { + if s.isAddressExternalServiceIP(specificRoute) { + common.SendEvent(common.CalicoVppEvent{ + Type: common.LocalPodAddressDeleted, + Old: cni.NetworkPod{ContainerIP: common.ToMaxLenCIDR(specificRoute), NetworkVni: 0}, + }) + s.log.Infof("Withdrawing advertisement for service specific route Addresses %+v", specificRoute) + } + } + for _, specificRoute := range added { + if s.isAddressExternalServiceIP(specificRoute) { + common.SendEvent(common.CalicoVppEvent{ + Type: common.LocalPodAddressAdded, + New: cni.NetworkPod{ContainerIP: common.ToMaxLenCIDR(specificRoute), NetworkVni: 0}, + }) + s.log.Infof("Announcing service specific route Addresses %+v", specificRoute) + } + } +} + +func (s *ServiceHandler) deleteServiceEntries(entries []types.CnatTranslateEntry, oldService *LocalService) { + for _, entry := range entries { + oldServiceState, found := s.serviceStateMap[entry.Key()] + if !found { + s.log.Infof("svc(del) key=%s Cnat entry not found", entry.Key()) + continue + } + s.log.Infof("svc(del) key=%s %s vpp-id=%d", entry.Key(), entry.String(), oldServiceState.VppID) + if oldServiceState.OwnerServiceID != oldService.ServiceID { + s.log.Infof("Cnat entry found but changed owner since") + continue + } + + err := s.vpp.CnatTranslateDel(oldServiceState.VppID) + if err != nil { + s.log.Errorf("Cnat entry delete errored %s", err) + continue + } + delete(s.serviceStateMap, entry.Key()) + } +} + +func (s *ServiceHandler) OnServiceEndpointsDelete(evt *common.ServiceEndpointsDelete) { + serviceID := ServiceID(evt.Meta) + s.deleteServiceByName(serviceID) +} + +func (s *ServiceHandler) deleteServiceByName(serviceID string) { + for key, oldServiceState := range s.serviceStateMap { + if oldServiceState.OwnerServiceID != serviceID { + continue + } + err := s.vpp.CnatTranslateDel(oldServiceState.VppID) + if err != nil { + s.log.Errorf("Cnat entry delete errored %s", err) + continue + } + delete(s.serviceStateMap, key) + } + +} + +func (s *ServiceHandler) sameServiceEntries(entries []types.CnatTranslateEntry, service *LocalService) { + for _, entry := range entries { + if serviceState, found := s.serviceStateMap[entry.Key()]; found { + serviceState.OwnerServiceID = service.ServiceID + s.serviceStateMap[entry.Key()] = serviceState + } else { + s.log.Warnf("Cnat entry not found key=%s", entry.Key()) + } + } +} + +func (s *ServiceHandler) addServiceEntries(entries []types.CnatTranslateEntry, service *LocalService) { + for _, entry := range entries { + entryID, err := s.vpp.CnatTranslateAdd(&entry) + if err != nil { + s.log.Errorf("svc(add) Error adding translation %s %s", entry.String(), err) + continue + } + s.log.Infof("svc(add) key=%s %s vpp-id=%d", entry.Key(), entry.String(), entryID) + s.serviceStateMap[entry.Key()] = ServiceState{ + OwnerServiceID: service.ServiceID, + VppID: entryID, + } + } +} + +/** + * Compares two lists of service.Entry, match them and return those + * who should be deleted (first) and then re-added. It supports update + * when the entries can be updated with the add call + */ +func compareEntryLists(service *LocalService, oldService *LocalService) (added, same, deleted []types.CnatTranslateEntry, changed bool) { + if service == nil && oldService == nil { + } else if service == nil { + deleted = oldService.Entries + } else if oldService == nil { + added = service.Entries + } else { + oldMap := make(map[string]types.CnatTranslateEntry) + newMap := make(map[string]types.CnatTranslateEntry) + for _, elem := range oldService.Entries { + oldMap[elem.Key()] = elem + } + for _, elem := range service.Entries { + newMap[elem.Key()] = elem + } + for _, oldService := range oldService.Entries { + newService, found := newMap[oldService.Key()] + /* delete if not found in current map, or if we can't just update */ + if !found { + deleted = append(deleted, oldService) + } else if newService.Equal(&oldService) == types.ShouldRecreateObj { + deleted = append(deleted, oldService) + } else { + same = append(same, oldService) + } + } + for _, newService := range service.Entries { + oldService, found := oldMap[newService.Key()] + /* add if previously not found, just skip if objects are really equal */ + if !found { + added = append(added, newService) + } else if newService.Equal(&oldService) != types.AreEqualObj { + added = append(added, newService) + } + } + } + changed = len(added)+len(deleted) > 0 + return +} + +/** + * Compares two lists of service.SpecificRoutes, match them and return those + * who should be deleted and then added. + */ +func compareSpecificRoutes(service *LocalService, oldService *LocalService) (added []net.IP, deleted []net.IP, changed bool) { + if service == nil && oldService == nil { + changed = false + } else if service == nil { + changed = true + deleted = oldService.SpecificRoutes + } else if oldService == nil { + changed = true + added = service.SpecificRoutes + } else { + added, deleted, changed = common.CompareIPList(service.SpecificRoutes, oldService.SpecificRoutes) + } + return added, deleted, changed +} diff --git a/calico-vpp-agent/services/services_test.go b/calico-vpp-agent/felix/services/services_test.go similarity index 85% rename from calico-vpp-agent/services/services_test.go rename to calico-vpp-agent/felix/services/services_test.go index e7c7f24cb..a7f1c5300 100644 --- a/calico-vpp-agent/services/services_test.go +++ b/calico-vpp-agent/felix/services/services_test.go @@ -9,7 +9,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -20,8 +22,6 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" ) // Names of integration tests arguments @@ -65,12 +65,29 @@ var _ = BeforeSuite(func() { }) +func serviceAndEndpointSlices(service *apiv1.Service, epSlices map[string]*discoveryv1.EndpointSlice) *common.ServiceAndEndpoints { + if service == nil { + return nil + } + return &common.ServiceAndEndpoints{ + Service: service, + EndpointSlices: epSlices, + } +} + +func sendServiceUpdate(handler *ServiceHandler, newService *apiv1.Service, oldService *apiv1.Service, epSlices map[string]*discoveryv1.EndpointSlice) { + handler.OnServiceEndpointsUpdate(&common.ServiceEndpointsUpdate{ + New: serviceAndEndpointSlices(newService, epSlices), + Old: serviceAndEndpointSlices(oldService, epSlices), + }) +} + var _ = Describe("Service creation functionality", func() { var ( - log *logrus.Logger - vpp *vpplink.VppLink - serviceServer *Server - uplinkSwIf uint32 + log *logrus.Logger + vpp *vpplink.VppLink + serviceHandler *ServiceHandler + uplinkSwIf uint32 ) BeforeEach(func() { @@ -90,19 +107,17 @@ var _ = Describe("Service creation functionality", func() { PhysicalNets: map[string]config.PhysicalNetwork{}, } common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) - k8sclient, err := kubernetes.NewForConfig(&rest.Config{}) - if err != nil { - log.Fatalf("cannot create k8s client %s", err) - } _, serviceip, err := net.ParseCIDR("10.96.0.1/24") config.ServiceCIDRs = &[]*net.IPNet{serviceip} - serviceServer = NewServiceServer(vpp, k8sclient, log.WithFields(logrus.Fields{"component": "services"})) _, ipv4net, err := net.ParseCIDR("1.1.1.1/32") _, ipv6net, err := net.ParseCIDR("f::f/128") - serviceServer.SetOurBGPSpec(&common.LocalNodeSpec{ + handlerCache := cache.NewCache(log.WithFields(logrus.Fields{"component": "cache"})) + handlerCache.NodeStatesByName[*config.NodeName] = &common.LocalNodeSpec{ IPv4Address: ipv4net, IPv6Address: ipv6net, - }) + } + serviceHandler = NewServiceHandler(vpp, handlerCache, log.WithFields(logrus.Fields{"component": "services"})) + serviceHandler.SetBGPConf(&calicov3.BGPConfigurationSpec{}) err = vpp.CnatSetSnatAddresses(ipv4net.IP, ipv6net.IP) Expect(err).ToNot(HaveOccurred(), "failed to configure SNAT addresses") @@ -120,7 +135,7 @@ var _ = Describe("Service creation functionality", func() { Describe("Startup config", func() { Context("Configuring snat", func() { It("Should configure snat addresses and exclude prefixes", func() { - err := serviceServer.configureSnat() + err := serviceHandler.configureSnat() Expect(err).To(BeNil()) cnatsnatoutput, err := vpp.RunCli("show cnat snat") Expect(err).ToNot(HaveOccurred(), @@ -143,7 +158,7 @@ var _ = Describe("Service creation functionality", func() { Expect(err).To(BeNil()) config.ServiceCIDRs = &[]*net.IPNet{serviceip} - err = serviceServer.configureSnat() + err = serviceHandler.configureSnat() Expect(err).To(BeNil()) cnatsnatoutput, err := vpp.RunCli("show cnat snat") @@ -154,7 +169,7 @@ var _ = Describe("Service creation functionality", func() { By("configuring empty service CIDRs") config.ServiceCIDRs = &[]*net.IPNet{} - err := serviceServer.configureSnat() + err := serviceHandler.configureSnat() Expect(err).To(BeNil()) }) }) @@ -164,7 +179,7 @@ var _ = Describe("Service creation functionality", func() { Context("handling service annotations", func() { It("should handle missing annotations gracefully", func() { By("passing empty annotations map") - svc := serviceServer.ParseServiceAnnotations(map[string]string{}, "mysvc") + svc := serviceHandler.ParseServiceAnnotations(map[string]string{}, "mysvc") Expect(svc).ToNot(BeNil()) Expect(int(svc.hashConfig)).To(Equal(0)) @@ -176,7 +191,7 @@ var _ = Describe("Service creation functionality", func() { "some.random/annotation": "value", } - svc := serviceServer.ParseServiceAnnotations(annotations, "mysvc") + svc := serviceHandler.ParseServiceAnnotations(annotations, "mysvc") Expect(svc).ToNot(BeNil()) Expect(int(svc.hashConfig)).To(Equal(0)) @@ -188,7 +203,7 @@ var _ = Describe("Service creation functionality", func() { "cni.projectcalico.org/vppHashConfig": " symmetric , dstport ", } - svc := serviceServer.ParseServiceAnnotations(annotations, "mysvc") + svc := serviceHandler.ParseServiceAnnotations(annotations, "mysvc") Expect(svc.hashConfig).To(Equal( types.FlowHashSymetric + types.FlowHashDstPort, @@ -198,7 +213,7 @@ var _ = Describe("Service creation functionality", func() { annotations := make(map[string]string) annotations["cni.projectcalico.org/vppHashConfig"] = "symmetric, iproto, dstport, srcport" annotations["cni.projectcalico.org/vppLBType"] = "maglev" - svc := serviceServer.ParseServiceAnnotations(annotations, "mysvc") + svc := serviceHandler.ParseServiceAnnotations(annotations, "mysvc") Expect(svc.keepOriginalPacket).To(BeFalse()) Expect(svc.lbType).To(Equal(lbTypeMaglev)) Expect(svc.hashConfig).To(Equal(types.FlowHashSymetric + types.FlowHashProto + types.FlowHashSrcPort + types.FlowHashDstPort)) @@ -221,7 +236,7 @@ var _ = Describe("Service creation functionality", func() { epSlicesMap := map[string]*discoveryv1.EndpointSlice{} - localService := serviceServer.GetLocalService(svc, epSlicesMap) + localService := serviceHandler.GetLocalService(svc, epSlicesMap) Expect(localService.Entries).To(BeEmpty()) }) It("should return empty backends when no endpoint slices exist for service", func() { @@ -237,7 +252,7 @@ var _ = Describe("Service creation functionality", func() { }, } - localService := serviceServer.GetLocalService(svc, map[string]*discoveryv1.EndpointSlice{}) + localService := serviceHandler.GetLocalService(svc, map[string]*discoveryv1.EndpointSlice{}) Expect(localService.Entries[0].Backends).To(BeEmpty()) }) It("should return empty backends for endpoints with empty addresses", func() { @@ -269,7 +284,7 @@ var _ = Describe("Service creation functionality", func() { }, } - localService := serviceServer.GetLocalService(svc, epSlicesMap) + localService := serviceHandler.GetLocalService(svc, epSlicesMap) Expect(localService.Entries[0].Backends).To(BeEmpty()) }) It("should support multiple backends in one endpoint slice", func() { @@ -299,7 +314,7 @@ var _ = Describe("Service creation functionality", func() { }, } - localService := serviceServer.GetLocalService(svc, epSlicesMap) + localService := serviceHandler.GetLocalService(svc, epSlicesMap) Expect(localService).ToNot(BeNil()) Expect(localService.Entries).To(HaveLen(1)) Expect(localService.Entries[0].Backends).To(HaveLen(2)) @@ -338,7 +353,7 @@ var _ = Describe("Service creation functionality", func() { }, }, } - localService := serviceServer.GetLocalService(svc, epSlicesMap) + localService := serviceHandler.GetLocalService(svc, epSlicesMap) Expect(localService).To(Equal(&LocalService{ SpecificRoutes: []net.IP{}, @@ -361,15 +376,16 @@ var _ = Describe("Service creation functionality", func() { }, }, })) - serviceServer.handleServiceEndpointEvent(localService, nil) + sendServiceUpdate(serviceHandler, svc, nil, epSlicesMap) cnattroutput, err := vpp.RunCli("show cnat translation") Expect(err).ToNot(HaveOccurred(), "failed to get cnat translations output from vpp cli") Expect(cnattroutput).To(ContainSubstring("4.4.4.4;3033 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) By("creating a service with cluster ip and external ip") + oldSvc := svc.DeepCopy() myExternalIp := "9.9.9.9" svc.Spec.ExternalIPs = []string{myExternalIp} - localService = serviceServer.GetLocalService(svc, epSlicesMap) + localService = serviceHandler.GetLocalService(svc, epSlicesMap) Expect(localService).To(Equal(&LocalService{ SpecificRoutes: []net.IP{}, @@ -407,17 +423,18 @@ var _ = Describe("Service creation functionality", func() { }, }, })) - serviceServer.handleServiceEndpointEvent(localService, nil) + sendServiceUpdate(serviceHandler, svc, oldSvc, epSlicesMap) cnattroutput, err = vpp.RunCli("show cnat translation") Expect(err).ToNot(HaveOccurred(), "failed to get cnat translations output from vpp cli") Expect(cnattroutput).To(ContainSubstring("4.4.4.4;3033 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) Expect(cnattroutput).To(ContainSubstring("9.9.9.9;3033 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) By("creating a service with cluster ip, external ip and nodeport") + oldSvc = svc.DeepCopy() svc.Spec.Type = apiv1.ServiceTypeNodePort nodePort := 9999 svc.Spec.Ports[0].NodePort = int32(nodePort) - localService = serviceServer.GetLocalService(svc, epSlicesMap) + localService = serviceHandler.GetLocalService(svc, epSlicesMap) Expect(localService).To(Equal(&LocalService{ SpecificRoutes: []net.IP{}, ServiceID: "/" + mySvc, @@ -470,7 +487,7 @@ var _ = Describe("Service creation functionality", func() { }, }, })) - serviceServer.handleServiceEndpointEvent(localService, nil) + sendServiceUpdate(serviceHandler, svc, oldSvc, epSlicesMap) cnattroutput, err = vpp.RunCli("show cnat translation") Expect(err).ToNot(HaveOccurred(), "failed to get cnat translations output from vpp cli") @@ -478,28 +495,29 @@ var _ = Describe("Service creation functionality", func() { Expect(cnattroutput).To(ContainSubstring("9.9.9.9;3033 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) Expect(cnattroutput).To(ContainSubstring("1.1.1.1;9999 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) By("updating the service to change the cluster ip") + oldSvc = svc.DeepCopy() myNewSvcIp := "5.5.5.5" svc.Spec.ClusterIPs = []string{myNewSvcIp} - newLocalService := serviceServer.GetLocalService(svc, epSlicesMap) + newLocalService := serviceHandler.GetLocalService(svc, epSlicesMap) Expect(newLocalService.Entries[0].Endpoint.IP).To(Equal(net.ParseIP(myNewSvcIp))) - serviceServer.handleServiceEndpointEvent(newLocalService, localService) + sendServiceUpdate(serviceHandler, svc, oldSvc, epSlicesMap) cnattroutput, err = vpp.RunCli("show cnat translation") Expect(err).ToNot(HaveOccurred(), "failed to get cnat translations output from vpp cli") Expect(cnattroutput).To(ContainSubstring("5.5.5.5;3033 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) Expect(cnattroutput).To(Not(ContainSubstring("4.4.4.4;3033 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051"))) By("deleting the service entries") - serviceServer.handleServiceEndpointEvent(nil, newLocalService) + sendServiceUpdate(serviceHandler, nil, svc, epSlicesMap) cnattroutput, err = vpp.RunCli("show cnat translation") Expect(cnattroutput).To(BeEmpty()) By("recreating the service") - serviceServer.handleServiceEndpointEvent(newLocalService, nil) + sendServiceUpdate(serviceHandler, svc, nil, epSlicesMap) cnattroutput, err = vpp.RunCli("show cnat translation") Expect(cnattroutput).To(ContainSubstring("5.5.5.5;3033 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) Expect(cnattroutput).To(ContainSubstring("9.9.9.9;3033 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) Expect(cnattroutput).To(ContainSubstring("1.1.1.1;9999 TCP lb:default fhc:0x9f(default)\n::;0->3.3.3.3;3051")) By("deleting the service by name") - serviceServer.deleteServiceByName("/" + mySvc) + serviceHandler.OnServiceEndpointsDelete(&common.ServiceEndpointsDelete{Meta: &svc.ObjectMeta}) cnattroutput, err = vpp.RunCli("show cnat translation") Expect(cnattroutput).To(BeEmpty()) }) diff --git a/calico-vpp-agent/services/service_handler.go b/calico-vpp-agent/services/service_handler.go deleted file mode 100644 index 845ac9db5..000000000 --- a/calico-vpp-agent/services/service_handler.go +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package services - -import ( - "net" - - v1 "k8s.io/api/core/v1" - - discoveryv1 "k8s.io/api/discovery/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -func getCnatBackendDstPort(servicePort *v1.ServicePort, endpointPort *discoveryv1.EndpointPort) int32 { - targetPort := servicePort.TargetPort - if targetPort.Type == intstr.Int { - if targetPort.IntVal == 0 { - // Unset targetport - return servicePort.Port - } else { - return targetPort.IntVal - } - } else { - if endpointPort.Port != nil { - return *endpointPort.Port - } else { - // shouldn't happen - return 0 - } - } -} - -func getServicePortProto(proto v1.Protocol) types.IPProto { - switch proto { - case v1.ProtocolUDP: - return types.UDP - case v1.ProtocolSCTP: - return types.SCTP - case v1.ProtocolTCP: - return types.TCP - default: - return types.TCP - } -} - -func isEndpointAddressLocal(endpointAddress *discoveryv1.Endpoint) bool { - if endpointAddress != nil && endpointAddress.NodeName != nil && *endpointAddress.NodeName != *config.NodeName { - return false - } - return true -} - -func getCnatLBType(lbType lbType) types.CnatLbType { - if lbType == lbTypeMaglev || lbType == lbTypeMaglevDSR { - return types.MaglevLB - } - return types.DefaultLB -} - -func getCnatVipDstPort(servicePort *v1.ServicePort, isNodePort bool) uint16 { - if isNodePort { - return uint16(servicePort.NodePort) - } - return uint16(servicePort.Port) -} - -func (s *Server) buildCnatEntryForServicePort(servicePort *v1.ServicePort, epslices []*discoveryv1.EndpointSlice, serviceIP net.IP, isNodePort bool, svcInfo serviceInfo, isLocalOnly bool) *types.CnatTranslateEntry { - backends := make([]types.CnatEndpointTuple, 0) - // Find the endpoint subset port that exposes the port we're interested in - for _, epslice := range epslices { - for _, ep := range epslice.Endpoints { - for _, endpointPort := range epslice.Ports { - if servicePort.Name == *endpointPort.Name { - if endpointPort.Port == nil { - // null ports not supported - s.log.Warnf("null ports not supported for port %s", *endpointPort.Name) - continue - } - for _, endpointAddress := range ep.Addresses { - var flags uint8 = 0 - if !isEndpointAddressLocal(&ep) && isLocalOnly { - continue - } - if !isEndpointAddressLocal(&ep) { - /* dont NAT to remote endpoints if maglevDSR unless this is a nodeport */ - if svcInfo.lbType == lbTypeMaglevDSR && !isNodePort { - flags = flags | types.CnatNoNat - } - } - ip := net.ParseIP(endpointAddress) - if ip != nil { - backend := types.CnatEndpointTuple{ - DstEndpoint: types.CnatEndpoint{ - Port: uint16(getCnatBackendDstPort(servicePort, &endpointPort)), - IP: ip, - }, - Flags: flags, - } - /* In nodeports, we need to sNAT when endpoint is not local to have a symmetric traffic */ - if isNodePort && !isEndpointAddressLocal(&ep) { - backend.SrcEndpoint.IP = serviceIP - } - /* Only append backend if it has the same address family as the service IP */ - /* we don't nat v4 to v6 and vice versa */ - if (ip.To4() != nil) == (serviceIP.To4() != nil) { - backends = append(backends, backend) - } - } - } - break - } - } - } - } - - return &types.CnatTranslateEntry{ - Proto: getServicePortProto(servicePort.Protocol), - Endpoint: types.CnatEndpoint{ - Port: getCnatVipDstPort(servicePort, isNodePort), - IP: serviceIP, - }, - Backends: backends, - IsRealIP: isNodePort, - LbType: getCnatLBType(svcInfo.lbType), - HashConfig: svcInfo.hashConfig, - } -} - -func (s *Server) GetLocalService(service *v1.Service, epSlicesMap map[string]*discoveryv1.EndpointSlice) (localService *LocalService) { - epSlices := []*discoveryv1.EndpointSlice{} - for _, epslice := range epSlicesMap { - epSlices = append(epSlices, epslice) - } - localService = &LocalService{ - Entries: make([]types.CnatTranslateEntry, 0), - SpecificRoutes: make([]net.IP, 0), - ServiceID: objectID(&service.ObjectMeta), /* ip.ObjectMeta should yield the same id */ - } - - serviceSpec := s.ParseServiceAnnotations(service.Annotations, service.Name) - var clusterIPs []net.IP - var nodeIPs []net.IP - for _, cip := range service.Spec.ClusterIPs { - clusterIPs = append(clusterIPs, net.ParseIP(cip)) - nodeIPs = append(nodeIPs, s.getNodeIP(vpplink.IsIP6(net.ParseIP(cip)))) - } - for _, servicePort := range service.Spec.Ports { - for _, cip := range clusterIPs { - if !cip.IsUnspecified() && len(cip) > 0 { - entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, cip, false /* isNodePort */, *serviceSpec, InternalIsLocalOnly(service)) - localService.Entries = append(localService.Entries, *entry) - } - } - - for _, eip := range service.Spec.ExternalIPs { - extIP := net.ParseIP(eip) - if !extIP.IsUnspecified() && len(extIP) > 0 { - entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, extIP, false /* isNodePort */, *serviceSpec, ExternalIsLocalOnly(service)) - localService.Entries = append(localService.Entries, *entry) - if ExternalIsLocalOnly(service) && len(entry.Backends) > 0 { - localService.SpecificRoutes = append(localService.SpecificRoutes, extIP) - } - } - } - - for _, ingress := range service.Status.LoadBalancer.Ingress { - ingressIP := net.ParseIP(ingress.IP) - if !ingressIP.IsUnspecified() && len(ingressIP) > 0 { - entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, ingressIP, false /* isNodePort */, *serviceSpec, ExternalIsLocalOnly(service)) - localService.Entries = append(localService.Entries, *entry) - if ExternalIsLocalOnly(service) && len(entry.Backends) > 0 { - localService.SpecificRoutes = append(localService.SpecificRoutes, ingressIP) - } - } - } - - if service.Spec.Type == v1.ServiceTypeNodePort { - for _, nip := range nodeIPs { - if !nip.IsUnspecified() && len(nip) > 0 { - entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, nip, true /* isNodePort */, *serviceSpec, false) - localService.Entries = append(localService.Entries, *entry) - } - } - } - - // Create NodePort for external LB - // Note: type=LoadBalancer only makes sense on cloud providers which support external load balancers and the actual - // creation of the load balancer happens asynchronously. - if service.Spec.Type == v1.ServiceTypeLoadBalancer && *service.Spec.AllocateLoadBalancerNodePorts { - for _, nip := range nodeIPs { - if !nip.IsUnspecified() && len(nip) > 0 { - entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, nip, true /* isNodePort */, *serviceSpec, false) - localService.Entries = append(localService.Entries, *entry) - } - } - } - } - return -} - -func (s *Server) isAddressExternalServiceIP(IPAddress net.IP) bool { - _, serviceExternalIPNets, serviceLBIPNets := s.getServiceIPs() - for _, serviceIPNet := range append(serviceExternalIPNets, serviceLBIPNets...) { - if serviceIPNet.Contains(IPAddress) { - return true - } - } - return false -} - -func (s *Server) advertiseSpecificRoute(added []net.IP, deleted []net.IP) { - for _, specificRoute := range deleted { - if s.isAddressExternalServiceIP(specificRoute) { - common.SendEvent(common.CalicoVppEvent{ - Type: common.LocalPodAddressDeleted, - Old: cni.NetworkPod{ContainerIP: common.ToMaxLenCIDR(specificRoute), NetworkVni: 0}, - }) - s.log.Infof("Withdrawing advertisement for service specific route Addresses %+v", specificRoute) - } - } - for _, specificRoute := range added { - if s.isAddressExternalServiceIP(specificRoute) { - common.SendEvent(common.CalicoVppEvent{ - Type: common.LocalPodAddressAdded, - New: cni.NetworkPod{ContainerIP: common.ToMaxLenCIDR(specificRoute), NetworkVni: 0}, - }) - s.log.Infof("Announcing service specific route Addresses %+v", specificRoute) - } - } -} - -func (s *Server) deleteServiceEntry(key, serviceID string) { - if _, found := s.serviceIDByKey[key]; !found { - s.log.Warnf("svc(del) entry %s not found", key) - return - } else if s.serviceIDByKey[key] != serviceID { - // do nothing in vpp, this service is not activated - s.log.Debugf("svc(del) entry %s not created in vpp for service %s", key, serviceID) - } else if len(s.cnatEntryByKeyAndSid[key]) == 1 { - err := s.vpp.CnatTranslateDel(s.cnatEntryByKeyAndSid[key][serviceID].vppID) - if err != nil { - s.log.Errorf("Cnat entry delete errored %s", err) - } - delete(s.serviceIDByKey, key) - } else if len(s.cnatEntryByKeyAndSid[key]) > 1 { - // the entry is referenced by another service, recreate the lexicographically smallest service entry - s.log.Warnf("svc(del) entry %s was referenced by multiple services", key) - var chosenService string - for svc := range s.cnatEntryByKeyAndSid[key] { - if (chosenService == "" || svc < chosenService) && chosenService != serviceID { - chosenService = svc - } - } - s.log.Infof("svc(re-add) adding service %s for entry %s", chosenService, key) - entryID, err := s.vpp.CnatTranslateAdd(&s.cnatEntryByKeyAndSid[key][chosenService].entry) - if err != nil { - s.log.Errorf("svc(add) Error adding translation %s %s", s.cnatEntryByKeyAndSid[key][chosenService].entry.String(), err) - } - s.cnatEntryByKeyAndSid[key][chosenService].vppID = entryID - s.serviceIDByKey[key] = chosenService - } else { - panic("this should not happen") - } - delete(s.cnatEntryByKeyAndSid[key], serviceID) - delete(s.cnatEntryBySidAndKey[serviceID], key) - // cleanup maps if empty - if len(s.cnatEntryByKeyAndSid[key]) == 0 { - delete(s.cnatEntryByKeyAndSid, key) - } - if len(s.cnatEntryBySidAndKey[serviceID]) == 0 { - delete(s.cnatEntryBySidAndKey, serviceID) - } -} - -func (s *Server) deleteServiceEntries(entries []types.CnatTranslateEntry, oldService *LocalService) { - for _, entry := range entries { - s.deleteServiceEntry(entry.Key(), oldService.ServiceID) - } -} - -func (s *Server) deleteServiceByName(serviceID string) { - s.lock.Lock() - defer s.lock.Unlock() - for key := range s.cnatEntryBySidAndKey[serviceID] { - s.deleteServiceEntry(key, serviceID) - } -} - -func (s *Server) addServiceEntries(entries []types.CnatTranslateEntry, service *LocalService) { - for _, entry := range entries { - entryID, err := s.vpp.CnatTranslateAdd(&entry) - if err != nil { - s.log.Errorf("svc(add) Error adding translation %s %s", entry.String(), err) - continue - } - if _, found := s.cnatEntryBySidAndKey[service.ServiceID]; !found { - s.log.Infof("svc(add) adding service id %s to cache", service.ServiceID) - s.cnatEntryBySidAndKey[service.ServiceID] = make(map[string]*cnatEntry) - } - if _, found := s.cnatEntryByKeyAndSid[entry.Key()]; !found { - s.log.Infof("svc(add) adding entry key=%s to cache", entry.Key()) - s.cnatEntryByKeyAndSid[entry.Key()] = make(map[string]*cnatEntry) - } - s.log.Infof("svc(add) adding service %s to entry key=%s cache", service.ServiceID, entry.Key()) - s.cnatEntryByKeyAndSid[entry.Key()][service.ServiceID] = &cnatEntry{ - entry: entry, - vppID: entryID, - } - s.cnatEntryBySidAndKey[service.ServiceID][entry.Key()] = &cnatEntry{ - entry: entry, - vppID: entryID, - } - s.serviceIDByKey[entry.Key()] = service.ServiceID - if len(s.cnatEntryByKeyAndSid[entry.Key()]) > 1 { - s.log.Warnf("svc(add) entry %s is referenced by multiple services; overriding previous value and using the latest", entry.Key()) - for svc := range s.cnatEntryByKeyAndSid[entry.Key()] { - if svc != service.ServiceID { - s.cnatEntryByKeyAndSid[entry.Key()][svc].vppID = ^uint32(0) - } - } - } - } -} diff --git a/calico-vpp-agent/services/service_server.go b/calico-vpp-agent/services/service_server.go deleted file mode 100644 index 6fa541315..000000000 --- a/calico-vpp-agent/services/service_server.go +++ /dev/null @@ -1,549 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package services - -import ( - "fmt" - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" - v1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -/** - * Service descriptions from the API are resolved into - * slices of LocalService, this allows to diffs between - * previous & expected state in VPP - */ -type LocalService struct { - Entries []types.CnatTranslateEntry - SpecificRoutes []net.IP - ServiceID string -} - -/** - * Store VPP's state in a map [CnatTranslateEntry.Key()]->map[serviceId]->cnatEntry - */ -type cnatEntry struct { - entry types.CnatTranslateEntry - vppID uint32 -} - -type Server struct { - log *logrus.Entry - vpp *vpplink.VppLink - endpointslicesStore cache.Store - serviceStore cache.Store - serviceInformer cache.Controller - endpointslicesInformer cache.Controller - - lock sync.Mutex /* protects handleServiceEndpointEvent(s)/Serve */ - /* and protects the endpointSlicesByService map */ - - BGPConf *calicov3.BGPConfigurationSpec - nodeBGPSpec *common.LocalNodeSpec - - // map entry key to a map of service id to entry in vpp - cnatEntryByKeyAndSid map[string]map[string]*cnatEntry - // map service id to a map of entry key to entry in vpp - cnatEntryBySidAndKey map[string]map[string]*cnatEntry - // map entry key to the active k8s service id - serviceIDByKey map[string]string - // cache of all endpoint slices, by service name - endpointSlicesByService map[string]map[string]*discoveryv1.EndpointSlice - endpointSlices map[string]*discoveryv1.EndpointSlice - - t tomb.Tomb -} - -func (s *Server) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { - s.BGPConf = bgpConf -} - -func (s *Server) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { - s.nodeBGPSpec = nodeBGPSpec -} - -func (s *Server) ParseServiceAnnotations(annotations map[string]string, name string) *serviceInfo { - var err []error - svc := &serviceInfo{} - for key, value := range annotations { - switch key { - case config.LBTypeAnnotation: - switch strings.ToLower(value) { - case "ecmp": - svc.lbType = lbTypeECMP - case "maglev": - svc.lbType = lbTypeMaglev - case "maglevdsr": - svc.lbType = lbTypeMaglevDSR - default: - svc.lbType = lbTypeECMP // default value - err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) - } - case config.HashConfigAnnotation: - hashConfigList := strings.Split(strings.TrimSpace(value), ",") - for _, hc := range hashConfigList { - switch strings.TrimSpace(strings.ToLower(hc)) { - case "srcport": - svc.hashConfig |= types.FlowHashSrcPort - case "dstport": - svc.hashConfig |= types.FlowHashDstPort - case "srcaddr": - svc.hashConfig |= types.FlowHashSrcIP - case "dstaddr": - svc.hashConfig |= types.FlowHashDstIP - case "iproto": - svc.hashConfig |= types.FlowHashProto - case "reverse": - svc.hashConfig |= types.FlowHashReverse - case "symmetric": - svc.hashConfig |= types.FlowHashSymetric - default: - err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) - } - } - case config.KeepOriginalPacketAnnotation: - var err1 error - svc.keepOriginalPacket, err1 = strconv.ParseBool(value) - if err1 != nil { - err = append(err, errors.Wrapf(err1, "Unknown value %s for key %s", value, key)) - } - default: - continue - } - if len(err) != 0 { - s.log.Errorf("Error parsing annotations for service %s: %s", name, err) - } - } - return svc -} - -func (s *Server) resolveLocalServiceFromService(service *v1.Service) *LocalService { - if service == nil { - return nil - } - return s.GetLocalService(service, s.endpointSlicesByService[objectID(&service.ObjectMeta)]) -} - -func NewServiceServer(vpp *vpplink.VppLink, k8sclient *kubernetes.Clientset, log *logrus.Entry) *Server { - server := Server{ - vpp: vpp, - log: log, - cnatEntryByKeyAndSid: make(map[string]map[string]*cnatEntry), - cnatEntryBySidAndKey: make(map[string]map[string]*cnatEntry), - serviceIDByKey: make(map[string]string), - endpointSlicesByService: make(map[string]map[string]*discoveryv1.EndpointSlice), - endpointSlices: make(map[string]*discoveryv1.EndpointSlice), - } - serviceStore, serviceInformer := cache.NewInformerWithOptions( - cache.InformerOptions{ - ListerWatcher: cache.NewListWatchFromClient( - k8sclient.CoreV1().RESTClient(), - "services", - "", - fields.Everything(), - ), - ObjectType: &v1.Service{}, - ResyncPeriod: 60 * time.Second, - Handler: cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - service, ok := obj.(*v1.Service) - if !ok { - panic("wrong type for obj, not *v1.Service") - } - server.lock.Lock() - localService := server.resolveLocalServiceFromService(service) - server.handleServiceEndpointEvent(localService, nil) - server.lock.Unlock() - }, - UpdateFunc: func(old interface{}, obj interface{}) { - service, ok := obj.(*v1.Service) - if !ok { - panic("wrong type for obj, not *v1.Service") - } - oldService, ok := old.(*v1.Service) - if !ok { - panic("wrong type for old, not *v1.Service") - } - server.lock.Lock() - oldLocalService := server.resolveLocalServiceFromService(oldService) - localService := server.resolveLocalServiceFromService(service) - server.handleServiceEndpointEvent(localService, oldLocalService) - server.lock.Unlock() - }, - DeleteFunc: func(obj interface{}) { - switch value := obj.(type) { - case cache.DeletedFinalStateUnknown: - service, ok := value.Obj.(*v1.Service) - if !ok { - panic(fmt.Sprintf("obj.(cache.DeletedFinalStateUnknown).Obj not a (*v1.Service) %v", obj)) - } - server.deleteServiceByName(objectID(&service.ObjectMeta)) - case *v1.Service: - server.deleteServiceByName(objectID(&value.ObjectMeta)) - default: - log.Errorf("unknown type in service deleteFunction %v", obj) - } - }, - }, - }, - ) - - // ---- Watch EndpointSlices (IPv4 + IPv6) ---- - endpointslicesStore, endpointslicesInformer := cache.NewInformerWithOptions( - cache.InformerOptions{ - ListerWatcher: cache.NewListWatchFromClient( - k8sclient.DiscoveryV1().RESTClient(), - "endpointslices", - "", - fields.Everything(), - ), - ObjectType: &discoveryv1.EndpointSlice{}, - ResyncPeriod: 60 * time.Second, - Handler: cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - newEps, ok := obj.(*discoveryv1.EndpointSlice) - if !ok { - panic("wrong type for obj, not *discoveryv1.EndpointSlice") - } - svcKey := serviceName(newEps) - server.lock.Lock() - if len(server.endpointSlicesByService[svcKey]) == 0 { - server.endpointSlicesByService[svcKey] = make(map[string]*discoveryv1.EndpointSlice) - } - server.endpointSlicesByService[svcKey][objectID(&newEps.ObjectMeta)] = newEps - server.endpointSlices[objectID(&newEps.ObjectMeta)] = newEps - svc := server.getServiceFromStore(svcKey) - if svc != nil { - server.handleServiceEndpointEvent(server.GetLocalService(svc, server.endpointSlicesByService[svcKey]), nil) - } - server.lock.Unlock() - }, - UpdateFunc: func(_, new interface{}) { - newEps, ok := new.(*discoveryv1.EndpointSlice) - if !ok { - panic("wrong type for obj, not *discoveryv1.EndpointSlice") - } - svcKey := serviceName(newEps) - server.lock.Lock() - if len(server.endpointSlicesByService[svcKey]) == 0 { - server.endpointSlicesByService[svcKey] = make(map[string]*discoveryv1.EndpointSlice) - } - svc := server.getServiceFromStore(svcKey) - if svc != nil { - oldLocal := server.GetLocalService( - svc, - server.endpointSlicesByService[svcKey], - ) - server.endpointSlicesByService[svcKey][objectID(&newEps.ObjectMeta)] = newEps - server.endpointSlices[objectID(&newEps.ObjectMeta)] = newEps - newLocal := server.GetLocalService( - svc, - server.endpointSlicesByService[svcKey], - ) - server.handleServiceEndpointEvent(newLocal, oldLocal) - } else { - server.log.Debugf("Trying to update endpointslice, Service %s not found", svcKey) - server.endpointSlicesByService[svcKey][objectID(&newEps.ObjectMeta)] = newEps - server.endpointSlices[objectID(&newEps.ObjectMeta)] = newEps - } - server.lock.Unlock() - }, - DeleteFunc: func(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - panic("wrong type for obj, could not get DeletionHandlingMetaNamespaceKeyFunc") - } else { - oldEps, ok := server.endpointSlices[key] - if ok { - svcKey := serviceName(oldEps) - server.lock.Lock() - svc := server.getServiceFromStore(svcKey) - if svc != nil { - oldLocal := server.GetLocalService( - svc, - server.endpointSlicesByService[svcKey], - ) - delete(server.endpointSlicesByService[svcKey], objectID(&oldEps.ObjectMeta)) - if len(server.endpointSlicesByService[svcKey]) == 0 { - delete(server.endpointSlicesByService, svcKey) - } - newLocal := server.GetLocalService( - svc, - server.endpointSlicesByService[svcKey], - ) - server.handleServiceEndpointEvent(oldLocal, newLocal) - } else { - server.log.Debugf("Service %s already gone", svcKey) - delete(server.endpointSlicesByService[svcKey], objectID(&oldEps.ObjectMeta)) - if len(server.endpointSlicesByService[svcKey]) == 0 { - delete(server.endpointSlicesByService, svcKey) - } - } - delete(server.endpointSlices, key) - server.lock.Unlock() - } else { - server.log.Debugf("endpointslice %s not found in map", key) - } - } - }, - }}, - ) - - server.endpointslicesStore = endpointslicesStore - server.serviceStore = serviceStore - server.serviceInformer = serviceInformer - server.endpointslicesInformer = endpointslicesInformer - - return &server -} - -func (s *Server) getNodeIP(isv6 bool) net.IP { - nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec) - if isv6 { - if nodeIP6 != nil { - return *nodeIP6 - } - } else { - if nodeIP4 != nil { - return *nodeIP4 - } - } - return net.IP{} -} - -func ExternalIsLocalOnly(service *v1.Service) bool { - return service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal -} - -func InternalIsLocalOnly(service *v1.Service) bool { - return *service.Spec.InternalTrafficPolicy == v1.ServiceInternalTrafficPolicyLocal -} - -func objectID(meta *metav1.ObjectMeta) string { - return meta.Namespace + "/" + meta.Name -} - -func (s *Server) configureSnat() (err error) { - nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec) - if nodeIP6 != nil { - err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP6)) - if err != nil { - s.log.Errorf("Failed to add SNAT %s %v", common.FullyQualified(*nodeIP6), err) - } - } - if nodeIP4 != nil { - err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP4)) - if err != nil { - s.log.Errorf("Failed to add SNAT %s %v", common.FullyQualified(*nodeIP4), err) - } - } - for _, serviceCIDR := range *config.ServiceCIDRs { - err = s.vpp.CnatAddSnatPrefix(serviceCIDR) - if err != nil { - s.log.Errorf("Failed to Add Service CIDR %s %v", serviceCIDR, err) - } - } - err = s.vpp.SetK8sSnatPolicy() - if err != nil { - return errors.Wrap(err, "Error configuring cnat source policy") - } - for _, uplink := range common.VppManagerInfo.UplinkStatuses { - // register vpptap0 - err = s.vpp.RegisterPodInterface(uplink.TapSwIfIndex) - if err != nil { - return errors.Wrap(err, "error configuring vpptap0 as pod intf") - } - - err = s.vpp.RegisterHostInterface(uplink.TapSwIfIndex) - if err != nil { - return errors.Wrap(err, "error configuring vpptap0 as host intf") - } - } - - return nil -} - -func serviceName(es *discoveryv1.EndpointSlice) string { - svc := es.Labels["kubernetes.io/service-name"] - name := fmt.Sprintf("%s/%s", es.Namespace, svc) - return name -} - -func (s *Server) getServiceFromStore(key string) *v1.Service { - value, found, err := s.serviceStore.GetByKey(key) - if err != nil { - s.log.Errorf("Error getting service %s: %v", key, err) - return nil - } - if !found { - s.log.Debugf("Service %s not found", key) - return nil - } - service, ok := value.(*v1.Service) - if !ok { - panic("s.serviceStore.GetByKey did not return value of type *v1.Service") - } - return service -} - -/** - * Compares two lists of service.Entry, match them and return those - * who should be deleted (first) and then re-added. It supports update - * when the entries can be updated with the add call - */ -func compareEntryLists(service *LocalService, oldService *LocalService) (added, deleted []types.CnatTranslateEntry, changed bool) { - if service == nil && oldService == nil { - } else if service == nil { - deleted = oldService.Entries - } else if oldService == nil { - added = service.Entries - } else { - oldMap := make(map[string]types.CnatTranslateEntry) - newMap := make(map[string]types.CnatTranslateEntry) - for _, elem := range oldService.Entries { - oldMap[elem.Key()] = elem - } - for _, elem := range service.Entries { - newMap[elem.Key()] = elem - } - for _, oldService := range oldService.Entries { - newService, found := newMap[oldService.Key()] - /* delete if not found in current map, or if we can't just update */ - if !found { - deleted = append(deleted, oldService) - } else if newService.Equal(&oldService) == types.ShouldRecreateObj { - deleted = append(deleted, oldService) - } - } - for _, newService := range service.Entries { - oldService, found := oldMap[newService.Key()] - /* add if previously not found, just skip if objects are really equal */ - if !found { - added = append(added, newService) - } else if newService.Equal(&oldService) != types.AreEqualObj { - added = append(added, newService) - } - } - } - changed = len(added)+len(deleted) > 0 - return -} - -/** - * Compares two lists of service.SpecificRoutes, match them and return those - * who should be deleted and then added. - */ -func compareSpecificRoutes(service *LocalService, oldService *LocalService) (added []net.IP, deleted []net.IP, changed bool) { - if service == nil && oldService == nil { - changed = false - } else if service == nil { - changed = true - deleted = oldService.SpecificRoutes - } else if oldService == nil { - changed = true - added = service.SpecificRoutes - } else { - added, deleted, changed = common.CompareIPList(service.SpecificRoutes, oldService.SpecificRoutes) - } - return added, deleted, changed -} - -func (s *Server) handleServiceEndpointEvent(service *LocalService, oldService *LocalService) { - if added, deleted, changed := compareEntryLists(service, oldService); changed { - s.deleteServiceEntries(deleted, oldService) - s.addServiceEntries(added, service) - } - if added, deleted, changed := compareSpecificRoutes(service, oldService); changed { - s.advertiseSpecificRoute(added, deleted) - } -} - -func (s *Server) getServiceIPs() ([]*net.IPNet, []*net.IPNet, []*net.IPNet) { - var serviceClusterIPNets []*net.IPNet - var serviceExternalIPNets []*net.IPNet - var serviceLBIPNets []*net.IPNet - for _, serviceClusterIP := range s.BGPConf.ServiceClusterIPs { - _, netIP, err := net.ParseCIDR(serviceClusterIP.CIDR) - if err != nil { - s.log.Error(err) - continue - } - serviceClusterIPNets = append(serviceClusterIPNets, netIP) - } - for _, serviceExternalIP := range s.BGPConf.ServiceExternalIPs { - _, netIP, err := net.ParseCIDR(serviceExternalIP.CIDR) - if err != nil { - s.log.Error(err) - continue - } - serviceExternalIPNets = append(serviceExternalIPNets, netIP) - } - for _, serviceLBIP := range s.BGPConf.ServiceLoadBalancerIPs { - _, netIP, err := net.ParseCIDR(serviceLBIP.CIDR) - if err != nil { - s.log.Error(err) - continue - } - serviceLBIPNets = append(serviceLBIPNets, netIP) - } - - return serviceClusterIPNets, serviceExternalIPNets, serviceLBIPNets -} - -func (s *Server) ServeService(t *tomb.Tomb) error { - err := s.configureSnat() - if err != nil { - s.log.Errorf("Failed to configure SNAT: %v", err) - } - serviceClusterIPNets, serviceExternalIPNets, serviceLBIPNets := s.getServiceIPs() - for _, serviceIPNet := range append(serviceClusterIPNets, append(serviceExternalIPNets, serviceLBIPNets...)...) { - common.SendEvent(common.CalicoVppEvent{ - Type: common.LocalPodAddressAdded, - New: cni.NetworkPod{ContainerIP: serviceIPNet, NetworkVni: 0}, - }) - } - - if *config.GetCalicoVppDebug().ServicesEnabled { - s.t.Go(func() error { s.serviceInformer.Run(t.Dying()); return nil }) - s.t.Go(func() error { s.endpointslicesInformer.Run(t.Dying()); return nil }) - } - - <-s.t.Dying() - - s.log.Warn("Service Server returned") - - return nil -} diff --git a/calico-vpp-agent/tests/mocks/ipam.go b/calico-vpp-agent/tests/mocks/ipam.go deleted file mode 100644 index 0b4e4458e..000000000 --- a/calico-vpp-agent/tests/mocks/ipam.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2022 Cisco and/or its affiliates. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mocks - -import ( - "fmt" - "net" - - "github.com/projectcalico/calico/felix/proto" - "gopkg.in/tomb.v2" -) - -// IpamCacheStub is stub implementation of watchers.IpamCache. -type IpamCacheStub struct { - ipPools map[string]*proto.IPAMPoolUpdate -} - -// NewIpamCacheStub creates new IpamCacheStub instance -func NewIpamCacheStub() *IpamCacheStub { - return &IpamCacheStub{ - ipPools: make(map[string]*proto.IPAMPoolUpdate), - } -} - -// GetPrefixIPPool returns cached IPPools for given prefixes for testing purposes. If no such IPPool exists, -// it is created. This function never runs out of IPPools -func (s *IpamCacheStub) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { - // get cached IPPool - ipPool, found := s.ipPools[prefix.String()] - if found { - return ipPool.Pool - } - - // create new IPPool and cache it - ipPool = &proto.IPAMPoolUpdate{ - Id: fmt.Sprintf("ippool-for-testing-%s", prefix.String()), - Pool: &proto.IPAMPool{ - Cidr: prefix.String(), - }, - } - /*ipPool = &calicov3.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("ippool-for-testing-%s", prefix.String()), - }, - Spec: calicov3.IPPoolSpec{ - CIDR: prefix.String(), - }, - }*/ - s.ipPools[prefix.String()] = ipPool - return ipPool.Pool -} - -func (s *IpamCacheStub) SyncIPAM(t *tomb.Tomb) error { - panic("not implemented") -} - -func (s *IpamCacheStub) WaitReady() { - panic("not implemented") -} - -func (s *IpamCacheStub) AddPrefixIPPool(prefix *net.IPNet, ipPool *proto.IPAMPoolUpdate) { - s.ipPools[prefix.String()] = ipPool -} diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 68669414a..bed88c339 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -496,8 +496,7 @@ func ConfigureBGPNodeIPAddresses(cache *cache.Cache) { cache.NodeStatesByName[*config.NodeName] = nodeSpec } -// AddIPPoolForCalicoClient is convenience function for adding IPPool to mocked Calico IPAM Stub used -// in Calico client stub. This function doesn't set anything for the watchers.IpamCache implementation. +// AddIPPoolForCalicoClient is a convenience function for adding an IPPool to the mocked Calico client. func AddIPPoolForCalicoClient(client *calico.CalicoClientStub, poolName string, poolCIRD string) ( *apiv3.IPPool, error) { return client.IPPoolsStub.Create(context.Background(), &apiv3.IPPool{ diff --git a/calico-vpp-agent/watchers/net_watcher.go b/calico-vpp-agent/watchers/net_watcher.go index d185d20da..710182704 100644 --- a/calico-vpp-agent/watchers/net_watcher.go +++ b/calico-vpp-agent/watchers/net_watcher.go @@ -113,9 +113,6 @@ func (w *NetWatcher) resyncAndCreateWatchers() error { } } w.InSync <- 1 - common.SendEvent(common.CalicoVppEvent{ - Type: common.NetsSynced, - }) w.currentWatchRevisionNet = netList.ResourceVersion w.currentWatchRevisionNad = nadList.ResourceVersion } diff --git a/calico-vpp-agent/watchers/service_watcher.go b/calico-vpp-agent/watchers/service_watcher.go new file mode 100644 index 000000000..090e6a92b --- /dev/null +++ b/calico-vpp-agent/watchers/service_watcher.go @@ -0,0 +1,267 @@ +// Copyright (C) 2026 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watchers + +import ( + "fmt" + "time" + + "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" + v1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/services" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type Server struct { + log *logrus.Entry + eventChan chan any + + endpointSlicesStore cache.Store + serviceStore cache.Store + serviceInformer cache.Controller + endpointSlicesInformer cache.Controller + + endpointSlicesByService map[string]map[string]*discoveryv1.EndpointSlice + endpointSlices map[string]*discoveryv1.EndpointSlice + + t tomb.Tomb +} + +func cloneEndpointSliceMap(epSlices map[string]*discoveryv1.EndpointSlice) map[string]*discoveryv1.EndpointSlice { + if len(epSlices) == 0 { + return nil + } + clone := make(map[string]*discoveryv1.EndpointSlice, len(epSlices)) + for key, epSlice := range epSlices { + clone[key] = epSlice + } + return clone +} + +func serviceName(es *discoveryv1.EndpointSlice) string { + return es.Namespace + "/" + es.Labels[discoveryv1.LabelServiceName] +} + +func objectID(meta *metav1.ObjectMeta) string { + return meta.Namespace + "/" + meta.Name +} + +func (s *Server) resolveLocalServiceFromService(service *v1.Service) *common.ServiceAndEndpoints { + if service == nil { + return nil + } + return &common.ServiceAndEndpoints{ + Service: service, + EndpointSlices: cloneEndpointSliceMap(s.endpointSlicesByService[services.ServiceID(&service.ObjectMeta)]), + } +} + +func (s *Server) resolveLocalServiceFromEndpointSlices(svcKey string) *common.ServiceAndEndpoints { + service := s.findMatchingService(svcKey) + if service == nil { + s.log.Debugf("svc() no svc found for endpointslices=%s", svcKey) + return nil + } + return &common.ServiceAndEndpoints{ + Service: service, + EndpointSlices: cloneEndpointSliceMap(s.endpointSlicesByService[svcKey]), + } +} + +func NewServiceServer(eventChan chan any, k8sclient *kubernetes.Clientset, log *logrus.Entry) *Server { + server := &Server{ + log: log, + eventChan: eventChan, + endpointSlicesByService: make(map[string]map[string]*discoveryv1.EndpointSlice), + endpointSlices: make(map[string]*discoveryv1.EndpointSlice), + } + serviceStore, serviceInformer := cache.NewInformerWithOptions( + cache.InformerOptions{ + ListerWatcher: cache.NewListWatchFromClient( + k8sclient.CoreV1().RESTClient(), + "services", + "", + fields.Everything(), + ), + ObjectType: &v1.Service{}, + ResyncPeriod: 60 * time.Second, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + service, ok := obj.(*v1.Service) + if !ok { + panic("wrong type for obj, not *v1.Service") + } + eventChan <- &common.ServiceEndpointsUpdate{ + New: server.resolveLocalServiceFromService(service), + } + }, + UpdateFunc: func(old interface{}, obj interface{}) { + service, ok := obj.(*v1.Service) + if !ok { + panic("wrong type for obj, not *v1.Service") + } + oldService, ok := old.(*v1.Service) + if !ok { + panic("wrong type for old, not *v1.Service") + } + eventChan <- &common.ServiceEndpointsUpdate{ + Old: server.resolveLocalServiceFromService(oldService), + New: server.resolveLocalServiceFromService(service), + } + }, + DeleteFunc: func(obj interface{}) { + switch value := obj.(type) { + case cache.DeletedFinalStateUnknown: + service, ok := value.Obj.(*v1.Service) + if !ok { + panic(fmt.Sprintf("obj.(cache.DeletedFinalStateUnknown).Obj not a (*v1.Service) %v", obj)) + } + eventChan <- &common.ServiceEndpointsDelete{ + Meta: &service.ObjectMeta, + } + case *v1.Service: + eventChan <- &common.ServiceEndpointsDelete{ + Meta: &value.ObjectMeta, + } + default: + log.Errorf("unknown type in service deleteFunction %v", obj) + } + }, + }, + }, + ) + + endpointSlicesStore, endpointSlicesInformer := cache.NewInformerWithOptions( + cache.InformerOptions{ + ListerWatcher: cache.NewListWatchFromClient( + k8sclient.DiscoveryV1().RESTClient(), + "endpointslices", + "", + fields.Everything(), + ), + ObjectType: &discoveryv1.EndpointSlice{}, + ResyncPeriod: 60 * time.Second, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + epSlice, ok := obj.(*discoveryv1.EndpointSlice) + if !ok { + panic("wrong type for obj, not *discoveryv1.EndpointSlice") + } + svcKey := serviceName(epSlice) + oldLocalService := server.resolveLocalServiceFromEndpointSlices(svcKey) + if len(server.endpointSlicesByService[svcKey]) == 0 { + server.endpointSlicesByService[svcKey] = make(map[string]*discoveryv1.EndpointSlice) + } + server.endpointSlicesByService[svcKey][objectID(&epSlice.ObjectMeta)] = epSlice + server.endpointSlices[objectID(&epSlice.ObjectMeta)] = epSlice + eventChan <- &common.ServiceEndpointsUpdate{ + Old: oldLocalService, + New: server.resolveLocalServiceFromEndpointSlices(svcKey), + } + }, + UpdateFunc: func(old interface{}, obj interface{}) { + epSlice, ok := obj.(*discoveryv1.EndpointSlice) + if !ok { + panic("wrong type for obj, not *discoveryv1.EndpointSlice") + } + svcKey := serviceName(epSlice) + oldLocalService := server.resolveLocalServiceFromEndpointSlices(svcKey) + if len(server.endpointSlicesByService[svcKey]) == 0 { + server.endpointSlicesByService[svcKey] = make(map[string]*discoveryv1.EndpointSlice) + } + server.endpointSlicesByService[svcKey][objectID(&epSlice.ObjectMeta)] = epSlice + server.endpointSlices[objectID(&epSlice.ObjectMeta)] = epSlice + eventChan <- &common.ServiceEndpointsUpdate{ + Old: oldLocalService, + New: server.resolveLocalServiceFromEndpointSlices(svcKey), + } + }, + DeleteFunc: func(obj interface{}) { + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + panic("wrong type for obj, could not get DeletionHandlingMetaNamespaceKeyFunc") + } + epSlice, found := server.endpointSlices[key] + if !found { + switch value := obj.(type) { + case cache.DeletedFinalStateUnknown: + epSlice, _ = value.Obj.(*discoveryv1.EndpointSlice) + case *discoveryv1.EndpointSlice: + epSlice = value + } + } + if epSlice == nil { + log.Debugf("endpointslice %s not found in map", key) + return + } + svcKey := serviceName(epSlice) + oldLocalService := server.resolveLocalServiceFromEndpointSlices(svcKey) + delete(server.endpointSlicesByService[svcKey], objectID(&epSlice.ObjectMeta)) + if len(server.endpointSlicesByService[svcKey]) == 0 { + delete(server.endpointSlicesByService, svcKey) + } + delete(server.endpointSlices, key) + eventChan <- &common.ServiceEndpointsUpdate{ + Old: oldLocalService, + New: server.resolveLocalServiceFromEndpointSlices(svcKey), + } + }, + }, + }, + ) + + server.endpointSlicesStore = endpointSlicesStore + server.serviceStore = serviceStore + server.serviceInformer = serviceInformer + server.endpointSlicesInformer = endpointSlicesInformer + return server +} + +func (s *Server) findMatchingService(key string) *v1.Service { + value, found, err := s.serviceStore.GetByKey(key) + if err != nil { + s.log.Errorf("Error getting service %s: %v", key, err) + return nil + } + if !found { + s.log.Debugf("Service %s not found", key) + return nil + } + service, ok := value.(*v1.Service) + if !ok { + panic("s.serviceStore.GetByKey did not return value of type *v1.Service") + } + return service +} + +func (s *Server) ServeService(t *tomb.Tomb) error { + if *config.GetCalicoVppDebug().ServicesEnabled { + s.t.Go(func() error { s.serviceInformer.Run(t.Dying()); return nil }) + s.t.Go(func() error { s.endpointSlicesInformer.Run(t.Dying()); return nil }) + } + + <-s.t.Dying() + s.log.Warn("Service Server returned") + return nil +} diff --git a/config/config.go b/config/config.go index 6dfecf86d..9a8f41050 100644 --- a/config/config.go +++ b/config/config.go @@ -65,7 +65,7 @@ const ( // BaseVppSideHardwareAddress is the base hardware address of VPP side of the HostPunt // tap interface. It is used to generate hardware addresses for each uplink interface. BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00" - // CniServerStateFileVersion is the version of the CNI server state file + // CniServerStateFileVersion is the version of the persisted CNI state file // it is used to ensure compatibility when reloading data CniServerStateFileVersion = 11 // MaxAPITagLen is the limit number of character allowed in VPP API tags