From 1efb345be3b361af6fa969db46992d427c960f8a Mon Sep 17 00:00:00 2001 From: Brandon Jacobs Date: Thu, 21 May 2026 12:06:28 -0400 Subject: [PATCH] fix(ateom-gvisor): pin gateway MAC via permanent neighbor entries On CNIs whose ARP resolution is delivered by an eBPF program on the host-side veth (Cilium being the headline case), the interior gVisor netns can't reliably re-ARP for its gateway after eth0 is moved into it. The kernel's neighbor cache is empty in the fresh netns and the eBPF responder doesn't necessarily see ARP frames originating there. Capture (IP, MAC) pairs for each gateway from the kernel's neighbor table at ateom-gvisor startup -- triggering resolution if needed -- and install them inside the interior netns as NUD_PERMANENT entries. The actor then reaches its gateway without re-running ARP. This replaces an earlier host-level workaround that flipped net.ipv4.conf.lxc*.proxy_arp=1 on every Cilium veth. That sysctl opens the host kernel as an alternate ARP responder for every pod, which is at minimum a layering violation of Cilium's eBPF-only datapath and could permit isolation bypasses for pods whose eth0 setup deviates from Cilium's defaults. --- cmd/servers/ateom-gvisor/ateom-gvisor.go | 91 ++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/cmd/servers/ateom-gvisor/ateom-gvisor.go b/cmd/servers/ateom-gvisor/ateom-gvisor.go index 0f8f18f..b06b4fd 100644 --- a/cmd/servers/ateom-gvisor/ateom-gvisor.go +++ b/cmd/servers/ateom-gvisor/ateom-gvisor.go @@ -97,6 +97,8 @@ func do(ctx context.Context) error { if err != nil { return fmt.Errorf("while scraping info from eth0: %w", err) } + + eth0LinkInfo.Neighbors = scrapeGatewayNeighbors(ctx, eth0Link, eth0LinkInfo.Routes) slog.InfoContext(ctx, "Eth0 link info", slog.Any("eth0", eth0LinkInfo)) // Create a new network namespace that we will pass to gVisor. gVisor will @@ -472,6 +474,7 @@ func netNSDo(ctx context.Context, targetNS netns.NsHandle, do func(context.Conte type SaveLinkInfo struct { Addresses []SaveAddr Routes []SaveRoute + Neighbors []SaveNeigh } type SaveAddr struct { @@ -489,6 +492,11 @@ type SaveRoute struct { Type int } +type SaveNeigh struct { + IP net.IP + HardwareAddr net.HardwareAddr +} + func scrapeLink(link netlink.Link) (*SaveLinkInfo, error) { rawAddrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) if err != nil { @@ -529,6 +537,72 @@ func scrapeLink(link netlink.Link) (*SaveLinkInfo, error) { }, nil } +// scrapeGatewayNeighbors triggers neighbor resolution for every gateway in +// routes and returns the (IP, MAC) pairs the kernel installed. The kernel +// only starts ARP when something tries to send, so we emit a no-op UDP +// packet and then poll the neighbor table until each entry transitions out +// of NUD_INCOMPLETE. +func scrapeGatewayNeighbors(ctx context.Context, link netlink.Link, routes []SaveRoute) []SaveNeigh { + gateways := make(map[string]net.IP) + for _, r := range routes { + if len(r.Gateway) == 0 { + continue + } + if r.Gateway.To4() == nil && r.Gateway.IsLinkLocalUnicast() { + continue + } + gateways[r.Gateway.String()] = r.Gateway + } + if len(gateways) == 0 { + return nil + } + + for _, gw := range gateways { + probeNeighbor(ctx, gw) + } + + var out []SaveNeigh + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && len(gateways) > 0 { + neighList, err := netlink.NeighList(link.Attrs().Index, netlink.FAMILY_ALL) + if err != nil { + slog.WarnContext(ctx, "Failed to list neighbors on eth0", slog.Any("err", err)) + return out + } + for _, n := range neighList { + key := n.IP.String() + if _, want := gateways[key]; !want { + continue + } + if len(n.HardwareAddr) == 0 { + continue + } + out = append(out, SaveNeigh{IP: n.IP, HardwareAddr: n.HardwareAddr}) + slog.InfoContext(ctx, "Captured gateway neighbor", slog.String("ip", key), slog.String("mac", n.HardwareAddr.String())) + delete(gateways, key) + } + if len(gateways) > 0 { + time.Sleep(50 * time.Millisecond) + } + } + for ip := range gateways { + slog.WarnContext(ctx, "Gateway neighbor not resolved at startup", slog.String("ip", ip)) + } + return out +} + +func probeNeighbor(ctx context.Context, ip net.IP) { + addr := &net.UDPAddr{IP: ip, Port: 9} + c, err := net.DialUDP("udp", nil, addr) + if err != nil { + slog.WarnContext(ctx, "Failed to probe gateway", slog.String("ip", ip.String()), slog.Any("err", err)) + return + } + // Write triggers actual transmission, which forces ARP resolution. + _, _ = c.Write([]byte{0}) + _ = c.Close() +} + func restoreLink(ctx context.Context, link netlink.Link, info *SaveLinkInfo) error { for i, saveAddr := range info.Addresses { addr := &netlink.Addr{ @@ -561,6 +635,23 @@ func restoreLink(ctx context.Context, link netlink.Link, info *SaveLinkInfo) err return fmt.Errorf("while restoring route %d: %w", i, err) } } + for i, n := range info.Neighbors { + family := netlink.FAMILY_V4 + if n.IP.To4() == nil { + family = netlink.FAMILY_V6 + } + neigh := &netlink.Neigh{ + LinkIndex: link.Attrs().Index, + Family: family, + State: netlink.NUD_PERMANENT, + IP: n.IP, + HardwareAddr: n.HardwareAddr, + } + slog.InfoContext(ctx, "Restoring permanent neighbor", slog.String("ip", n.IP.String()), slog.String("mac", n.HardwareAddr.String())) + if err := netlink.NeighSet(neigh); err != nil { + return fmt.Errorf("while restoring neighbor %d (%s): %w", i, n.IP, err) + } + } return nil }