From 8478b3a4fde70a9658c6b6f340989fae49b6b01f Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Sat, 2 May 2026 10:58:57 +0800 Subject: [PATCH 1/2] feat: attach eBPF in shim We currently rely on the node-daemonset to monitor the shims and attach the eBPF program for it, which can be flaky in some cases. Plus, we want the activator to work even if the node-daemonset is down for some reason. --- activator/activator.go | 26 ++++- activator/bpf.go | 95 ++++++++++++++++-- activator/bpf_bpfeb.go | 2 + activator/bpf_bpfeb.o | Bin 46360 -> 16816 bytes activator/bpf_bpfel.go | 2 + activator/bpf_bpfel.o | Bin 46408 -> 16864 bytes activator/redirector.c | 9 +- {shim => api/shim/v1}/config.go | 46 +++++++-- {shim => api/shim/v1}/config_test.go | 2 +- cmd/installer/main.go | 36 +++++-- cmd/manager/main.go | 4 +- cmd/shim/main.go | 43 +++++++- config/k3s/kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- e2e/e2e_test.go | 3 +- e2e/setup_test.go | 21 ++-- go.mod | 2 +- go.sum | 4 +- manager/redirector_attacher.go | 58 +++-------- shim/container.go | 15 ++- shim/log.go | 4 +- shim/metrics.go | 2 +- shim/probe.go | 5 - shim/probe_test.go | 9 +- shim/restore.go | 4 +- shim/task/service_zeropod.go | 2 +- 26 files changed, 278 insertions(+), 120 deletions(-) rename {shim => api/shim/v1}/config.go (85%) rename {shim => api/shim/v1}/config_test.go (99%) diff --git a/activator/activator.go b/activator/activator.go index 61f3d93..e30409e 100644 --- a/activator/activator.go +++ b/activator/activator.go @@ -12,6 +12,7 @@ import ( "io" "net" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -74,7 +75,15 @@ func parsePidFromNetNS(nn ns.NetNS) int { return pid } -var ErrMapNotFound = errors.New("bpf map could not be found") +const ( + IfaceETH0 = "eth0" + IfaceLoopback = "lo" +) + +var ( + ErrMapNotFound = errors.New("bpf map could not be found") + DefaultIfaces = []string{IfaceLoopback, IfaceETH0} +) func (s *Server) Start(ctx context.Context, connHook ConnHook, restoreHook RestoreHook, ports ...uint16) error { s.connHook = connHook @@ -107,6 +116,17 @@ func (s *Server) Start(ctx context.Context, connHook ConnHook, restoreHook Resto return nil } +const AttachActivatorFlag = "-zeropod-attach-activator" + +// AttachExec attaches the activator using exec on itself. +func (s *Server) AttachExec() error { + out, err := exec.Command(os.Args[0], AttachActivatorFlag, strconv.Itoa(s.sandboxPid)).CombinedOutput() + if err != nil { + return fmt.Errorf("executing external attach: %s: %s", err, out) + } + return nil +} + func (s *Server) Started() bool { return s.started } @@ -465,6 +485,10 @@ func (s *Server) initActivityTracker() error { return nil } +func netNSPath(pid int) string { + return fmt.Sprintf("/proc/%d/ns/net", pid) +} + // convertBPFTime takes the value of bpf_ktime_get_ns and converts it to a // time.Time. func convertBPFTime(t uint64) (time.Time, error) { diff --git a/activator/bpf.go b/activator/bpf.go index 41b0ac5..0b33f58 100644 --- a/activator/bpf.go +++ b/activator/bpf.go @@ -11,8 +11,10 @@ import ( "strconv" "github.com/cilium/ebpf" + "github.com/cilium/ebpf/btf" "github.com/cilium/ebpf/link" "github.com/cilium/ebpf/rlimit" + "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) @@ -28,6 +30,7 @@ const ( PodKubeletAddrsMapv4 = "kubelet_addrs_v4" PodKubeletAddrsMapv6 = "kubelet_addrs_v6" trackerIgnoreLocalhostVariable = "tracker_ignore_localhost" + taskCommOffsetVariable = "task_comm_offset" tcxIngressPinName = "tcx_ingress" tcxEgressPinName = "tcx_egress" ) @@ -89,8 +92,8 @@ func InitBPF(pid int, log *slog.Logger, opts ...BPFOpts) (*BPF, error) { return nil, err } - // as a single shim process can host multiple containers, we store the map - // in a directory per shim process. + // as a single shim process can host multiple pods, we store the map in a + // directory per sandbox pid. path := PinPath(pid) if err := os.MkdirAll(path, os.ModePerm); err != nil { return nil, fmt.Errorf("failed to create bpf fs subpath: %w", err) @@ -109,12 +112,24 @@ func InitBPF(pid int, log *slog.Logger, opts ...BPFOpts) (*BPF, error) { } binName := [probeBinaryNameMaxLength]byte{} copy(binName[:], cfg.probeBinaryName[:]) - if err := spec.Variables[probeBinaryNameVariable].Set(binName); err != nil { + if err := setVar(spec, probeBinaryNameVariable, binName); err != nil { return nil, fmt.Errorf("setting probe binary variable: %w", err) } - if err := spec.Variables[trackerIgnoreLocalhostVariable].Set(cfg.trackerIgnoreLocalhost); err != nil { + if err := setVar(spec, trackerIgnoreLocalhostVariable, cfg.trackerIgnoreLocalhost); err != nil { return nil, fmt.Errorf("setting trackerIgnoreLocalhost variable: %w", err) } + // in case you wonder why we do this instead of just accessing comm directly + // in the task struct in eBPF: this would make the btf loading way more + // expensive, at least the way cilium/ebpf behaves at the moment. It is + // about 3x slower and uses 3x the amount of memory. This is rather ugly but + // I think the performance gains are worth it. + commOffset, err := getTaskCommOffset() + if err != nil { + return nil, err + } + if err := setVar(spec, taskCommOffsetVariable, commOffset); err != nil { + return nil, fmt.Errorf("setting probe binary variable: %w", err) + } for mapName, size := range cfg.mapSizes { spec.Maps[mapName].MaxEntries = size @@ -131,6 +146,46 @@ func InitBPF(pid int, log *slog.Logger, opts ...BPFOpts) (*BPF, error) { return &BPF{pid: pid, log: log, objs: &objs, noPin: cfg.disablePinning}, nil } +// TCXPinned returns true if all TCX programs for the pid are pinned. +func TCXPinned(pid int, ifaces ...string) bool { + for _, iface := range ifaces { + for _, attach := range []ebpf.AttachType{ebpf.AttachTCXIngress, ebpf.AttachTCXEgress} { + _, err := os.Stat(tcxLinkPath(pid, iface, attach)) + if err != nil { + return false + } + } + } + return true +} + +func setVar(spec *ebpf.CollectionSpec, name string, value any) error { + if _, ok := spec.Variables[name]; !ok { + return fmt.Errorf("could not find var %s in spec", name) + } + if err := spec.Variables[name].Set(value); err != nil { + return fmt.Errorf("setting spec variable: %w", err) + } + return nil +} + +func getTaskCommOffset() (uint32, error) { + kernelSpec, err := btf.LoadKernelSpec() + if err != nil { + return 0, fmt.Errorf("failed to load kernel BTF: %w", err) + } + var taskStruct *btf.Struct + if err := kernelSpec.TypeByName("task_struct", &taskStruct); err != nil { + return 0, fmt.Errorf("task_struct not found in kernel BTF: %w", err) + } + for _, member := range taskStruct.Members { + if member.Name == "comm" { + return member.Offset.Bytes(), nil + } + } + return 0, fmt.Errorf("comm field not found in task_struct") +} + func (bpf *BPF) Cleanup() error { errs := []error{} for _, link := range bpf.links { @@ -157,6 +212,22 @@ func (bpf *BPF) Cleanup() error { return errors.Join(errs...) } +func (bpf *BPF) AttachInNetNS(pid int, ifaces ...string) error { + netNS, err := ns.GetNS(netNSPath(pid)) + if err != nil { + return err + } + if err := netNS.Do(func(nn ns.NetNS) error { + if err := bpf.AttachRedirector(ifaces...); err != nil { + return err + } + return err + }); err != nil { + return errors.Join(err, bpf.Cleanup()) + } + return nil +} + func (bpf *BPF) AttachRedirector(ifaces ...string) error { for _, ifaceName := range ifaces { iface, err := net.InterfaceByName(ifaceName) @@ -193,11 +264,7 @@ func (bpf *BPF) attachTCX(iface *net.Interface) error { } func (bpf *BPF) loadOrAttachTCXLink(iface *net.Interface, program *ebpf.Program, attach ebpf.AttachType) (link.Link, error) { - name := tcxIngressPinName - if attach == ebpf.AttachTCXEgress { - name = tcxEgressPinName - } - pinPath := filepath.Join(PinPath(bpf.pid), fmt.Sprintf("%s_%s", name, iface.Name)) + pinPath := tcxLinkPath(bpf.pid, iface.Name, attach) l, err := link.LoadPinnedLink(pinPath, nil) if err == nil { return l, nil @@ -208,7 +275,7 @@ func (bpf *BPF) loadOrAttachTCXLink(iface *net.Interface, program *ebpf.Program, Attach: attach, }) if err != nil { - return nil, fmt.Errorf("could not attach TCX %s: %w", name, err) + return nil, fmt.Errorf("could not attach TCX %s: %w", pinPath, err) } if bpf.noPin { return l, nil @@ -216,6 +283,14 @@ func (bpf *BPF) loadOrAttachTCXLink(iface *net.Interface, program *ebpf.Program, return l, l.Pin(pinPath) } +func tcxLinkPath(pid int, ifaceName string, attach ebpf.AttachType) string { + name := tcxIngressPinName + if attach == ebpf.AttachTCXEgress { + name = tcxEgressPinName + } + return filepath.Join(PinPath(pid), fmt.Sprintf("%s_%s", name, ifaceName)) +} + func (bpf *BPF) attachQdisc(iface *net.Interface) error { qdisc := &netlink.GenericQdisc{ QdiscAttrs: netlink.QdiscAttrs{ diff --git a/activator/bpf_bpfeb.go b/activator/bpf_bpfeb.go index 3f1521d..c83d75a 100644 --- a/activator/bpf_bpfeb.go +++ b/activator/bpf_bpfeb.go @@ -82,6 +82,7 @@ type bpfMapSpecs struct { // It can be passed ebpf.CollectionSpec.Assign. type bpfVariableSpecs struct { ProbeBinaryName *ebpf.VariableSpec `ebpf:"probe_binary_name"` + TaskCommOffset *ebpf.VariableSpec `ebpf:"task_comm_offset"` TrackerIgnoreLocalhost *ebpf.VariableSpec `ebpf:"tracker_ignore_localhost"` } @@ -131,6 +132,7 @@ func (m *bpfMaps) Close() error { // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfVariables struct { ProbeBinaryName *ebpf.Variable `ebpf:"probe_binary_name"` + TaskCommOffset *ebpf.Variable `ebpf:"task_comm_offset"` TrackerIgnoreLocalhost *ebpf.Variable `ebpf:"tracker_ignore_localhost"` } diff --git a/activator/bpf_bpfeb.o b/activator/bpf_bpfeb.o index 80497e52f06c39d45c84134a2fe9f2163961fb25..2c7f38ada331dd86de7764d9802d5d95faddd0e7 100644 GIT binary patch delta 5035 zcmai&YiwLc701t=ySsNE_QrKyTRS039?dpQ(oJYxe{kB(6DJMCl)Pv{Z6~p_ArEhz z2w9xj2lA(bZ8YCkjSiC{95D3!`5FsQ&slcg%WGEjjic||VYE;7G|DQY8 zou9sweXo@|)K+3Cgj0sbRgRP9779%kvLP!tn1= z$Gj{_n%3l`dg1D>6^lq;;Y(y$rY;b|cjp6vXhn_`EBvIUreNvsSy!A>3!AyL_}|N;fG3W@ z-wA5TLVgeOTs(Qm9~ikAHOrHJY4_S4@>D?XIO;8dED{k>kh3E0FHslppt!%mtAM9p z#B*TdKt_VSB!}!3z_i z=j3+OAH|b}{0So$An$>k!0;WA_fvL7Iw60SG9o(y`H-t`If(WePg*5gkWh?dov9Ig)^p=FsUdz`AYC5LVLkX!}2r)_D7~_-!c7}CA)tc zjZObg!R|IALwQDIN9~8&5*^?L`ZWIOvI}q93DX@qSqi%cjot3RF1^Z+?|W`&zS-pr_85VlCpQC#Qn3{Z6JhkOs3{r@ z2kj9?%ytuy<_@q4D7IPLYuJ3wjlf3+ky$D~$SnknPvn*`fIASdaVYMPv42h63})yKTa1IkH37%j zz6{^rH4$qCJKVUh!-U0H#qOlVw4bt=1DF{H1AxtpL;pC$f(bEzGCD-!mI4Dr!Wo+x zL^4)8?XwosK4&okisUV(eY3^1FU)e8DT=htDl+WQW-&YLx0v=F7Sq0HG3`qh)4tDQ z+V{_J8Q%bkffc=W)nSCPV4$r#)g=lgUp7;w!3aXQ#x z#A?S3W8bm{1=J5iUr^axQrPpb6)LCx7}vGeVeWsVLf=MSDKOAXhkb_4 zW#kP6tS=wj4c?%|xQx7!fWuGbRRZQGdmkC~#w^As@ovaIJ)_zsq9SxS6&TcZiK@XT z@um%bLDW2~0q>5*NO7DgL`G`Aha-S{st)`j>Lvcz@H1tn01Zo3kNLB8bsT`7$9`WUCH*ue$*)3 zgqqEO+;p1XK#jspruhSn5*0>mVVgM@3Qpb_rJbyPn*?x z#cJhLS?KMh*_gu+ojU0GUj?Cpb}X!a>qoAxD$6)ThU(y9mBow8Hsm5?rXu7q$egNS z{_&>Whs=;He`*43`Ev{R*q8M!VmpjZ!XS>vIOG|rf3k%<2*zzQ(>@LvXLZQfz}S4V zoMH~JZ*@?pQj73nJA>qUOfeaBs1;!Rl+g}DE`S+O$U7l3BO&jH%mjyAg3KA41(bRL zGBXzP0A$*d8CYr=G^zguX?4@;^4n)V$pw~kBgnKAlY&oB8v0)#nA)Kd;X%) zr7B+M_m4J4--SBi=SLSL^R3!OWZEAeZFGL;AFSg4b!sn;HYi8=i>i&d&4ZHh3o!V) ze`~Zc(HE$k47@eO9R3n?^q2{LUX+G~utp0%Ofb;8X5AoCxp=K9xojk$)MI`DxNgcI k_RFzA?|xKuf9KS>+qvhSyHri$=y+rxai0wt9Ze_*EY-UOC~`3zbiICn>&dwp;7mOt<@NcY@JhBA(He7v(mr z>~`-JD4reU?|kdyM?`<5S9j;??|A%oiXUFCyK_qS_~VcNZr*H9wr(#8D0iE4=Lj0T zUTYFvot^eEo zbgt4rX+Pber}R^A=%}7BY&as zy##z8!$05nojzXUd<=h)@x6YK_c46mPvnnke0_Wjf4<3k`#|2u@aGx7%jA6wpQibG z&NRM{;d>2`zxNO1<74=zn*50-?_>CWoFPBU_&$ao(&z2@nh?Ve^~<}HP2R`wL;LgY z6wf~`#0yV%3q@!B%Wkh7>?CjLPLq_ko3j+76kor@ZccrTxPE;4nDOK5!Fct0h4wMD z*8{ZgpJY5^4{B$wd{*G^vhnp2pPQS0=j7u@1+#wT;md+RS5Fu>*>;$J>9Bk{9`>~l z{*aV&R?Z9kO1VBH<)-TK8$HOB`|u^lJUli12GV=WDX+8iJ+gEB?BdWq>`V4J{D8Ah|c?soR?w_BF^g>#*+kz$tTE@67iO%UAP-&)3?ddF4)P`x)RD8rHacu{A=GXa66xQRBGDm2Tr}tS$_-Vv(*5h_Q`rAm|7z}U za;4k;H&(vZ3+eI~L4VuIUkv`9;U(Z7T3x7u%X4MxVvl`YRIi9jx={OLF4_v4#=KrWpOG(z`O4TVh-|Dmse!5O8 zAXiYBbVHil24Z1CPJJMp2VZMA0&h0l3%=cO3@#X+1P>c-g2xOW0mlKW3b}g>=fQt$ zI0C=ba4-0sVAh^Wm;3X;@5qDSZ#V+~h2dWCrwqs7&l{cuf5mVU{0+lL!2cAmw4eJa zw^j5+9{lfyC-Hvj3-mAzeuw%*I0DNKg?ot=R)b^s7l5(*X-zQtmCSUa4*4(PJh8$W za0LGwiF?FT=fHpCH1HY2n60Ror^@fjt-wKXG7fgQud^H$)y2c0cxnA%G49DP) zfvIP168uTGEWY}2j~^Cp zMj!Yla0GuCO#L;*%$VRtSqsJA58exo!G8?C6Wj#f7x1}6Gv3AR`Of8q-^c9*!7b1D zG?;w4Blv$~xEK62FlBYe;2(l%Z+8GrLIdhe>yk@X^n$P4LNJ`czdv;cVfY+nxunG`tA>RKttGvNy?)<^;7- zI09o+9;1&FXrIT|fM09)TJW05yCky({8Pg> zfd9iVoqgg7hHnJpH@!T1Jn?+PH-k4Cz6DI#(!=%`ybb&ca1-1M{uAeJkHG_mC&6Vf z^0zm^<6z4F1ELe}HXMQP0n?#>5QEir@D8FA-^3019pXP={3iIl0Y4)GKLmb@bI%Z8 zdU%^3{_e!T1(VOSiB9}Bx3`01@KG@1_SsGFF9UuK(Jb|E#3>cmtTVy1^*J$EV@
fdBN1PjIf@z2QgmZ&2nCBjg|M7r#63v2kpLcF&1pXTM%itLN z4e(c-8zP$ZJ#JqGM_}3i*TFIP7huMTx^UL7xE%&Z;NOD(9vtIyr8n{_H^HZXzv1`O+N!z%6$GKtgp9@|Bj=@Uw zQE(G1yP^#vL?`Xz_DOIAekBhYd&IFM$6U zehfwj?j7Js@c#s(qe>I}eeegJs}h~`Gj2D4W3cq|6L6DxXcm|{>?S&S4z~!*b+T;f zr{Ea89Q+u#30@5*uNu+GS91F~I08#2{|b)58XVY0tqE2q|AMy+L?<`6{Sq93y>D@D zO#Byfqm5(Yzs7J3{!_zE@Vmgja!#r|`4G2%ckX$@pX1g6&V%U(_Zx5oMmHK?&x^s- z*FD8gi|*uOhV$^H2d+CKmUbtAe+Q1i3&5nkvk6w)A7|=}z?Xs_0>@xJayW=}(ViPQ1y_%S?jjg0DiR30?+9&o3f6b-m#Te3RiA+z&p4 zOcRX3>UGB55qJVT0~~|*IiKgyXu?g0nBo*QL7AepBd?gjUOsm}v3ShgyeCiouGp61*^ShEjsyT#@5M6;Ed za0LDYGN*%M@aLS;=JHMOVZ%qjKR{-N%TZl7TQ<269I@4=9FJr08Q^Q-H^CAVJ_4pK z)GfD|=(J7F<<0;{VELaV;26Bi_)YKxc#g|0;qhrN1fL0xz^?$$1xx<5JS6RsCV0~D z5%BwwIm_kFhjkkEi=G2+g0BY0@c$Z28_%Bv%MNbZl23s0PD2hffu;k(g;(g zACz`!44w}r?b1o`h2RL>1ot41PQ*I>3h=q$2z(v*Ja7!&&O_ucY=WN)CjV}t)63lG zi|zEDEp-j++|AHe6k+zR;9Uo_1*{^|XU-=|y!CY=6H+=L_W&xx@`7lUQn!js_N<^Tl7 zjO5OkB|uoLTZe8woN=o0uR-P<3R5zoGgPr_O=h*pTqimuvk}s=IwOM5KkY5gxRqFA zA_-i2@+@QB~I%3BCF>K zdS+?q_cLBXqO(HN&XA4XsORwC$n7Ss7(7XgFUv*XKQkPIKWw-O=914<&VDa92OPuy zJPhIe;J-I~FZlb04}h7=lz&GP{JVfV_4ri75%>bbF?bbtn{%B_@VdaK6>iR@h9j`_ zL}jMMtnKaw-v(~NSDS9fW{Bp~nan2tRCo=bgZ{~{NzsGP)>@y^yE^|I?{jvc3v36O$ujMZZb3VgOIO0mu zqU$-Iv$WXMoG%)tO>@4I*f()H&C}ZHb;dR4+mt7JKxWPli0K1`ImfsON9>W6m2eC` zg;;HFg0UHokARngX|qJnTx~c4V_%*hgExVfAkzfjOu1dqJ>cgO(-ws@cXE^NVsOQ9 z6Z}H3(h_;wIuV?Qe-8{{zVbQqvxZ~v7Y#SThr!ri2T?LMIwJE!)UzW-=7^)tIoB}>{;}b=f~8yW-{#z0 z>hAH|Y1d}M_rX8YF!nQ7x|KZjoh#i6KLD1Vgb#w%j%QJB6HJ?hrwL1+{p2lt2Y4W; zeXjg8bXNrb6=3YWD+a%ITFWQSeG~j=a}nmgg&RIe`kDJ~ZuD6$0)GhH2adrX1D_6V zg1J0?1pHaUkAlBs82!!tJIay*^I(-_{WA9lVEMl${GXalqw63{R1iKnRHj#$P(=1u zGlB@&p4AoDXPr%~N+BM&2z;F{fO1TC zNmxCs0e@K$gP z{tg&Bq~`9dA9KSFr$ylZFdQ@Vx(qj|&$@u86V01zI0EOvJA?0?w+uWEZoT%*$y%aejPylUCbl=xA#wz;AdMu><0H5_Sq>D3+f9s1pd51lWBk( zhQ}m_fRqpGjE%(JF2%RH{bd`w3E>lUbGcvWZZiK$o_FmFWvM)syJVIG+`M~)sT1V9 zKN6-eeC|Ar58+dZeZv(Kp2wBc`2oPfP`!ETNBTlx-lw?5AB#s14{uRRL-0D1EHvC%gNFzTZJ0Q$2a{C`1 za-(k28*Q8~The$Y%pc~ax<)B~)eFCB{206kjJ>%g`1!=74K=g+D82IMpM2o_7Zbl; zJc9K0My@=U!U3)bKluY<nUr8*l9=TF zO3M9|o`w7s&AJc)C7~~p^I}OP=f%o@!73Q<;EKWPte*I`1=o-k{VM+j>YsyLgatcH z=Fh-GZ5jXYo&L_vY?8;`uC7h^#P3RB`4k@zRJFMl)QC?H+>;{1zDScHTCf+s#u8z{ zes03(aMpSrYHatw*Eo9*7hwVRl=uX{9()iwrWp0(60XUVewuClsQJlxdM+ZLNo)Ii zl|N-A`ca#d4`IOvu~pfU>e&>B_)FZ7T_8WDHhzz|llZ5^$j!Qo8?=1P0f@IU0)ZsAVi zo4F#%TUwgy_a@@|ph{YJKQUse_pDa~^k^e#>%@n+2n#2;J;W7*W!tnRC%hkdtAn@u zkMfMLQ2q2Vu87ME5c#|$O!&BN<{t$WNV{-?{P^ecVEH-Kfw1tc++^DkSbd~9sTcfS zV(BCXf0TsM6Op&2Yr#HPnVP$jxw1)HKaSuzu&CE`#!0KJFM&VF6_Mvxi2stS7cBoH zehmH=vG|kV?+}aM1b?4c{3Breg7`V2g+Jp4KbHr~XGwpNbLXmRp99BWjTPZ0I3KVc zpS#L%1ir*@3|9M?^E#Sf+0Exe*PUB19Ko*`j=}0T$ut=+7lNap>T_QW{sI{KT+K6I z1jk_c&c6XS!S{jD%XA_i9tB76-)rOOQ{tjeHI~TxicgD&OmhD48Sq0UPy5eJ&J+Gh zQkk@$6%U!On*1Z+Z=}-x$nalF%F@E} zVvVhcD@n^3JWp$7{&|_V5FgcZ_y@VEoQP^~FdV~wFL)A2J*8ENAOQG9D~m?+ypNSSWU}!8;-!T2lC0q z;N+Y^V?m0MFnqt4__cTn>O>2P&CsX$+f29EYQAh~iPH0WtBv@-7Cw@~-$~)`3a84F z-z2}#U#7Q*BxCp=QyAZtzf+Yb=&!t}UrX+LZ5-D5sWOuGc$Ve+PvoU{1n{P*oy>VT z+lu8icVTDwh)ZiJ;muqhK%V5&P{}#>!GAFqAy2)$jHfLB55fOd$Y1r$WB<96y+E+< z#o!3O>{GaxD`|5xg}+HNlW{VE{A1+f+YQDiX)dA(VNXCiJ0ll93lolckk)mghm)rA zbEiU6RhJjv>i_c;)_6(y%qcCL9vigpwD&8M5Ee}Xi^L`uDgVj9_xlXhBWBO>ZrXA- zxQWO9bK<8!6SeZ!{GM57^8!}RZ>y@vDP7a68ri|zrFHWz{KH5`NA2oJej6a2Qo?+|`Zz_O7=A2l3- z|H5z#{!7D6@T25$cWA;Q{JYAU7Qz1>m@}qnG59B7`fA!F_+P=2X@cbk!D!Mg#>R3l z21j7Vcn(#}5PvQ};{bgwUQE1&i?DbZH(}boc(pJBdE$$Or+GW_=NDUY&aFVk^E>>v zx{EKB41CfpzTD)|)#7W6k1t$&yS8qKD_{vDuiyPOhT?3 zd7Up{2h;Q5=fK>mC*U$S;TWtw1yj554h;F6_;P?h$AvlJA6pVhSHNpSloQpyqX%CiLZhfS= zfwa?mQTxBczX%+|QfD^6O|Z>DGhi(_pBwd`5mCJKBOJp=k2(5bMiYF6L!YeGiI!|O zJRAH6hV$Se^7O;32s{8rQ5r8xhQOpbEe2PeU*DhB1Z!MSA59xeUTZi4zYEN~ae55? z2pD~y0oJ=c>0u76C7zFnCOZGA+{p9n2)v!N*w5K9xHsT)gd5K9gXRmr2uz*}h|ZT?QRfADuw?AH$R{6xSHiD?W904z?*=#FZvfZu zF^FDpHJIlY!cX=>-Qr)*jlSrPz&C@*x4TKKjv(Y{>jgVr8XOUIxuDG7ME{2PF5=%3 zzn=Ja#P244ocNPInY-YdN(}w4Gd;t9V4#r*Pk!tI(&nhQ!Ud0;%qme+Ws*JW(yo2Z zUBTo1To-duPleW6AkUvJodrXk7;zDDJ=|jW>ICtdVA%{<)mVyt_|C%53rnA3_-gpF zAMvjNODDqE#L^vxuLGA1ZvxAfB!e!O-es6PmcGL9X2vSE;bj;DOJ8Fc`(FA6!%qXh z-7vPk^!~-nS3}cr|=^tN*+pvRq zhHnS2HjMpUDBY?pwDrP7C)nbJ)X(#u3Fh+nS>UqaXM+1IsU|4*g*4&*K5`=M1C!3;$O59G4p;Cgj#|#qj?DVljA8AwTJS8MdxDn%sh(Nk1)uAmlcIV`PdD@X@wgMqhdyf$uaNgI{jANi4}OqF!P0 z*%)GvFH7OeQ}~J$zA}ZcO5y7gOl@VCy^LS^H?^6t?9JSSBk=viN*jaUXSfM|2u$0h z>1AIuoCj;{h#yh=KR4V9{~Pcw<%+@I3%DZ!|CBoLPiG+MIxLssmvAAoTsk06)iWte z_3;VCpN-7P#8&_1}q)Nnza>cngRk{pD|yeqpZB7;%y0QZk>0Ok^H491;71Nb~U&{wj6*7L_IB zo(BGj$$YT#0*PFNqI%CgiK8}&WALO16 zZgM4Qef=;p>D)q2`6thBBHbMFwm$K5EdE2`q6Jnabxfc0rQb1KB>P&zHOZCa`QsF3 zUVXye`@+=ozfR#DCaZ{gUv#bM>zOGTKMrV*bYVejO@-XEEiNlI-7)W8^eozOPH5sq z*kaOW7wt71Arl)`ESr|zn(*%--E%|Q>L4GCP%w>|=4@ zuRyO_f2eLN?l9Z~-rdgAzbi=Rxf3}b_OU<(D9;xYFOb5(FB3k+yOR~K6^4HX{5K0z zQO3%O`-Qu_&8?uXoyO53_#YF_vA75SrO7V^KPueG1PnfG7=N?kJHj}*Rp1|zhx9NT z{0s592rG`EOWAuQN&?Wdb;WOyKNI30M;lZz@#8C{5Bc;6KEo)PBUhegxCvkBkfElI zUp5Ii0?#)bgU>VE1TP6#rLUwtO4|{Eml=-1D-1Wms{^LRZsl6T5m@7nw6gh?8x1$H zXN@WNw8)3tE^uV?wD>XVt-2u7BqroW!nEP&iN#-1x@5=9T+e(=+_{XHB%`?RRC7Almh`_&tK|V8(S#=_MBIFvn z({F^F^ru5Zn$`-)`(Uy1QpoMm-O|-kc!Iq3ZqCB0^TBEa{jh31>7ExJth(B81pg*@ z4{^m@NuGb3!sus}eTR{IL5lxEVd|@DgsS+qikrKOGLcB^x0+3@NQ7=~D_eiWZ7uKt2y z>gWCN{!stbUqKQ4qsnge5!!VRfUx@e;FlZzG5C)R{~Y{!!^gmX9B_|*z~KZi?dv&( zSaT?$=X7p1pZ1_KN5HX|FwLs8VjLh`oSxjDECV7LdePA`9Pgbrl$bI?59WeV>aLif!e70 zCpv^|=`asYbQpmX9rhx0JYD6VgdaT#|0MCPE$FIeCvBm<3O&^nem<3?j@oPezqW^- zdnku~?d+v2=~8$ItnngDy?Q1M;}d({X*dS|xnb(*_nEIlz6t+>@acD{I&=zc`yu>K zz+nt^MBu-MXJe%2OJK@VlY71nKH!UXJz5JPFI#GjWyvshl0Ma%GxtX5m@C~!>T^|K zMutzDtYQFVZo&Af!&<$E>jHF>cT9Yo-}+eF%d`K%vzf@3L#(l|R&&o8Ts>gaZwuH9cYyOG`78K;8ulm##b98@O$cZ@k9GU=i+BB z{xnH^eqijuW9oJ>_TUfh^Wd)-j=Z_}Y7T!;+zK!9ZN-X}pT$;;CDqVEQ zHZDT!wG4iEx%>2-dM82is`4kvrO%Y}N9G;637x2$%l(<|C+HTwHAgt`Hu1+xG)fyl$?!0t8|CP6grfJPP=N6kguO zm%g6-Wfz3pyF%=3@I8j9>!p8!jIT4%1ICw5?oTQPeoq^_>)ZI!hf??pZG7&1ZG7oB zQus%0y!dDvUwW*KFFUo3=iJoBb9c7!WsBQ5|NS<;Y)uMZm%_KC@H5)@!q>F%WvTp6 z@;_)3baC#RQ!?Wz{K6D|MGC*ZjZaDG;j#zX{L9{-!hey%pH1Pfr0};>_@7hw-`e=( zMJaq`8(-d;!l$?KNvXD6o=SUpN{5$6ZTXW|r|{-BzIMEg&%I? z%lD`7>)QDAR6j0yw9UVKvW-vdZR5*RwtD%8Q~WQc@OM-A=WTq&v=pA5!gJer#_er< zMM`g1_`~R6GgoX($=r~_DchTs8W&fj>VHK_KUa*m(w0XXu28so+qSEpQMh%>GdC5s zKV$2rLIGZ1Y0Uz+ZCJNf;@n$pP>Xks^_7N^EDj9R z>V@(3u3R18;3+p2;*D7u>7u@?&^I6G2ljv)p7yuU~!~8yw}xABb7#}P_C|b!(?d2}Y*QriruzPu_F|;h-CQ-w({8f41CX}@-f2WGwkSRxceDbpN^m0eWLjyH8 zR2V4L8$|WWSgpSV@SRSsYg<+7s}2@M8}OX+P=B4wXkKYDpIa0 z!}1X3QmWOlaM!5Rxf)*C>d{(pXQ6`KH5$XNf2h>IOSU*RLjAn~ySgGQLHC}^F7Xw} zTaa#O{mQEnqs(7%4ng#I_CBica^*!-s@qh zFgRSU)>9Ze?Jw>v;DPZPje_J$4EcUlXsk{-ty~1iwYJ-PC8xTS`!VZs#f?^KBe)Qc zYYiWJuT>|jh8q~>A0DBG{knSp1lN!GCWWjYos{k zfr%*;bErDBorNVS3^N>3Y>XQVog(R3UaV1*%PtO}p`T86`v|4k+e#zDTRsA zH@wRl)uL2r=yz}ZzG03o8uAN(42gjS&d5lzBmfN-$A%mALed?r3D~zF45rXN*Y?=jHd(Kt8gJPW$8)hBk$D#zl@UCJjR0z0#7B5TW<5CI8auhc zWI*&6M~gMBP2^09a*CAZqyC}t@Bqlw%YDPjG&E<}8=v_+suspN>Bz-gr;6ahu19w< zZ3chphtIfrtuRm-#ihy(tB67a@qV&p%(v)0$YJ8Rw{iL3u||1>B>ukB-|;%*GIvjm z`#Zjoyh~MV4rdX@=V;jv)1Uw^WxA+T8lD_MZ-c`PJk%XNu2$;hi6mouS%xmCV%BKt z%L*x|&ZBTEdRL2^97FH)k(T296^6!Waimn%ddJoG*2n9mkq{3tXzFgTJj}AaTq%sz zr0dZ-i-QS_J`4wX=Dl)5vxWCuRqsoL6L-rLMAwY<)r+I<4pww{fiNfcp(O7LtHWbB@)FZh zdBo;s9wlvHnrVzNRz=rX_?w}Q&?3#;e(FW7-qDpS#eP?m2uEDoK zjgO=V8O}tNsfK~un!VD5pZYLP**JVU!OuY2gD{`@al|ydGc==r7?Z<{Ms%z6jj<%~ z&6B!|)FZ`0J(w?hh|vCo>X*s6LAfndPd^3^78pK(9O)qQ2J0Tbob{ylE(I3u)$lmb zA#F+^l%@2D7QFq^CY#vHT7s0u9r2zLMvC{Sl~`kI3iTd(bZn$pz+^_fPTCX&lq+nJ zT8{&Ru{#l%%BRFCjX4U%cscXtv_sCumqF_=O5d5@n!Q|gnB~}rTv_Plv5{brwXsn@ z=Dm;~$R35|jfY|O(n=p|OLJ0Kb{u9T&y>f;QvA|H-axA!-TYR;1FS4?Ou>FBGg9cM zS-yTF0Vl0vmVm)b9QeWQvp&P2`Z_m;`xqAePQJgtRH&BO`unzPQ{w!rQrFswZGNLn zjoJ_gk4dtjuQm1tOqN4<_+e~nw12M*q*7yrRzXo2rjzDsg*vKb6;-DkuPdDQ6!xyM zQS6V2xZ$+MR#(Yn8Tc&{58W7I+$IkO!wjN~LT?(QK3qZ)t_uPc*0S`Vr*M%4)`C?w zaK1@FqHW(8D8mc?QK^h*m_ne2&{fK(FodnPRxNt&Dt&iwXR=~qV5NqCW05bt;?i*C zSdJV}WhzxDj_-84w0G%m)coqvuj8~Rz_DQ&K}pGK$Wa|sQpDE75Khrvkv1vgZJ2|Z zC;Sx6KnOkJ9d5D<(OA~9NLw1MO61gp8KgcQ)%M)2)vz=mT~>`h;XRWE;^E_s*iJxQK~!I&nyOOh)o6in>7QtRMw9^ zp{tGCVxTpPc!MTcV+?;DM(dB5{QOkM zL}-X#bqsqKR4@Vuj9>8%XHPW#Bm$DJUX6mYJH)}g#sw73hDJ> z<}m1*_x6C{(jZFY_)l6c4ADZ(0pqR?i~X}6w2uS*+Q^rzt?yWs>7?X+NT>=8ZP!x- z>pS+b0Gl5CY+mjQ(#ogRDiL0~1>}TUi+bxzb)&Zi4L5Dq_Gl3^P}wQ{Gn0sIy}};D zOZLfoSyv8gUnrYk^OJNvu+^<#o7g~odn7hc;;5DsDe=U~a9%iE*jp|Q4`3d^QJo-Y z$cD`=oW$n=!)8924PfAbG4}-EFh)Wyp0C>ZV_%*J&r%+>bK5{-6_z7sn6{BKMpf%* z@A^|9qKa}_+D(Q*K@A4#)q+Z8W#DmXG^v5OVV9%1 zL?w$L4LwEL# z`J;R45)3CC%V1qNzhpzL_My9Szs|Rq_(R`$sD+#dVV($&8OUKPZL^0F8sL7qPrRU} ze!mpt3ByrTGJgF2z~ll}Ns`*eqEXNkn&`HaWq(D>ot(_orU6{+Ng!o z6Q8N~(D?Bm>S#f!Dx5#3a}+b2GWeC5!9soSXn*QavWQHx{8}^32=5EhoOW%Kd;_OY zY&3K*H>F5U66(CXNvAIl4BP3SueV6=afiI|cz-@vt}#Hca#=V=V#xKC$zA7J$w5;e ze@{1BqV2nRy);o}nZ-iVGe-JcHCO@~#CPfJgFaEWCKOX>oAN8;Vom%1;jm2h&V#vS zicQ;AP$nKmak19VivVr12U*wZ6_nowvH2dYctg^WpHk5dSh8qgpwH!vU zQ^oErrojbv@wL&?Fej+~5iBK8v%@Qv?c4EGH*a!^uin1pc8{*T<)*DyZ)a!MsOkj3 z@8*QZ1_w}+Zr*c|k!GR>^L(ja(Wqh_pri^8WayJHp#3DP>bP2Q4?R#0Cz-5^Qluxm zkXKAfYIlOslFm0RY8|aM7#3RWhEDSX+v_M{QMs@%X3gV0lR3NF<+lr4Zoc|juLdKp zyK$2X@YZMCYUs9GH*GWc)Z1|Keiq@p2z~Gh(43B+C-x@P)YF)PpLVo}D~+SS-QHGx z@-WeOT1yoUd;F4xm;*1lTajA0jPUBh`BiAx_j+GEhCgsxMQZ8~D;mvoE}YAGPv%ux zVfqM^=8uXtE8ZFU&M5dgIGv*S(>8ycB2*G82@Tc-2kZX1*7GGlpg6f;F+IcvxuIEu znpCx_7%4NJRj+EHG&ra?uZ&|+eX?JH;z(ba^@+O!Pt2*4KS;6@oEFVmoh5p`Mce6I zB5uc4dR!=uv4YfACU|}}ePiAjc+0340^u}f7frwv!o4+2AlI#hy^E$1g?sBMWO!<0 zimKx!^rEAGjJIjLbJhVFqk;9`=ztCJz~Zp1KE(Fe&)96Nb;5!nvcu*|j=t>S^R%vL z6{^>eX?o1&z7frroW8Mlut_Nptv!k~&_3=8b3RL8-u_AN)Vg&PmW(LX$cCegBiIen-q&g)NKUV5&a1_C4(2G${lZ5d1mz%vn#^vxV_I z?$=kWd^l;K9|m|O&RJoIJjb3$kN(ghEW*stZSU9^#>3QVOhO!xqD_49AX`Sg=ih1Z zVBNcyoguR0!#>{RjdOm&e^Oc)sBK726$fgUxc=H&olR6ZX>goWRAf$=79Go|Mw6f_ z`-hP+7RN$+iZy++P#bfDy!l}8Qw0P5s1yy?hw6=8ZjjX_ueP;C^!wgn+)Sg#d&_WN z;|8i6aIyj80SA`j%$R(rx6_NPO&#vJZ~_%{YRCFEZH8|;{K!d0x0I0_N`)k0`U%)} z6M!pR>@+E#E1$UNsvk~4eR!7DhSL=wM)e4^3CXw(WAWS{sFivirWxMQ+ z&4fRn^Q>e&Mshv@V0bWCymVK0$3Yq0U}g?>qc*%4wJDKTXj)j$!7TEQ660UQ951 zVbwb23NNi2%!R%HJ2Pa@<@ZId&h+jlWsWJr3XK_*7Zct#gG1Q5Y1{RMO}B5ld3%8k z=Jrj6>o#3|oqGaz(~d3MkHg)z?Urp%WZ!cA^-pBB)1eOZgr@{8$@Jwc|9?FYUkDuNX|@7F?QU?z)nxZY|C^*v?1sj|6| z`3Bd^){ob5yu;Fg<*xePkv>igWLd$UWV$TMdB14yYn)z;6{?k8+8E1GuyRY35~dbB z3A=IL-kMsN;58kv0!fZg3u`DL^j#lgM&2&forY_C#uYlr7+5Q-P~X_J2s^8qA7O3H z+hc93e4=#GZ24(YlBItY$jI=06HIGeVq12AWum_K!^SeGCVa`l@~>Skjq+}0S2zgv zYvr+k+4FD;>223HULUKOSny2#=-Ad$K`zu((+ktOue5)A?GO4ni=Y+Z%wGu2jp*Bz*>38}B#3#~>fC1HtR2dHzdS7r#Z82d)FeKjK>(!}mZ78eVaHc;# zm6CI-?x|_f$ReapQTkqqp72lnij1uL@DaPJ745>MZWjKmuQ;Ttr8f7qu&mOjjA5@+ zW-w_s8r{W0MX+C{RwVK*J-a54*9Gh1A95i z*ZAPG1e=`cJVW@jmjBV#nx3?wmVlrv9JHCky8o7 zVrVbxQ++(>bJA)#$Z1LG7t?|C!^0{|IsO2bBNCavwbM@Z2gh)`2>;~~w@V$|y6 zR!ynI75mCwOIWhT2TLHnKdr4<&$W(gEq3ktSgNgA!`0(zm%7?U$3B4%+ZbT=GSi>q zqg_5IgvSb?KB2M8xhx;L4!b>r9EgGvu5hF5>U)B#A06TQU$)ciC3p!sjJdjDKGt9| zV*cc%)~Nrs*gixF*6cl8n5i{v{5d$s+#t>r17IBccjTNSwxRX)SYO4)sm_RfW+Qxt zvy<||x7nU|Zvv<(yNDo?l-SNuoc@sk zcm4KlS6{oSaLd+P3)kLsU18JBS6_4Eroye8uinOS&eof5xo*q#e2~8B#_RuUl3O=% zx|7NCx-GXJFHgbjB83ag~_23x%-x8#EHz9dUl);7>?+oIGLE zq>;%M*~{36oKp!u%=s2w8~h;aIf2%wnn*Te;RK{jPVi9{Z#FnTl$QpD#S6fDYu@?j zbureX%-O)H*Fq6&Gi70e6F%~QebX(H>d)NhXHMj=xK7*L_V5Pz(BWCNa z4^IatQYi*CPAB^7wQvtFB_>!N1zBG7I_9Rtov$JH*7^jT&Zt37EWudY>u|8LtLP^R zK5PrG?KszDmeQ7!alCUint=o{Bp z;OoMN%GYeYzHlw?e+t`JnX^dW#`rQkAsh24{UORUeAm6sN<_LlO=K1=4~ zYJaDs?s72JQlG9KIPKu9`84_@=eOpx7YH$sc*QS@U8rY?6>l& zpc1N*zvqkg+exf%+Q&EOt9HN98t&t~pt#q+{*`;v8LIz=-a96BX?fKC`a*yg^MTS| zo&>}@q@K&xJ5?hcRHbrt!^*2HM=AB@O*^)4zHZx-ecG<%O7?&AdAsrmrz82K;{P3= znHSgi`t!Yg{`t?(7uP0K(sL@7j~)0pE&pVl9KKnX;PHY_6%My~N1WLHu#VC`G7z53BPn!T(j~j-$ARAKazP?`8oP}zqs}|57N~>UTu`)qSoKt6gcNA z4tHbRbf;POasDV^gZi6K{`q?Z`>TKrZ4*zY_J0Psw8o|DeEdXfvWdcW3`=*zXNdndpP{%G?KTxunPJL9>E*7c_hYO87Xcn-|$6sI+yZ-|MNSMXaLb=DLk+2H%f9)nm zp0v3zlxky=K0P)x4w3_vZmUe|k6P?+pu!);4HWd(Nr|4*f4`C(ul=VLP+uv8JXt!h z^^l^TBu_hXm7M+Ioi)p}Yt2%QE&pLz$w=$nEp3^k*$JQaaB9NY8!LgJu5SJ~j2X5z zRzfoUM`Gb`2Aztq&hmMNUVM^jCz&3%veLEs!0f>Oa!C27%;SlTm|`}+rx}CA;kw#U zU~kMH*{~pc!sM?t`f=xIw+h3lyH&M{W@t{>Lk8+`{b?f`lR>@_N&c!%-(vDwOnWrH z9S`TWta<#oy~^MW+m6R|4CbM|Zt~l(!0|`Z$%$v{C__t2|8=(ixWhib=DV)s%Nc)` z8jd8il;V#mb%3IUUnpMAPjCLhI?yk2TSs7QnRFV0Ct}mv=Bjn*Djd0n{Z8_!A)iay zA2|38UHGV(#cfiWZ8!WbgwGkR4z|?x>hKNLA8&Ht%d5UZ`&Hl6H)MLVr-R`n5C6K% ze?SnF%*%4M*LMH>^;0-$=X^@vm#06e4yOeYp88#N;D^tplTWOj9bow*ap^&yCg^#SJixUdxn46K#p{Ke@8n$YLGx5M~ofz05f|u1x`tZ4J5K0aN{kO7S!c^+Ns15H3 z!-vXxJ>b6p;LVc%V}bCkR_jHl-!$=Ig5QfK1j{vj_rSJACnc?3Vq;$LpV6aZePD0h z^rN@8s*I|BUK=e%F7ZKZU4!HIF49F3FSKxkl{7!;@X1_hsaS!wX z*ZHV^yFlqgf68@C=ywq21)5)L=={Pm=zQw=Lg=0py#kuwf>8R-+xYK^Pk`QR^w(YI zQT=|x))cMZP1q6WX)|2sV{4&%Q}hz(A)}w`I*%=ej*b4T>v%VH)&Iq&@}~Wr>o}zP zJQ(Qd3th)UD*uqt8(ha%yP=N+nm?`YI6OW4+=9}xaPIgP@;PK$^d@Ml-Z9kADY_T>pr!wu>wJ)(uE;f0w9-G4l2`h}_WUQVV;vFnWZ zt)C8SFZ%1ePUZE6=hHX1&Sxv_{y?L@&R+DV|0~z@)j1&fM^iNI>#+8%+s6-f@Ovj6 zN5b=U2VLg}RoA0|PW0Jf?Ope`uJbpvzcU}6qyNrf^6wn7^p;=eM4;C%a-DOie`lPc z`R$BOtFM2KJ#|`rTk;QPg=;{z!_J{LvIG`NvYU01MR@qE{LK+kuiXg%*WdaLVvvz`y7 zXg#kQ{SwzXspk`cUZ0eIZ=g5KceyM0J*nvjQuHk7rqL1qRmDlrj~V@9H|^PHK_4^v z^DY-NU(9eR8u=MBjXuVIkD}+ZQ?#CU2l|pF{Djs#Xlwr^J8{+MZ^l-8ekZP4>8nP+ znV-~^{QlH)+Baj;=%f6%YI*2ppf_ISa$h|Y`jHgP?^w-v)aW5hb2{{qKqvF-jH5=| z{5r$h+ledC{MzMG&zTRptbKky#vgW3M~(Mne(mZu+UD1;o)pde+7$(QjmDnLY4c1>>_Zm%qJ%&EJ4y0)MqwB#yZ@izWi}|YS5u<;^ z)J1=EJsRjs`ES=})BjzEQ#AVRI%4!{H|;+3*;>0S0xQslj(q`Yt>WqbFnS~L#pvqk zMsFs*1p3QH(^k>bZa2C~d!;9a(n%^>8Ki{6m8T#I-^mOW{@{;^C_I-N&E*L>$UG&9Pp2oJPr3Zq> zvZr5QbPWA=e#eV`P|#TP^k)ODv6p@>+n+)oK`zxFvh8$#$ad5ElI^DZL$)~O`4IW1 z^eJ0Q*GIOJrb+!IL8I$9kBT^`3}53lcgATLp2^cjA5_;z;L9PU!)%B-DZXe0Y$(@} z%IC4bcFO0iDgI=Ne^rWaY%i}LEKcgPH6{P56n`ScA8WTu?NP|7URzW0s)vH&Pr%pM zN(X$O4+pqQN6DB_K3bF{wE8Ul{2;XQvgaKUSUOASL*P_;t?$y$HUFj4Pj5o2?Ur8t z-OWtVRWN>}^?VY{yqVDQKmPxYu=F>*8(QgA2f|IN4}Pjee+I1llIM?tm0v=C3#|Gi z^iim4`t|y481=? zAIQ)LGxVVh{YZvBoS~0q=wnmq+{_F;FGF`{=#3eAbB5lLp@%Z`M26mb481W!Z_dy=GW1Y}p2*PqGxUKBeK12G%FvHw=))QMXofyEmDU<6-QP3jW$5k< zy)i>?&d@tD^iYPL$k6*U^nnb0Fhd{8(2r#3!x{Q$hCVix?wXmQ=Vj>b481W!Z_dy= zGW1Y}p2*PqGxUKBeK12G%FvHw=))QMXofyEm7X~>L(j|5-5GjghTfc^cVy_H3_X#d z_h;w>8Tw#`K9r#!$=&>zgupUKc)&CuV=(7%{UpExZ;&&kl|XXy1A`nn9gJwp#<=thR#m!V&mq3_Gk zAI#97$hzHWK) zH+0v7gLD*y>+{9oF=nenee#o( zfB*iF?!+U+DqnkSg>?B3Rvy36>n-+=(zgePjMDx(9TI(u_s?9JeoI8(_DDcE^@GwR Ja!L8~{uk&NBya!# diff --git a/activator/bpf_bpfel.go b/activator/bpf_bpfel.go index c5b3faf..e22f3da 100644 --- a/activator/bpf_bpfel.go +++ b/activator/bpf_bpfel.go @@ -82,6 +82,7 @@ type bpfMapSpecs struct { // It can be passed ebpf.CollectionSpec.Assign. type bpfVariableSpecs struct { ProbeBinaryName *ebpf.VariableSpec `ebpf:"probe_binary_name"` + TaskCommOffset *ebpf.VariableSpec `ebpf:"task_comm_offset"` TrackerIgnoreLocalhost *ebpf.VariableSpec `ebpf:"tracker_ignore_localhost"` } @@ -131,6 +132,7 @@ func (m *bpfMaps) Close() error { // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfVariables struct { ProbeBinaryName *ebpf.Variable `ebpf:"probe_binary_name"` + TaskCommOffset *ebpf.Variable `ebpf:"task_comm_offset"` TrackerIgnoreLocalhost *ebpf.Variable `ebpf:"tracker_ignore_localhost"` } diff --git a/activator/bpf_bpfel.o b/activator/bpf_bpfel.o index 7061de3be5174fa1fd2fa58d3e9d7d8a6697a7cf..bf39eb39e7ddb90715ed2a8a511b7b628ed8b21c 100644 GIT binary patch delta 5077 zcmai&e{56N701tec7h!eV@Mz-BgBM239B_bs*pn(1hA* zP8n^(Mzx?@q*QI!urMi|Cg#Od%Bo4r`p1Tf>N(;aaKzR+OczC<@co+ z+B8>$p36^NJ~#Ye-8`@H8C(~l#2gQ&E*&$enqsHPIcC;7uQ=Z^zjx{%T_p6I?K~q{ z5vNENc>zDK_bQn-P4DpB_dUbyjE3adKQbfcA=2V}Zz`kh;N_>DG<)59>f&sa|C7=s z7b)dGbo0fgFBCF^;nI?k=^`wTMDmxyRmJWeF>lSNHQjX~)887+2ctV3_uXQ%Wq0kG z-ywtbxCsa71L&;dC?7*FN8O0>nd;5ZDI3hw50#tT?s?nqDHvotAhJnyRv6h}9Q7|@ zPok#$TJ=8Y4h9AT7mPwLfxc19oR;}BqfqaKiBbjqKC`Q=+_bf<&L%zEot-q5h|Cf> zU4gds(3v`Alj`hz8}tRJ<0ucPo`l{4ovR^*Jla(65D9!;^#e$5pXzCmtaDJI3jv2z zKLq`#>W88Cs@@I#8>%0H{)FmBp?_2L4D?gbnRE}zC`O0l+lO*S?E}y$j+k$^Mk1wU zRHbwaOz)MMJ*%p+TcHL~&?47(5s7TW1#Z5S?C>=_G5u9=Kau-U7|)^lK;s{PZpZHg ztAuw!`Ly5{dWhkl7rw&+J{owW#K*-){9=8Zk|eJl5I>GZLFY z$FK1MVQvq5TA|BU)tPrQ{Bax4f4kb5QL@lImJs@hUuS!7XUBo|!=1fHJG&E$_h*hAX^0>1dZIIPFhN+-5clGj zc~v`2?U{oIk9VHPo3r(ypx0@~PpZGXFO%t(_Ir6f(c7^f9kzEpel*kD+1{Pm-_f1U z96urK$?C%V;St7ZS@Y3~UUG^%kBKsyvO|g+EVL;~K|feibzbif!NZ zTCj|u527r3FW|+I-$aQ#!D;2k&|Z5hAbsR4I{Xz3p`0s&>hM0e`e&}> zl&g!7z_=@8%1z*^30KCI9|y1fgDVpjuL0k*xCk7@Xfoj(`@4<@1ZRE2W*E}yaElH% zTp3bk#+#9G&f*k!O!+>%ElVy3;E!s{?)%0Gtx;!T(m&OdMH4e{21^r(Y3 z#HoOsRQ87WKtKj8UIQMnxCnej*&E_?KyF&R89b@{A0$u_#6rrVFo9Yu^XM2>t+GGl zQRVy5-hh`)QrVlLo`Cc#`yCD{zXboqv49L&Tm~Mocp*4@#WFAeJHtze9WY?RVg_8d znEp2`rvFWg=|5>P{rU3c18e_T30VdPL@j1O%wpakZZZ9_45m7yKc;<(>ECQI{gW0i zj$!<_S_TH}w3q>H7V`!vi|L=XnEpK$)BmK!^zXBHVHSpd%fNsEiy1IzF>f$rG5tp@ zrhm?2`j1&m|8a}647g$$7;x2M225B?|LYdh|Axi%ziBc3CoQJG;QgX&AnUEt&~1aa z=%N-gAg1iSXwG7eEe?a5mAzdwfH_vahDE%mIVdUR6wZ{gK}^{cWB)IMp+^np>F_e< z%;II>KIOOAjy=_H@iOp$@)vBsh$&O{6BtuIkMEpWNk_(g&cZMqh8t>l16;bpkr>{9 z9HM`ND^qyWDR0I%?wr$(G+JB=ZdUGPd%q)FEiMJ8lwU!6>}5y#l(T<^A$-x1el>i` z4u0gwi1Hl#qKKuCvDIFN_6g;EXpiF{m{cAE&&G>Cgf|=?fLspdhc5Xq;G!4sKn2cD zo!5NF)4@Nlo0SFYh39qBYUgpi(_$XiZOZ;}omTdb>mG}FT=#j5`RC!o(`Y~)$UKY& zmHoqLM45^5Fv?lX!{~~#KV?^yy(#;|k?R(hf+xwl@x(Rweya-Nw8LuU2y}wOrTAh| zeiB?+;Yz~d+2BUy*U(-x5}f)Psxco9J)Fg&lC}5(XvOBE@bI>oXYCr;cVT=fQSds{ zbx4cCeM){FccJv$#n?5Hg#Em>V;9(*mE4^M*@-1?cgRi24%)D1DBOcd)Xyt(&yn9$ z<{l$|raT1R3_gb~Lg5x6Z^R15!HQ31HTVu8JlP!q;0Cy!b{PPF<$o!2McE^i26F`8 z#e-3}KTwW~v zwMWs3lxybE;T3bsF`qMKn)j8*)&wt*QFCFqaxQ<3u+%#8Wb zMIp1l35(fZQin_Zv)=IlyBEL?`7))@A%v2m5kltT;k%7O=HlVIj6&w(;m)Lxv9~1Z zt*AmWjGt$58CP1uGO)wd7E?`H%!GDW%!g^SnD_2cW*eX7DT|qFzs25v6o#x0BKg;U z@SD)eN!|#;5#)n}bldpM-&Yi$6ZxNB`jfM~!s>}(b$4;;oY;(Lq`b0X_N=m*=3;L7 zjJtl-<=@VI#lin&ii`e<@BmhNd3X{3-vU`$KD-R*)5c8KXwBRgVWF2U6gqe^VlL$Ciz4iU zd6kUo7*tl7L!AJkOB(|zRGTr>}^8DIgfC4+d j$e`-e2Cs_g2$YFgIy%4Ta~NH3Xre0$ZfUdR^_u?ztC(=? literal 46408 zcmeIbdz_qQdH;X!WFyNtoKp~;5X1xtBq79z3Iib!5J(IO1~tyk&dz3s?CwlvW;YwL zI$)~7Q^hK#*6M;FsaQN#Q$-o8Em%?UR8vn4meMGqc=&3pTEFklbzjfylP95m`)Ysx z{q9$0@Aq{**L^?t`Sd(9dDV(5FU|Gzgv9lPUxjHY(T4EAho-q;SO`a9SvvMglkS@z zuOD(~Kp#SB_UfpArt1$0;WTi;YnPa!I_C)U*Yzw8ebDAx2#8zP^R@64{6lgfEX7}N z(+8yvrCXx?4JYsUz2uL}2-m#+KTW>x_9!ddfBydWzn`7S#Gi0ks&(OHGn71+&&N?d zLDWajGIhA`w&)*%elPLdaPsbWKjM|ieKX>9g>UlNdfWcjXZ!VN6@u(O#%EoK>DT>4 z_uU#Tjy^H^e<=O;m80Fqd!w&-Nm974KguO9kFB@G@;SA1vt3BaZMKY4ZnH9vTQ_5_ zmCFe|uMV%Gh-O-;5O1p9tR7xFW%kqc_O9@n@2cM7jo0;NdbG>MhP?T>=x>-C6S6rM%V+a((cdsP*3;(POpibM8|I$k z@x=DA`MBtBm^<6`3<$W*$3=g`+!I`{+)E$*4ReomeH`WxoP zezrMxxa*_8VXlYI`Ss(XzhSPYFL$KdM}Nay&woy%8R5h_cNVsV5Mo)QbPdRq*?>Er|RbFlO$@tk0z67k*c=_fP))%xF|Ef(0i+|d^Pz5%<9&7)Rd z$42`D^7Y>wm%HVoO2T!^CnooR^0{QZG?KsS^0nt&wmKG64EF#3{R1WN?#FsU963Vx z;0xnt+w;1CY?SdTu#qAkyNwL20jsrY1W~JsEiNQ>Xw2oji%HloOxrHoA6jDwK2-H7 zuDq#3pGLDYBHPHMvB#Z{uqN#nucue(>7B>r^Y|pX^I@g4z~@pvnibl7u5j+)t4G;r zPN4NL?kJ6dNgV$h@m_DyhNCpQB=Led4evL^Yklg0uXI-5dUa%5MaR+F^77xxqei(7 zpL8B?iSZt#nK7M~q+OuiuY7HwM)_pra1z%Fe6r)Lwe| z$VTqJo2D}e)8Ug&(CKuo`QR^J572jWtMXRKE=6Tm4!QpPaA#)dbRrw ze3sj&63=ta!*YxG6yOV7--a)PRsU1)HL%6w_Nc9WoCfHY)9U^8o}54yW~!gPlz0~6 zr(90uCzsd^VdG1Zc!9R@k!hPFxg4tec^3XCK9`O-7hX(?R76_CJO}XQM<_V!zL&#F zM?W8TXyUq1X?+TxZD%RB6eF0WG zRh%pNd>H*Q{MCHqp2Me~&o1;ho3{jH)5=e*@kp+N&2uB#Uu=4nj(9+z)>Pwz7hM2&+sw z@PELZiG;u)JA(zuO<(~&2EGw)!>7On@#TzbFl|0`;DYl*@M>6b?tzEkKY>*tuY}cR z3-BA^*TNn6U9jbyLcIsp+M@t>oL9nMf)!^!{4lI|+VDQt2UgpM_j&yA(_z_n;Kj~+ z;EP~O4+XzAv4<jX?>6`HDq+i3j!MOw9 z2frSDxF&>e!b)2K{=RcR{3GW!yw7maW24HoiBp7J6{aH#d$gWPUlPD_dBnEKjC~Se82N$@VA{;!rynk9R7*(74UDI zUjQF^Sj@|n@bS)9!KXW44KH(E1+Rdu-IDh;@Ea(G4%`p_CEUI?ghA&ixa#&Dcmlpc z<@O@lwQ~W!8Md|rzr|S{`WrvIX$$BJ@CooL*~6zeci=hB;bn|V@SU=Um%x7whazJIg~j6gcW}Q?ssm(wZtCYlGqP09>5=xJ^ZKe#}(%w`G7wSw}-gy+<`yo z95!MHe?ej4Z@^!bJ^U^BYj7B5Jqv$bHt>(&f07OS3;3I|sWOJcyJZ9a4*qA^z)v}X zxZnEV~bbKy3; z3I1ES15dz@z#8DU!Jh#O@Ec*-_rq_8{|#=#?}q;!4rAkyb}I7+=kb~)~?`<@GoTF450=8QugpxSa}cQ>=ofv(!;knx8Zj= zci_9>U&(%gH4OY8;+Kc;c{m5=;fLYh!UgyT@M^dX{}jFu*2K8aIS>C1{+;aMLyzJN zO!n|oVTJ9$XTrai%_~?#!S~7rz5=$o{$u8AIP_5da0S+;w*!y6y*9?Lffc7F&s$*~ zJ-6XI;2E-q?}2;a(B}CFd@MK%?zlb=e;HPO3-EVgYd`Rho!jt?qod6fd?IXh0ndk3 z&YGx~I2YinoZE02-i1jA9))N2aIc$KZ@^E1+weBn;)HL7^;}3B{<{-Bd>^cRVAvMI z=im#$0{lhie)ylapyhoubnj^A2uu6~CS+s{p^yxgQ>Ym9IA3a(xHB89qi~cZRS7UIm7HPq@pu0Dl_R zUbhW@9=5!|yPfyI{|>7R!Xm!t2%bDa0U=fO%_IISnFfc40t z0Iz}72HNl@*LUDa_;}rGZclh6e1h!ZH^3*tZTKy)!gk;(=RNR;;gb~h^qz1pta282 z;ETXE{Ex8mI0b(f*4Wa4ABXMS{24vrKjEk9zGw7=BWTPP7Cr^Gu<$eCg2KWJ;Zt?r zXZD2W!B10I_!9W(a2sBm*uyV_pNe^y*AuF+>ZJfTT;HD8({qFKjqvN7Z<^P0!Jos{ z&auA}R(cBad%_1`s)}`NPx!2J2Y%4)!3LYv(EWcQCG}k3R)Ihabluy3lFn*&ih_9PRpx;gex*-OD}0nXuZ8 z+&OM@Nff&_%g|dS=nJsYN;3@Cz?ZtMyp?j-=qe#+&N zxVYC$rA^P{)XwC#!bv)BgmK-yD4cs9fBpphNlG3!c?*1{tZ;9KuflKhnS!mpvH2_K zHvBQ?4%~rPW2;PSFLylHhQEMbydC~$=QqQTI`4pg4lDi+{GZNY2IWscyD$6<=Qeyc zyhir$xvp0MKPRz=FN2BB&QG+yTqk?RX?HlY6PIUuc&Y)6t6y zF5~4?)}QyV*T7$NUJHLMa?-x39do%CIN;@U#`Rt74kg_M6#ovZtmXDO7vP=~qP`95 zAXs_s@HxqO4}1n}{f{`$aW25mb5>hf0nf!&ZRKkC5bnDW`DL)R8+c=457(SK@GD`3 z4TM$OqB?EdxEZ~8A^KZjmi=~afhU(+&!1b-Ge&IN2dkb`=5p^vZ*30#kh9|dlyd>T z-?r&!{|@V39X>ySaeY?Sh9OsszLIiezjF!yS24b)knL~?1Df0);bWJ$oXW^u?laRD za-W9h$1r`B#De4vIdl2Qjm|2o3*B}Yz9e#wqEp$qS?i5iCULnDC&hV%v*LUS zw*2dT2p$2({8}BZ!392BVJjyB-vnDRks<@^ii9z}yw`svG=W$q$+uK~M`90Mxbt(7vTy&S4Oo$=s7b*SE;a3qS_u8iW zsSV7Hwuj%0UVJ#&`!i>@8a?lDK6<)2TbkK!DIQDnUD&Y3wRwCOtTLDTtaAZ=(76qN z)42ohN$g46I~;VyISU4fUimH;i1GH zo=EH&BX5RP99oaP6@Ddt%H^h}ghqp~%7f~Xy zpK&>~`Fsc)iq+P8pMqb5PVPR~`hFA@ovl}O=5nm-K7)-|>Hoa*EcmO=RC(fZ|A&mW zF{B7=uKpG_*59#s7`8nX{Frka{6-KXd&EtU4<2Nye*wxbFHkyal!~6MjV! zHkJ3Jj#a;^uUBL9Hhj7B_`ID@o{!uPJ_SCSR~2WQ&+DjtFoiOETbLhvS zK71Nqw|)R$HgfngX%^wD=>FU^j3@9roZZX)DwXDa&UC-tJO=-Y{X5-oug6_Le%!eq z{xz&RZS&FQQ``|pm-{tw+QJ#{UFeR|A{tK83Fml?C7g;Lovq=uu70JNv6V%C8(eT1 zFPB_{t=~-nl(T*ClUwgLe+3VxY;^5?`s0rK+Bl$gwLDSc-<{(3rnG&hJd854Bgnv4lDm5uVe z6`RYYz-@QdSfMj+hC&+?-iZDlvtX?aCu4%vgC9iS=A*s$T>L3MN%?f(tTgW^nVl8VI%sh z;Q}AoE3EpG_9oJ&oRS_s%tvXLp!0sEe;M2)>@Iw{N%&s;HlJ6*%1fJwx5G&q)b9Tl zz1*$nKY?G65M3^b3o-XeR}#gMjuuw;AuZN_{{n9M#oJ)D?E;^>;ZNcB^Z5X*vTpPF zTUdEjUeuPBmG4y@-EOeMWam!3%{_xdvNdC&)49;$M5I!P1m>K@3?*n ze%SRL_)*vIfgf|d67^Hp=iy($D!&4pdm3{j+=h>J?!bBHK;fS4T!5dQ*u%?Vs?^rc ztKcuds=T6e0j@c>;a56$;N+Z6h5e`KA0%i2ek=S%xDCG({u11Q?|{Dyt3${=#HRp% z06nhn@64Mne~lHNb=}9@=5xsRI?si_===`{346cmzV0>;Ab%?r_A%#wOode&*L{;d zvkO>v4EFhIu|W!H?n>__)jIz1*A8leXSG%6G$i{o+no_bKpM zik-O4=L6_B;&=GC%c-tDi5}PYL;m~&Ob{DKq#XBvu;xpR+cWdiec*8C0{UZ| z+we)w9r#pdWqO`-0bb(VhOdBeef!Ld<m-w~o+}p3ss12{n+m*iG;8kIwv$9Rk z>t;due4dL5?@95)DgOR6+r6}|pbgk3348CfP3~V){6+YNyjdmh^d?txk0eAoQf+j< z{Fo4LlKVI8ZzL{sy|YNbt?*$U$6vr|X9fKCIj_Y32p`&aZy)yBlf61b_eJ!w5BTnK ze?wf~LALl&J_;Y{@vD8GhOWTpG4$eoKJIcIw?oc-8`d)~ZrMAD;)y)tl3)K1crH5K zM0*13GjN#WeMTIek**Vuf78n@nEq(ulRj@M`;+^b-vcT4YcMu7f_=>jY$Xxs#poy6X&ouT&;Ed$tvgb6-t`doacS zp5ndm33Fl_R)5$2{IVE6_zWp=by#K4;nRcz+Kg@U5m~)PEo-yk@WZBFkMLNF_pG9e% z=Qazdp6589gZ{bB3$ef0c?tSUor$k!rSnqw3g>6TS2{BeD|vF12+x*CJK2kZ6x?$j zx;5Be#icxX+yJkR*Fx2~jXaU)vDY&&>}|bx6MC(63b_#8l-R@Xgs0H!W;+u-{5LR{ zEg_$A&XcGIoeS_cVbxJTtoNaKENg4Qz36#tYHPtC!3D)bnzaYxv8mP3z9cMqjVsSm zSa>E8wlMF&$H2BeqaIIpF2H*KSo${nZ1m^oUhwl^tKY*}S2)jtU*w#JOR(nn0z3%! zi0LoG@JgituESd2>zVgfSn1J*^exT>_-@$tIPk||)kT}~*0Vj8gBII=hn}Q{0<1l! zxF3EXv4^#9xdMGSf%lv|PQ4TQFYt1OrObW^U#_&lkHflG2mU2&eUtO;-03m?Hmn!( zufSes=EuWVD(r0bw=h9M2R_qTXRXh6))!kYgDp;YEqs;2!u<);CtC0>;%PGn+z2a9 zork;$UJmBrcfe~1TY&F^uYt80{{(FD!(V`{&A{J)*J9IwwNF(1dUy4^=&w~8;M-@! zdhft{(6?m6-1a#9BHauAIeeY$;ofJU!=wYB3fItwJmUateF0to*JTfD-MAUqE}rM;H%-&;129A_X)zRM^DpyO)ttStnc`}3jcTbZ^PGj z@7|A(_xsP_%Y7SPxBhvQ`_7Vp!?U&b@6a!B-S6Gz?1R{-4a*&J27QAua67E=P3e?- zEFT#!oP-TkW#xru&oJBgPjh_-KGRtVS(@1Qgq{nXFM=;~z8JpRc{#k^`4YI|yaH}H zUkbm<`7-zo&h*J1y&I^s(Fb~NaXsz3=T_$z!0&OUt@nJ`nYP>Wap$Yy&p5B*wVMZ> zUx@ynoN2E;k2up_dw%LnyX^VB^E!Ao?Lg_FUG|*gO#ACO&v`w3p)+l<=LOC$hF|Rb z61eL8Qn=;30e*$^%iz~I^WBr4Tb%ph9nK~AcIN^3Zs#)m3Fkrh^Uf9cE6zhb@yq{< zsLwW^@1a-QnDVx?7{B9uuKXP9N)KCrf9>3c52M{HYzNl)?`C)p{8aQRt3cmf;5G&L zx$vQ=`{65{>+p4M(}v}iaNjAoieB+_;D&Qhr^0a?TSvV-(WBd*;>1SgZ$+& z=4d5EFMSz|$u2{1iVo#m`Ie z^HY35iWy_Nj_J6!A4%hzr44>(k~a8GSo2Yv&xf2l@V&6|9*FCU&UyGN&IMR|2-)<* z-$MT!{5GFQoC{q12~08B8Av(}GtP|jW`XdBz!ndj+^Y@Cp|`V+Be6-+KRXpxHr%qe zfUox@ULu7%>1V-nvDZA6xFpVJrQ$r-x!^{!p_;=o^cND5-1A}Oy92L)lYFg*Rfcl? z&IMTOES5c1wwiau+xb+mNz$+xX5KKTGhyb6P(a#(9<=#Rdgha~%Ozn@=9DHms>90p zCc^4oa&L6*gLlA}zJ3iE=m909#$7V4y%t8+$yp8 zY|7>#=fnon+$X2w&3zsI4mP+x=C$x+Sn;q-vu8FxuuH5P(A(O>WxU+Su%GcP8)|#= zDD*DE3cs#1cIH5O^zw6>j%bzo*ie zKIhx-851^hxjyA#@N$o*;%8nphu6Kot)rtl_m3%C=GVj>sXAOv+_*mG*Tm(RU(GR$ z9S|S$t+{H-o_ROCgm_rq_9~MjaMItlIu{7lc5cIJ)3WLCxfwn4wG~kEyj1NnS+lm` zKf{K3+~(B3^mtUcQ*b}}0{lLhIo;k>ya#4Zw>kCWFmt+9uHK_zPPg&;@6j`l+xstH zf-OB{`m6A;?CGT6ac;wpx{W%^FP#gp))orehPC0fc;K1N;VC?eaxTD6b8f>65_?!q z?N$9v-@CccG>iqBM$^*NC{ zZZnFF`hpx$Soiy(;H^IH||4I;$SLa}M?Ree|n|TjSW{@GFQXFfRQPw)DXN>0vc? z9Y&|Y^|fRdU2fc&sEy6ce$^m;#c{s#|GE;$0_|Qohhy(`It-6_bRSkPZ;iP?*l%W#+cz2>%K3wc)2Zci;u^Ym^o~i=Er>^WoP@Kb`2DxeJ`;W&_u2y|-(l&eu-zHyo@yg3O~Sd@Gbi?{j+CBctW$c# zv-l`I;yj;%GoHCOaIXS%2@1vA;d6!a9$4x5Q{|n{I{3|E3R3I)Tg2XL-vYOfLdKDl z?wbCMvT^sO6k}uUr%0Tu{Ce5B;4*nMo1_e-PRcM3CuLZGlQQgw-R)Oa`9y0?H|AAk zSa=fs5w2HWNLJX$AImRXPw^|@S)_A2`v0%n!_C}Fdo8txH#rZ(w>s-gXUcgJexGw2 z{%hy$u=bg6r7k+~N6}jwR$)Y!`ytmpjh@TB1q$FF;iQjz8NOBZIE(KU!aKwy>W8q| zBPM-W(d|R8`YrI$n)B`0w_$e#iPv>|rijFuUU{8PmR;IkPa4)QwsCJ8>ohwLU@DZ` z@3%hsvH1(`tMx+_mY?S{4$Gb-hCg;@zBczx>nP}=Tg({(K4w#J#y!byDp=uGj4FT;vA&*y8-1^CJ>@3gex zqh0@ISgyzC@+t+wvNpU$i#Cq(?%Sc;P}#e) z@H@Gt@G7@K5C6VQ_`gnt--z9W+r8iI{uG&c{%FPP!>zRq#VI$B&xuBn+3%Q>{^==x zMv55@%*p0$#7{9;{C9g=6!$&O^8OMVe6zXB^(vD)V_Hcb?^C4sQYkFithrM92UD!~ z*brSy-bSkF2Wn|&$$(JZDDy}sj@@WxmWmY&m6Hjn0~Z5E~Um!x=Aimywt_U<@~ zb7M-c-#N-5r&Vtsc%2|H>nEta{y z@1E$lr|jRFVttP}v3Y3DG*6{$bY7C!+>_FOEX7|;@%K~wvlRETc}wD%mEsdqd~KZA ztj)|x>Ca5@vJ_vD;_FhZ-|9&2)t}Ojrg$R7p*q$I#o|S4)?9R5@tVtDyrQ`Fy45R+ zMRWs|MN9F<`8z(=tZp2u4Cb4|wMHwnw$>|Qv@}_)jI|ooN;7P#Yz-5ok?~5XSI5Sx zV;hNfW24e+78{kpYNJwaHA5w<7+B1LGUsnM56bbT@qx++mZibLMzc7v zBvk7YOHB&n(O;IeQCT!M6$i$LhQdf?EYvq~qgW7)O=0tRWxP@xE!8Q2jiKJCwQA+s zNSGKYjTP&Z1SJ)PR=JvJvB9!hYgAiXy9HJqs*Vj-CPU)4%EhQC4}{@Tb2y+VkCd9t z>R=cwwMzDfiWm%IrFyl9cBnL39oZThmC;(OQmodOgppdgG-A|MF6}CpCYE(oZ8SAh z?8VUvw`et6rO|rWLQNKx^^MJ1v02&Jj8J`f#Vula3~zD!p^@5_B2A+inw$7Hprj}5 zflOI@Ko=ICzv#Sh_BqL=TI1|;$m1qz)xrFkN=E+q`FT;AKTj1szucN!aDlEo*DR|; z`FUpdtbCK?)Q0ArS!oT=&rgdK!~FaOdCL=pH9voYZo4#dAC=>i-%D4Y8;E##un~rf zgOz3ruUQ*!lq-N$InXAsf+>87<`49V@9t;Tq{m2Z_86r9hLkZM1=(?&*_@_3`c08rHBdBmFv z6I#QKN@GoQlitLsg;EQFxHz2;Cv2rg76k_p>yWnY9`kvP*-MP?8s^s?0R4nuSVPB)O7m zzne)hFtW)r)s=M9Snt;QEn|Ty8ny*MMx=p7&FE;dB!GrW<0GwRF{zI31Z+8oohjx| z?Jp`>$WZe2!Ahyc;2ovwYXfD5);i6p##J?!m@v)UtsJnpwHDuO< zgQ2yNKPc%C<Wjg=TeNM3Khqj9BCIF^{ChzD_t|@u5?< z;6u9}+r+dP$5ZP*6RNf1U~P;cRl~4uQEXveCR+M@SKfjhO`PFvLgVjvt2#;$`@7Np zGU`mIxtlOye;FHzyHcmkF<8X@Iaal98VkUzyew+9mWiX3+t5gh5$byL>$PTeGKrY6 zEF%|{FwZpUWrY-r&M5vV*-*r6Ft-0tj^7HsOHajK1@rxI)i(jJ8Q%=!`2@Wz;p@(4|i zW;CilwSjS#1eQG&cZqbQR1^=*mpw$xe#*p@P*#WLBl^o>SNa>p!zf|)q#Ai~-fs+!`66Ta<&w{&J;Qud?;GylYbu zY*uM%ZN)afRV78!$N*21WJBL*ZH-Kn!;J7Fw5hT3RyB}XgB4nhg32g^O0H3CQnak1 zn%u|Aiott7^``ML+8+~fD`<_as#24s<99_y=+-d(Hn}hqXAngcYtt3YkqVY*Lo}$d zmZb(wVu&oV7Ob;@vn)l6>Gn+nrF$`c)M}&ZrU(>6tSZG*9Hy;yS1r08Y6I8vm&uBW zft4EL8;g9ED?=JXIW0#6P;GyySen=vHfirtZZ&N6XzMsF3K-aE8L^O()leWg6iJD; z9=mW#wu1CYnbC$hn0dmcXgWfy5gTxmU5NU!mPOjqXjP&?O-v^>e@yeMkLRPblL_04 ziZ;d3U)a?(*v$oLwy;j5Q7N`!Q3@@%*hVngDQK0UpBfRzG;N|HnW_dj*BB$YV`XMB zRKsj4*xxJ~WJqQGXaPfG!WRSGS;QJN!CK>thcx-_k&qo8>5wSNd{L{fSPo=Y8$HQP zTydpCb5o78#Vr$Hr_O?*p}(uCkU4+^(*U zO(dGeK+JNn2*p()wLZcehTO2x9vG<%QHUJ>sg#Sulxsn{42lR`$P(x|M34*$8+}xr`#yl`?=9Ad~1tT!cJpl%c(HM*Al{!22<$4@hs$+g` z8%0`$$H*C`Z{%E2*E-rpKCL%6NMjyhkhcjpaJn|cI-}0XBa7DYF*^Vb110R#kJ&Bh zh?%_%OHVB2bBN5ALs7&tS?e1~6q|H`&h0nRaAO@|<(jpZU$tUQ@#+iIIJ)($iR!|7p5?qNl{9E+HyJwxDHv?l zi@GZ-1LMkAk^*VtE=O~TPExdvWUEt~P-?Vhutfmp653dE=$^F1h7*rvXk83`$%b0_qwK0>oo_Mm$GS7rLe3!06Y(+~ zIc}wW_HaQR+@||v6x7sjOF^#C9VI3G$My$q7g=|bU6c|#8&vI*Et70rnI0iElG);D zsKnN5Fg11-OGC<|7ET9VQ|(dW2fwJJ1%;|{{+y0c%6Q6PD>KJpbL&_+btzdyrd77q zOe^B^g0yBjwMo2DQ!KSwI+)wqC3t?}4+K2&YcL1^V_ z;WQGbFi<6Soo6KnO#^%_bgV+&H}iODvd%J#g`_D)2SPoz1WJ&xOJ^U{iK;b`m_nx| zTN#%c+W(KlWwH$(%q{!1Y2ON}iN{`CYLs~ppiTA=>smd6vTYEX@3ERSBppePgmGK> zC~oDU!V@%xp{QNVVFWu>+P(W^xX3QPF;*Gj1l2B~QUGN;K4MwBmXYe}6+!+*YcIdn zX!)wER`cIOG)7x>odDQwPCPy|NHOV;ja)>enW)7)Un$EQldl7m)QzJU>m+t)n`D)o z&?s%82CDHSlXX!_G{KClU{Y3_6Q-1OzUf}~XthPR&|)`Mnssa|qeMmG;=-6UkBv+| z*j=k}yLkE47cIA9aPg8WSA>YyTz8Grx@%UfalC9DgS^cmoEK3ZJOcDV$L5KxiM0P| zOwpztE#fK@l;37+s}?*?G$w1Q!eNgsS@1dV(r_zL3ztzIT?AW&#(l4i#p8?zL90kj z{c%O3nJ&b0IUC8WNNY?VQPTWT(`Lm6L#vFUr6K4P#ZKGoI7L(uRYXHg;ZW1AYdv4F z4#mj@i|JuD$Sutpq@=D@#b}lOtaR0jm7yU$d8Hpq8c_Qcltu@ttWU!AjKrKe*+G(@ z;B;ly>MYTlUD;3P5_3Pc(&b`loE4HF7fv1dmAP`SuHjxFILj2q9CWz-Q zM!kuw;l;n*tBCQc_bHOjD4_=(<#C>-@yuBVWb_8se`AB*#iNSDvgR<`W1F$rSnGs^ zhR6<^Pjd9-7hkS*Mfaw944Ib4eeN68e97q>dk3GCqR`!=s05~uyW*VB5}2odDtA)d zJqk;Dlv3o~F*K~DXk;#Kj&rk*B&1hoPuSY#os%mLnzV<}?7|>MDQro~qHHBn=5Yh7 z>*Byx_Sazx&7beoH+0`Swni9X8Vx2P z4oE3Y#^NEijC#($(fy&O4KEv`=f{TwJjt8j{Dl9cv^dyUnw%;QHl7{Ijm0{fsB_W~ zIH{4MJ>qAz!pQ8C zNPG|wr=EBir(vCCyV@I@2|J%NRk9uxIKF&-*#9W)DA$u;{7lkI%yG_a*Q^XY-Gbj%xtZl{t zVfBhNmljuCyW;A#MK+jgR}?QLqVBcfi7wi%K#i5Bmh6Q+TI-%6b__j))h5g4FP)>XL@DXv7{{PeN)jm$R;y=?t>9LF;(9awH^ZXF%q#6T@8wkI`R7Ueu&^yf7}55|i1 z+9qv`HBhi}O9~}UEsP}W#(8?{#li%y>3|hTa)er3#2sSY4bW#a+NG+~xxs5(v65Ut zYh@K`nN6Fpvuaomn=X03tZkJAO5d6-KUqqmv`2yT46B>iw9ZZ1mLFibsnvemSjM7> zZ?drbo4zlZvSDUZJP5Y6@_1zSJe)#W+qKN=Wi>a8Ba66{cEtB3etg(=rtw zhk?p_4CD%b&cwRHauoL;aZItb!Za2egZ0sJ&?_JoNHYQrs8J%xj9{_$h5MzJCX5LU zi6{7abZWVc_nOW)(;u%&X>hA=X=u^NBBaT!^u7``VOMNLM$`k05u55Y?ZQ>uEc{tt zaY)rjZSEU!S*2GQ<6fsqXVPpmwuyy`u&kz5Cjwhik#)=eqL?#pF42pJ!SR7KFDf>w zJf@1TPL0$!DzMGc^xlkNE~Bv-JTN82M_qsnG$WdC^{R{{(@;#Xj$AmyNtAKyJYWv- zG1c33dQ^&se4dfPt(@emfACs@PtH`H6JuI;{L$8$n)I%gKrAf9Uep7J*5sI~(|fd$ zb=>Jr9dE|;SSgKaP>J1Qcq{8uy*y|!X|){fw4}7fbQG<7cw)Ja9pG|AqUP`U^i%zz zafaQAQe#nZaEtuKMf1+ZGiyDi?j~A7jnNx#gtesn=2$J!=)}RyS@9<}dRA)e$5Y)P ztRM-^t48JzYkroZ(VXyPsz^d!SjYUiNoXcl1?K%vwR9m!&&%)4n zPG~F(>=SsgjSkkVGW`W!+U12pbgTfHlj^&i%krY@NZ2yOfhd${#jR>+Ziz$v*eLJ+ zvYlox!9&mynrj&0Wep}H=1(4KjoG`!{vt|j%{IctnOfb(&cQk6hB#AM<8+MDlt%}{rEAw*w0uQz)#_`C%dfhmxZ>)IF1~U_@tTzvt>HLl^;N4b zx%^UINMCW~rT=f1t58f@WUCnQ`>uCQrR&t!{iHvS^#{(^OL-bL32KiZlm%IZ~<$%ZVRfJ}>%yp+Y0 z4bBfWN<(q+0@!HHGao%Jru8UtHgKx7cnh|fYGFf@=Gh~Xh()?}9ARm?UF-4|c7c3a ze1;f(hG^15wHhBaVz%yj@pNc16=F#Jbh6xR#J};O#3ak3Xv>4%z}%D!=ZlEFyFP)_ z5w(bkB^Yaa9S+tum29Hm#kTm^j&n_BDQ!9FH*C_26{1e3qug%I? zd2#B+;P^DWF;*U}&x>YjFS_Oh#B$YD#Vc1_eO7*+&Q$Ve&X3EJ0iKGHaxDet>n-es z$-Bh5z2edW)}K;7*=xzXoLsFhxWHb0KFiX4(bdeSGn}K$Hon3(sE?l#H}flB)Hk3oh^&Rj605Sig4VC2OAS)pi9} zu>Z;H?TRCwj^vYs|8KlzURq@7=Y9M9D_)T=El#AQ=5$+LcHrf-{F7C3d}m#T2XhuG z9&YoDI8~Xv#Lw3$82skbk`|u7)L!1_1^(14{3^53q6HUN1)ZIIN#-Th4SIXOwD=$w z(#bwpY82y~?)Pqrob#1N=FzyRPH)`@`BK1A)i0EfCY}y$Ukgwc7JDYV zH+5r6*};shLri~{WYN<6nPZhL zJU?uA==Q1@$Cx`jM;b{EK&(q;wA|9$39t5W zYQotYD}h*C^Z0TYGi+nLf@S(kV(~kJLAUVAvN&TcK1s5ZNDrD=m9^@?+kt;`Nb#qd z$AOvHui5;8Yz&n~n#xC!y)j?1VL^7lx@px{_n#a!Vbq~(4{dio*V21v<$+lrpV@K1;iD&mHLrY70JKJ9F z@Yk<-uPb?T#?DgXk%X2~e3?=QC|dZ%+iURClfSqQv_)?B2#hV0PD2=p*z`_ol`+aH z9=XQ-PV%ZDuS@zD9Be}uUutG?o7~N}8@3DKbw*EvFSV^WEW`TpCI`Me>MKq^>f8Sb znV#(FU^t1x9+%k*1hJ5LSg!o~?w@Zz#glf7->Z)L__}oRigoY< zEISfcIq1~{Jul$2by!bv;>|hcrl(W$swS1NCnKJTM|Jnazz-43RuAdp>$cG-IS{mW zWzE8i)ZVC#&j{m-%6dFtZvgOQ$-XQQ-_`0q=(J4}FDBStG!a;?>AeTGEjlUb))E`@ zqP<2>8S4dmucmiD))VsfwdMGGZBK>ov$2t^U%;5p4{YRJ*7Z}lzAz=9g{!l?xwq`BZ*O=g@m|_8Z)~|5D_6U(ba-e*KU|{0PWyE>IqJ9r@7j@2;ey>sC&pVgj`YBK!e%;FN66(WcE5Db{u>W&f_EvuVmi@AnEc-%Ami@|< zEc?|dS@!Ewvh4d)vh0Ubvh3?AS@x4DS@!LeEc@*#S@v&E$+F*(l4U=Yl4ZX$CCmPU zDOvWrQnKtjDOvUpq-5DYl#*q?J0;70PfC{k-jpo+CsMNP_qlB22aR_YplAndV+T~ODou4CjP+uweS;+k^uh#WT-8}b-6pgrew{p{Vpf-OFboPewlPRnP0ZMoXjt8PRW{I zcDkI@*9Tp;`eL%`q~s%zce{Ki<#95T*Ap&3hJ0vWF6?u;huqO%)yuAr_aJRWVzXVB>%=`ZehP~{6flBZY3Y8F?`i!tt}*9OEt-L z_~>U|dgyC%3z>V<*J(cc$(oz#1|f-GbMrgUy^pz9ZWuWo-#)*79DP%6HR+IjB2Oag z_sdT4>+OvE=8U|MzLO5GzOV4#OX7F=I3*INzOCykT~=R~{Kqc0k>7)S5`D`YZCUdD zF01dQudD5+WwrJF>ci!3eITdHYbA2JKGYV|*VPu&<*BxoPLJA3TJ}^Q)SICWmT@KI zW7B%#mphI;A46syh+_9Q;v^pv$h14PMY-eoNT0|AH^=X0T0Bpf|uCCPQoeR7f&UUP+ym86(=~QnI4no0654U!-Iu<%p~K zgVQx-ZFWjl))u;~Ye(`?-;tB7GT#jZS@k&||j*l*8VPwF?hzB6-u zS4MsyBk#`0do%LBjNGS8;gb8!&dBpJ^0JJ)G9xGLFNtqBbA2)+Z_mg(GV;!hyelI= zkdb$1brUq)uS+duyqd0s|ldfv}|Wkz0~ku~4QCFL=hk+)~$9T|CNM&6Z?AIQkN zGxFYyye}j7d4EsRH#;NG%gD~z z-jdu-uPRn)A`THr)T6P8Tpcoyfz~bX5>~z-jdu-uSz!-TWVM!bwLRee8+H9e>P`v%=Z-pzLhEh<^6P zYcD;Uw_Be4319X}Ew8?0e;Dsaddr); bpf_map_update_elem(kubelet_addrs, &key, &ip->saddr, BPF_ANY); @@ -212,8 +213,8 @@ static __always_inline struct in6_addr* lookup_kubelet_ip_v6(struct ipv6hdr *ip) } // bpf_get_current_comm is not available in a tc program on arm64, so we use // bpf_get_current_task to get the comm. - struct task_struct *task = (void *)bpf_get_current_task(); - BPF_CORE_READ_STR_INTO(&comm, task, comm); + char *task = (char *)bpf_get_current_task(); + bpf_probe_read_kernel(&comm, sizeof(comm), task + task_comm_offset); if (bpf_strncmp(comm, TASK_COMM_LEN, (char *)probe_binary_name) == 0) { // bpf_printk("found kubelet addr v6: %pI6", &ip->saddr); bpf_map_update_elem(kubelet_addrs, &key, &ip->saddr, BPF_ANY); diff --git a/shim/config.go b/api/shim/v1/config.go similarity index 85% rename from shim/config.go rename to api/shim/v1/config.go index 91e8776..f120604 100644 --- a/shim/config.go +++ b/api/shim/v1/config.go @@ -1,8 +1,11 @@ -package shim +package v1 import ( "context" + "encoding/json" "fmt" + "os" + "path/filepath" "runtime" "slices" "strconv" @@ -15,6 +18,8 @@ import ( ) const ( + ConfigDir = "etc" + ConfigFileName = "shim.json" NodeLabel = "zeropod.ctrox.dev/node" PortsAnnotationKey = "zeropod.ctrox.dev/ports-map" ContainerNamesAnnotationKey = "zeropod.ctrox.dev/container-names" @@ -42,9 +47,15 @@ const ( mappingDelim = ";" mapDelim = "=" defaultContainerdNS = "k8s.io" + // DefaultProbeBufferSize should be able to fit kube-probe HTTP requests with + // reasonable path and header sizes but should still be small enough to not + // impact performance. + DefaultProbeBufferSize = 1024 + DefaultProbeBinaryName = "kubelet" + DefaultTrackerIgnoreLocalhost = true ) -type Config struct { +type AnnotationConfig struct { ZeropodContainerNames []string Ports []uint16 ScaleDownDuration time.Duration @@ -63,7 +74,13 @@ type Config struct { ProxyTimeout time.Duration ConnectTimeout time.Duration DisableMigrateData bool - spec *specs.Spec + Spec *specs.Spec +} + +type Config struct { + ProbeBinaryName string `json:"probeBinaryName"` + TrackerIgnoreLocalhost bool `json:"trackerIgnoreLocalhost"` + AnnotationConfig `json:"-"` } // NewConfig uses the annotations from the container spec to create a new @@ -156,7 +173,7 @@ func NewConfig(ctx context.Context, spec *specs.Spec) (*Config, error) { } } - probeBufferSize := defaultProbeBufferSize + probeBufferSize := DefaultProbeBufferSize probeBufferSizeValue := spec.Annotations[ProbeBufferSizeAnnotationKey] if probeBufferSizeValue != "" { probeBufferSize, err = strconv.Atoi(probeBufferSizeValue) @@ -191,8 +208,22 @@ func NewConfig(ctx context.Context, spec *specs.Spec) (*Config, error) { return nil, err } } + cfg := &Config{ + ProbeBinaryName: DefaultProbeBinaryName, + TrackerIgnoreLocalhost: DefaultTrackerIgnoreLocalhost, + } + e, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("getting executable dir: %w", err) + } + b, err := os.ReadFile(filepath.Join(filepath.Dir(e), "..", ConfigDir, ConfigFileName)) + if err == nil { + if err := json.Unmarshal(b, cfg); err != nil { + return nil, err + } + } - return &Config{ + cfg.AnnotationConfig = AnnotationConfig{ Ports: containerPorts, ScaleDownDuration: dur, DisableCheckpointing: disableCheckpointing, @@ -211,8 +242,9 @@ func NewConfig(ctx context.Context, spec *specs.Spec) (*Config, error) { ProxyTimeout: proxyTimeout, ConnectTimeout: connectTimeout, DisableMigrateData: disableMigrateData, - spec: spec, - }, nil + Spec: spec, + } + return cfg, nil } func (cfg Config) IsZeropodContainer() bool { diff --git a/shim/config_test.go b/api/shim/v1/config_test.go similarity index 99% rename from shim/config_test.go rename to api/shim/v1/config_test.go index e7a5ffe..572232b 100644 --- a/shim/config_test.go +++ b/api/shim/v1/config_test.go @@ -1,4 +1,4 @@ -package shim +package v1 import ( "context" diff --git a/cmd/installer/main.go b/cmd/installer/main.go index cabf27a..144017e 100644 --- a/cmd/installer/main.go +++ b/cmd/installer/main.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/x509" + "encoding/json" "encoding/pem" "errors" "flag" @@ -21,6 +22,7 @@ import ( containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/cmd/containerd/server/config" "github.com/coreos/go-systemd/v22/dbus" + v1 "github.com/ctrox/zeropod/api/shim/v1" "github.com/ctrox/zeropod/manager/node" "github.com/pelletier/go-toml/v2" corev1 "k8s.io/api/core/v1" @@ -31,12 +33,14 @@ import ( ) var ( - criuImage = flag.String("criu-image", "ghcr.io/ctrox/zeropod-criu:v4.2", "criu image to use.") - runtime = flag.String("runtime", "containerd", "specifies which runtime to configure. containerd/k3s/rke2") - hostOptPath = flag.String("host-opt-path", defaultOptPath, "path where zeropod binaries are stored on the host") - uninstall = flag.Bool("uninstall", false, "uninstalls zeropod by cleaning up all the files the installer created") - installTimeout = flag.Duration("timeout", time.Minute, "duration the installer waits for the installation to complete") - versionFlag = flag.Bool("version", false, "output version and exit") + criuImage = flag.String("criu-image", "ghcr.io/ctrox/zeropod-criu:v4.2", "criu image to use.") + runtime = flag.String("runtime", "containerd", "specifies which runtime to configure. containerd/k3s/rke2") + hostOptPath = flag.String("host-opt-path", defaultOptPath, "path where zeropod binaries are stored on the host") + uninstall = flag.Bool("uninstall", false, "uninstalls zeropod by cleaning up all the files the installer created") + installTimeout = flag.Duration("timeout", time.Minute, "duration the installer waits for the installation to complete") + versionFlag = flag.Bool("version", false, "output version and exit") + probeBinaryName = flag.String("probe-binary-name", v1.DefaultProbeBinaryName, "set the probe binary name for probe detection") + trackerIgnoreLocalhost = flag.Bool("tracker-ignore-localhost", v1.DefaultTrackerIgnoreLocalhost, "set to ignore traffic from localhost in socket tracker") version = "" revision = "" @@ -60,15 +64,12 @@ const ( configBackupSuffix = ".original" templateSuffix = ".tmpl" caSecretName = "ca-cert" - defaultCriuBin = "criu" - criuIPTablesBin = "criu-iptables" criuConfig = `tcp-close skip-in-flight network-lock skip ` defaultOptPath = "/opt/zeropod" containerdOptKey = "io.containerd.internal.v1.opt" - criPluginKey = "io.containerd.grpc.v1.cri" zeropodRuntimeKey = "containerd.runtimes.zeropod" optPlugin = ` [plugins."io.containerd.internal.v1.opt"] @@ -239,10 +240,11 @@ func installRuntime(ctx context.Context, runtime containerRuntime) error { return fmt.Errorf("unable to connect to dbus: %w", err) } + opt := optPath(ctx, runtime) // note that if the shim binary already exists, we simply switch it out with // the new one but existing zeropods will have to be deleted to use the // updated shim. - shimDest := filepath.Join(optPath(ctx, runtime), binPath, shimBinaryName) + shimDest := filepath.Join(opt, binPath, shimBinaryName) if err := os.Remove(shimDest); err != nil { log.Printf("unable to remove shim binary, continuing with install: %s", err) } @@ -256,6 +258,20 @@ func installRuntime(ctx context.Context, runtime containerRuntime) error { return fmt.Errorf("unable to write shim file: %w", err) } + b, err := json.MarshalIndent(&v1.Config{ + ProbeBinaryName: *probeBinaryName, + TrackerIgnoreLocalhost: *trackerIgnoreLocalhost, + }, "", " ") + if err != nil { + return fmt.Errorf("marshaling config: %w", err) + } + if err := os.MkdirAll(filepath.Join(opt, v1.ConfigDir), os.ModePerm); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(opt, v1.ConfigDir, v1.ConfigFileName), b, 0600); err != nil { + return fmt.Errorf("unable to write shim file: %w", err) + } + if runtime == runtimeK3S { // for some reason, k3s containerd only has access to the busybox tar by // default. This breaks criu checkpoint since it needs the full gnu tar. diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 7dc3831..2ad856d 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -40,8 +40,8 @@ var ( inPlaceScaling = flag.Bool("in-place-scaling", false, "enable in-place resource scaling, requires InPlacePodVerticalScaling feature flag") statusLabels = flag.Bool("status-labels", false, "update pod labels to reflect container status") - probeBinaryName = flag.String("probe-binary-name", "kubelet", "set the probe binary name for probe detection") - trackerIgnoreLocalhost = flag.Bool("tracker-ignore-localhost", false, "set to ignore traffic from localhost in socket tracker") + probeBinaryName = flag.String("probe-binary-name", "kubelet", "set the probe binary name for probe detection (Deprecated: this has moved to the installer)") + trackerIgnoreLocalhost = flag.Bool("tracker-ignore-localhost", true, "set to ignore traffic from localhost in socket tracker (Deprecated: this has moved to the installer)") statusEvents = flag.Bool("status-events", false, "create status events to reflect container status") versionFlag = flag.Bool("version", false, "output version and exit") maxConcurrentReconciles = flag.Int("max-concurrent-reconciles", 10, "num reconciles the pod controller processes concurrently") diff --git a/cmd/shim/main.go b/cmd/shim/main.go index 61fbf89..31d1969 100644 --- a/cmd/shim/main.go +++ b/cmd/shim/main.go @@ -3,16 +3,28 @@ package main import ( "context" "io" + "log/slog" + "os" "path/filepath" + "strconv" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/manager" "github.com/containerd/containerd/v2/pkg/shim" + "github.com/ctrox/zeropod/activator" shimv1 "github.com/ctrox/zeropod/api/shim/v1" zshim "github.com/ctrox/zeropod/shim" _ "github.com/ctrox/zeropod/shim/task/plugin" + "github.com/opencontainers/runtime-spec/specs-go" ) +func main() { + if attachActivator() { + return + } + shim.Run(context.Background(), newCompatManager()) +} + // compatManager is a wrapper around [shim.Manager] that allows us to control // the task API version. This makes it possible to use the containerd v2 shim // with containerd 1.7. @@ -56,6 +68,33 @@ func newCompatManager() shim.Manager { return &compatManager{mgr: manager.NewShimManager(zshim.RuntimeName)} } -func main() { - shim.Run(context.Background(), newCompatManager()) +func attachActivator() bool { + if len(os.Args) < 3 || os.Args[1] != activator.AttachActivatorFlag { + return false + } + log := slog.Default() + pid, err := strconv.Atoi(os.Args[2]) + if err != nil { + log.Error("converting pid", "error", err) + os.Exit(1) + } + cfg, err := shimv1.NewConfig(context.Background(), &specs.Spec{}) + if err != nil { + log.Error("loading config", "error", err) + os.Exit(1) + } + bpf, err := activator.InitBPF( + pid, log, + activator.ProbeBinaryName(cfg.ProbeBinaryName), + activator.TrackerIgnoreLocalhost(cfg.TrackerIgnoreLocalhost), + ) + if err != nil { + log.Error("unable to initialize BPF", "error", err) + os.Exit(1) + } + if err := bpf.AttachInNetNS(pid, activator.DefaultIfaces...); err != nil { + log.Error("attaching activator", "error", err) + os.Exit(1) + } + return true } diff --git a/config/k3s/kustomization.yaml b/config/k3s/kustomization.yaml index 4173476..99d3d9b 100644 --- a/config/k3s/kustomization.yaml +++ b/config/k3s/kustomization.yaml @@ -10,7 +10,7 @@ patches: kind: DaemonSet - patch: |- - op: add - path: /spec/template/spec/containers/0/args/- + path: /spec/template/spec/initContainers/0/args/- value: -probe-binary-name=k3s target: kind: DaemonSet diff --git a/config/tracker-ignore-localhost/kustomization.yaml b/config/tracker-ignore-localhost/kustomization.yaml index ecd7d72..6921ce0 100644 --- a/config/tracker-ignore-localhost/kustomization.yaml +++ b/config/tracker-ignore-localhost/kustomization.yaml @@ -3,7 +3,7 @@ kind: Component patches: - patch: |- - op: add - path: /spec/template/spec/containers/0/args/- + path: /spec/template/spec/initContainers/0/args/- value: -tracker-ignore-localhost=true target: kind: DaemonSet diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 8faa191..0a74c07 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -11,7 +11,6 @@ import ( v1 "github.com/ctrox/zeropod/api/shim/v1" "github.com/ctrox/zeropod/manager" - "github.com/ctrox/zeropod/shim" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" @@ -191,7 +190,7 @@ func TestE2E(t *testing.T) { "pod with large HTTP probe and increased buffer": { pod: testPod( scaleDownAfter(time.Second), - annotations(map[string]string{shim.ProbeBufferSizeAnnotationKey: "2048"}), + annotations(map[string]string{v1.ProbeBufferSizeAnnotationKey: "2048"}), addContainer("nginx", "nginx", nil, 80), livenessProbe(&corev1.Probe{ InitialDelaySeconds: 3, diff --git a/e2e/setup_test.go b/e2e/setup_test.go index 4eb22b7..0ce2ab8 100644 --- a/e2e/setup_test.go +++ b/e2e/setup_test.go @@ -23,7 +23,6 @@ import ( v1 "github.com/ctrox/zeropod/api/runtime/v1" shimv1 "github.com/ctrox/zeropod/api/shim/v1" "github.com/ctrox/zeropod/manager" - "github.com/ctrox/zeropod/shim" "github.com/go-logr/logr" "github.com/phayes/freeport" "github.com/pkg/errors" @@ -196,12 +195,12 @@ func startKind(t testing.TB, name, kubeconfig string, port int) (c *rest.Config, }, { Role: v1alpha4.WorkerRole, - Labels: map[string]string{shim.NodeLabel: "true"}, + Labels: map[string]string{shimv1.NodeLabel: "true"}, ExtraMounts: extraMounts, }, { Role: v1alpha4.WorkerRole, - Labels: map[string]string{shim.NodeLabel: "true"}, + Labels: map[string]string{shimv1.NodeLabel: "true"}, ExtraMounts: extraMounts, }, }, @@ -378,43 +377,43 @@ func annotations(annotations map[string]string) podOption { func preDump(preDump bool) podOption { return annotations(map[string]string{ - shim.PreDumpAnnotationKey: strconv.FormatBool(preDump), + shimv1.PreDumpAnnotationKey: strconv.FormatBool(preDump), }) } func disableCheckpointing(disable bool) podOption { return annotations(map[string]string{ - shim.DisableCheckpoiningAnnotationKey: strconv.FormatBool(disable), + shimv1.DisableCheckpoiningAnnotationKey: strconv.FormatBool(disable), }) } func scaleDownAfter(dur time.Duration) podOption { return annotations(map[string]string{ - shim.ScaleDownDurationAnnotationKey: dur.String(), + shimv1.ScaleDownDurationAnnotationKey: dur.String(), }) } func containerNamesAnnotation(names ...string) podOption { return annotations(map[string]string{ - shim.ContainerNamesAnnotationKey: strings.Join(names, ","), + shimv1.ContainerNamesAnnotationKey: strings.Join(names, ","), }) } func portsAnnotation(portsMap string) podOption { return annotations(map[string]string{ - shim.PortsAnnotationKey: portsMap, + shimv1.PortsAnnotationKey: portsMap, }) } func migrateAnnotation(container string) podOption { return annotations(map[string]string{ - shim.MigrateAnnotationKey: container, + shimv1.MigrateAnnotationKey: container, }) } func liveMigrateAnnotation(container string) podOption { return annotations(map[string]string{ - shim.LiveMigrateAnnotationKey: container, + shimv1.LiveMigrateAnnotationKey: container, }) } @@ -442,7 +441,7 @@ func readinessProbe(probe *corev1.Probe, index int) podOption { func disableDataMigration() podOption { return annotations(map[string]string{ - shim.DisableMigrateDataAnnotationKey: "true", + shimv1.DisableMigrateDataAnnotationKey: "true", }) } diff --git a/go.mod b/go.mod index 8bf42d7..fa500c1 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.0 require ( github.com/checkpoint-restore/go-criu/v7 v7.2.0 - github.com/cilium/ebpf v0.19.0 + github.com/cilium/ebpf v0.21.0 github.com/containerd/cgroups/v3 v3.0.5 github.com/containerd/containerd/api v1.9.0 github.com/containerd/containerd/v2 v2.1.6 diff --git a/go.sum b/go.sum index 0cee487..325cba7 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ github.com/checkpoint-restore/go-criu/v7 v7.2.0/go.mod h1:u0LCWLg0w4yqqu14aXhiB4 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.19.0 h1:Ro/rE64RmFBeA9FGjcTc+KmCeY6jXmryu6FfnzPRIao= -github.com/cilium/ebpf v0.19.0/go.mod h1:fLCgMo3l8tZmAdM3B2XqdFzXBpwkcSTroaVqN08OWVY= +github.com/cilium/ebpf v0.21.0 h1:4dpx1J/B/1apeTmWBH5BkVLayHTkFrMovVPnHEk+l3k= +github.com/cilium/ebpf v0.21.0/go.mod h1:1kHKv6Kvh5a6TePP5vvvoMa1bclRyzUXELSs272fmIQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= diff --git a/manager/redirector_attacher.go b/manager/redirector_attacher.go index 73472c7..e298399 100644 --- a/manager/redirector_attacher.go +++ b/manager/redirector_attacher.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "log/slog" - "net" - "net/netip" "os" "path/filepath" "strconv" @@ -25,15 +23,9 @@ type Redirector struct { } type sandbox struct { - ips []netip.Addr activator *activator.BPF } -const ( - ifaceETH0 = "eth0" - ifaceLoopback = "lo" -) - // AttachRedirectors scans the zeropod maps path in the bpf file system for // directories named after the pid of the sandbox container. It does an // initial iteration over all directories and then starts a goroutine which @@ -72,6 +64,10 @@ func AttachRedirectors(ctx context.Context, log *slog.Logger, activatorOpts ...a continue } + if activator.TCXPinned(pid) { + r.log.Debug("skipping already pinned attach", "pid", pid) + continue + } errs = append(errs, r.attachRedirector(pid)) } @@ -105,6 +101,11 @@ func (r *Redirector) watchForSandboxPids(ctx context.Context) error { continue } + if activator.TCXPinned(pid, activator.DefaultIfaces...) { + r.log.Debug("skipping already pinned attach", "pid", pid) + continue + } + if err := statNetNS(pid); err != nil { r.log.Warn("ignoring pid as net ns was not found", "pid", pid) continue @@ -144,24 +145,19 @@ func (r *Redirector) attachRedirector(pid int) error { return err } - var sandboxIPs []netip.Addr if err := netNS.Do(func(nn ns.NetNS) error { - // TODO: is this really always eth0? - // as for loopback, this is required for port-forwarding to work - ifaces := []string{ifaceETH0, ifaceLoopback} - r.log.Info("attaching redirector for sandbox", "pid", pid, "links", ifaces) - if err := bpf.AttachRedirector(ifaces...); err != nil { + r.log.Info("attaching redirector for sandbox", "pid", pid, "links", activator.DefaultIfaces) + if err := bpf.AttachRedirector(activator.DefaultIfaces...); err != nil { return err } - sandboxIPs, err = getSandboxIPs(ifaceETH0) return err }); err != nil { return errors.Join(err, bpf.Cleanup()) } r.Lock() - r.sandboxes[pid] = sandbox{activator: bpf, ips: sandboxIPs} + r.sandboxes[pid] = sandbox{activator: bpf} r.Unlock() return nil @@ -213,36 +209,6 @@ func (r *Redirector) getSandboxPids() ([]int, error) { return intPids, nil } -func getSandboxIPs(ifaceName string) ([]netip.Addr, error) { - ips := []netip.Addr{} - iface, err := net.InterfaceByName(ifaceName) - if err != nil { - return ips, fmt.Errorf("could not get interface: %w", err) - } - addrs, err := iface.Addrs() - if err != nil { - return ips, fmt.Errorf("could not get interface addrs: %w", err) - } - for _, addr := range addrs { - if ipnet, ok := addr.(*net.IPNet); ok { - // no need to track link local addresses - if ipnet.IP.IsLinkLocalUnicast() { - continue - } - ip, ok := netip.AddrFromSlice(ipnet.IP) - if !ok { - return ips, fmt.Errorf("unable to convert net.IP to netip.Addr: %s", ipnet.IP) - } - // use Unmap as the ipv4 might be mapped in v6 - ips = append(ips, ip.Unmap()) - } - } - if len(ips) == 0 { - return ips, fmt.Errorf("sandbox IPs not found") - } - return ips, nil -} - func ignoredDir(dir string) bool { return dir == activator.SocketTrackerMap || dir == activator.PodKubeletAddrsMapv4 || diff --git a/shim/container.go b/shim/container.go index b21c9d2..7ca99c9 100644 --- a/shim/container.go +++ b/shim/container.go @@ -38,7 +38,7 @@ type Container struct { id string createOpts *anypb.Any activator *activator.Server - cfg *Config + cfg *v1.Config initialProcess process.Process process process.Process cgroup any @@ -60,9 +60,9 @@ type Container struct { runcVersion string } -func New(ctx context.Context, cfg *Config, r *taskAPI.CreateTaskRequest, pt stdio.Platform, events chan *v1.ContainerStatus) (*Container, error) { +func New(ctx context.Context, cfg *v1.Config, r *taskAPI.CreateTaskRequest, pt stdio.Platform, events chan *v1.ContainerStatus) (*Container, error) { // get network ns of our container and store it for later use - netNSPath, err := GetNetworkNS(cfg.spec) + netNSPath, err := GetNetworkNS(cfg.Spec) if err != nil { return nil, err } @@ -126,7 +126,7 @@ func (c *Container) Register(ctx context.Context, container *runc.Container) err return nil } -func (c *Container) Config() *Config { +func (c *Container) Config() *v1.Config { return c.cfg } @@ -440,12 +440,17 @@ func (c *Container) startActivator(ctx context.Context, ports ...uint16) error { if c.activator.Started() { return nil } + if err := c.activator.AttachExec(); err != nil { + log.G(ctx).WithError(err).Error("failed to attach activator") + return err + } + log.G(ctx).Infof("attached %s", time.Now().Format(time.RFC3339Nano)) if err := c.activator.Start(c.context, c.detectProbe(c.context), c.restoreHandler(c.context), ports...); err != nil { if errors.Is(err, activator.ErrMapNotFound) { return err } - log.G(ctx).Errorf("failed to start activator: %s", err) + log.G(ctx).WithError(err).Error("failed to start activator") return err } log.G(ctx).Printf("activator started") diff --git a/shim/log.go b/shim/log.go index d6141e2..d2c01e1 100644 --- a/shim/log.go +++ b/shim/log.go @@ -5,6 +5,8 @@ import ( "os" "path/filepath" "sort" + + v1 "github.com/ctrox/zeropod/api/shim/v1" ) // getLogPath gets the log path of the container by searching for the last log @@ -13,7 +15,7 @@ import ( // containerd only passes that to the sandbox container (pause). One possible // solution would be to implement log restoring in the sandbox container // instead of the zeropod. -func getLogPath(cfg *Config) (string, error) { +func getLogPath(cfg *v1.Config) (string, error) { logDir := fmt.Sprintf("/var/log/pods/%s_%s_%s/%s", cfg.PodNamespace, cfg.PodName, cfg.PodUID, cfg.ContainerName) dir, err := os.Open(logDir) diff --git a/shim/metrics.go b/shim/metrics.go index 11ad1e6..7ea99d6 100644 --- a/shim/metrics.go +++ b/shim/metrics.go @@ -4,7 +4,7 @@ import ( v1 "github.com/ctrox/zeropod/api/shim/v1" ) -func newMetrics(cfg *Config, running bool) *v1.ContainerMetrics { +func newMetrics(cfg *v1.Config, running bool) *v1.ContainerMetrics { return &v1.ContainerMetrics{ Name: cfg.ContainerName, PodName: cfg.PodName, diff --git a/shim/probe.go b/shim/probe.go index 42a2f25..9653e67 100644 --- a/shim/probe.go +++ b/shim/probe.go @@ -13,11 +13,6 @@ import ( "github.com/ctrox/zeropod/activator" ) -// defaultProbeBufferSize should be able to fit kube-probe HTTP requests with -// reasonable path and header sizes but should still be small enough to not -// impact performance. -const defaultProbeBufferSize = 1024 - func (c *Container) detectProbe(ctx context.Context) activator.ConnHook { if c.cfg.DisableProbeDetection { return func(conn net.Conn) (net.Conn, bool, error) { diff --git a/shim/probe_test.go b/shim/probe_test.go index 4cc4808..5b2568a 100644 --- a/shim/probe_test.go +++ b/shim/probe_test.go @@ -8,6 +8,7 @@ import ( "net/http" "testing" + v1 "github.com/ctrox/zeropod/api/shim/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -37,7 +38,7 @@ func TestDetectProbe(t *testing.T) { }, "probe request header bigger than buffer": { clientFunc: httpRequest("kube-probe/1.32", http.StatusOK, func(req *http.Request) { - rnd, err := randomData(defaultProbeBufferSize * 10) + rnd, err := randomData(v1.DefaultProbeBufferSize * 10) assert.NoError(t, err) req.Header.Set("random-stuff", base64.URLEncoding.EncodeToString(rnd)) }), @@ -45,7 +46,7 @@ func TestDetectProbe(t *testing.T) { }, "probe request path bigger than buffer": { clientFunc: httpRequest("kube-probe/1.32", http.StatusOK, func(req *http.Request) { - rnd, err := randomData(defaultProbeBufferSize * 10) + rnd, err := randomData(v1.DefaultProbeBufferSize * 10) assert.NoError(t, err) req.URL.Path = "/" + base64.URLEncoding.EncodeToString(rnd) }), @@ -56,7 +57,7 @@ func TestDetectProbe(t *testing.T) { probeDetected: false, }, "random TCP data bigger than buffer": { - clientFunc: writeRandomTCPData(defaultProbeBufferSize * 1024), + clientFunc: writeRandomTCPData(v1.DefaultProbeBufferSize * 1024), probeDetected: false, }, } { @@ -72,7 +73,7 @@ func TestDetectProbe(t *testing.T) { conn, err := l.Accept() require.NoError(t, err) - c := &Container{cfg: &Config{ProbeBufferSize: defaultProbeBufferSize}} + c := &Container{cfg: &v1.Config{AnnotationConfig: v1.AnnotationConfig{ProbeBufferSize: v1.DefaultProbeBufferSize}}} newConn, cont, err := c.detectProbe(ctx)(conn) require.NoError(t, err) if cont { diff --git a/shim/restore.go b/shim/restore.go index 8ae0475..c50d60f 100644 --- a/shim/restore.go +++ b/shim/restore.go @@ -208,7 +208,7 @@ func createContainerLoggers(ctx context.Context, logPath string, tty bool) (stdo // MigrationRestore requests a restore from the node. If a matching migration is // found, it sets the Checkpoint path in the CreateTaskRequest. -func MigrationRestore(ctx context.Context, r *task.CreateTaskRequest, cfg *Config) (skipStart bool, err error) { +func MigrationRestore(ctx context.Context, r *task.CreateTaskRequest, cfg *v1.Config) (skipStart bool, err error) { conn, err := net.Dial("unix", nodev1.SocketPath) if err != nil { return false, fmt.Errorf("%w: dialing node service: %w", ErrRestoreDial, err) @@ -300,7 +300,7 @@ func setCriuWorkPath(r *task.CreateTaskRequest, path string) error { return nil } -func FinishRestore(ctx context.Context, id string, cfg *Config, startTime time.Time) error { +func FinishRestore(ctx context.Context, id string, cfg *v1.Config, startTime time.Time) error { conn, err := net.Dial("unix", nodev1.SocketPath) if err != nil { return fmt.Errorf("dialing node service: %w", err) diff --git a/shim/task/service_zeropod.go b/shim/task/service_zeropod.go index bbe5406..b866d3b 100644 --- a/shim/task/service_zeropod.go +++ b/shim/task/service_zeropod.go @@ -123,7 +123,7 @@ func (w *wrapper) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ * return nil, err } - cfg, err := zshim.NewConfig(ctx, spec) + cfg, err := v1.NewConfig(ctx, spec) if err != nil { return nil, err } From 0542eb2363a1928913ada140fe280e5bc17d32d5 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Sat, 2 May 2026 11:22:18 +0800 Subject: [PATCH 2/2] refactor: move containerd annotations to shim/v1 --- api/shim/v1/config.go | 16 +++++++++++ cmd/installer/main.go | 56 ++++++++++++++++---------------------- cmd/installer/main_test.go | 8 ++++-- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/api/shim/v1/config.go b/api/shim/v1/config.go index f120604..8dd2822 100644 --- a/api/shim/v1/config.go +++ b/api/shim/v1/config.go @@ -55,6 +55,22 @@ const ( DefaultTrackerIgnoreLocalhost = true ) +var ContainerdAnnotations = []string{ + PortsAnnotationKey, + ContainerNamesAnnotationKey, + ScaleDownDurationAnnotationKey, + DisableCheckpoiningAnnotationKey, + PreDumpAnnotationKey, + MigrateAnnotationKey, + LiveMigrateAnnotationKey, + DisableProbeDetectAnnotationKey, + ProbeBufferSizeAnnotationKey, + ProxyTimeoutAnnotationKey, + ConnectTimeoutAnnotationKey, + DisableMigrateDataAnnotationKey, + "io.containerd.runc.v2.group", +} + type AnnotationConfig struct { ZeropodContainerNames []string Ports []uint16 diff --git a/cmd/installer/main.go b/cmd/installer/main.go index 144017e..c092eee 100644 --- a/cmd/installer/main.go +++ b/cmd/installer/main.go @@ -81,21 +81,7 @@ network-lock skip [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.zeropod] runtime_type = "io.containerd.runc.v2" runtime_path = "%s/bin/containerd-shim-zeropod-v2" - pod_annotations = [ - "zeropod.ctrox.dev/ports-map", - "zeropod.ctrox.dev/container-names", - "zeropod.ctrox.dev/scaledown-duration", - "zeropod.ctrox.dev/disable-checkpointing", - "zeropod.ctrox.dev/pre-dump", - "zeropod.ctrox.dev/migrate", - "zeropod.ctrox.dev/live-migrate", - "zeropod.ctrox.dev/disable-probe-detection", - "zeropod.ctrox.dev/probe-buffer-size", - "zeropod.ctrox.dev/disable-migrate-data", - "zeropod.ctrox.dev/connect-timeout", - "zeropod.ctrox.dev/proxy-timeout", - "io.containerd.runc.v2.group" - ] + pod_annotations = %s [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.zeropod.options] # use systemd cgroup by default @@ -106,21 +92,7 @@ network-lock skip [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod] runtime_type = "io.containerd.runc.v2" runtime_path = "%s/bin/containerd-shim-zeropod-v2" - pod_annotations = [ - "zeropod.ctrox.dev/ports-map", - "zeropod.ctrox.dev/container-names", - "zeropod.ctrox.dev/scaledown-duration", - "zeropod.ctrox.dev/disable-checkpointing", - "zeropod.ctrox.dev/pre-dump", - "zeropod.ctrox.dev/migrate", - "zeropod.ctrox.dev/live-migrate", - "zeropod.ctrox.dev/disable-probe-detection", - "zeropod.ctrox.dev/probe-buffer-size", - "zeropod.ctrox.dev/disable-migrate-data", - "zeropod.ctrox.dev/connect-timeout", - "zeropod.ctrox.dev/proxy-timeout", - "io.containerd.runc.v2.group" - ] + pod_annotations = %s [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.zeropod.options] # use systemd cgroup by default @@ -444,7 +416,11 @@ func configureContainerdv1(ctx context.Context, runtime containerRuntime, contai optPath = containerdOptPath } - if _, err := fmt.Fprintf(cfg, runtimeConfig, strings.TrimSuffix(optPath, "/")); err != nil { + if _, err := fmt.Fprintf( + cfg, runtimeConfig, + strings.TrimSuffix(optPath, "/"), + annotationsToml(), + ); err != nil { return false, err } @@ -542,7 +518,12 @@ func writeZeropodRuntimeConfig(containerdConfig, optPath string, existingOpt boo if version == 3 { zeropodRuntimeConfig = runtimeConfigV3 } - zeropodRuntimeConfig = fmt.Sprintf(zeropodRuntimeConfig, strings.TrimSuffix(optPath, "/")) + + zeropodRuntimeConfig = fmt.Sprintf( + zeropodRuntimeConfig, + strings.TrimSuffix(optPath, "/"), + annotationsToml(), + ) if !existingOpt { zeropodRuntimeConfig = zeropodRuntimeConfig + fmt.Sprintf(optPlugin, optPath) } @@ -552,6 +533,17 @@ func writeZeropodRuntimeConfig(containerdConfig, optPath string, existingOpt boo return nil } +func annotationsToml() string { + var buf bytes.Buffer + enc := toml.NewEncoder(&buf) + enc.SetArraysMultiline(true) + enc.SetIndentSymbol(" ") + if err := enc.Encode(v1.ContainerdAnnotations); err != nil { + return "[]" + } + return buf.String() +} + func restoreContainerdConfig(runtime containerRuntime, containerdConfigPath string) error { if _, err := os.Stat(containerdConfigPath + configBackupSuffix); err != nil { if errors.Is(err, os.ErrNotExist) { diff --git a/cmd/installer/main_test.go b/cmd/installer/main_test.go index 1442150..a816e37 100644 --- a/cmd/installer/main_test.go +++ b/cmd/installer/main_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "os" "testing" @@ -90,11 +91,12 @@ imports = [ "runtime_zeropod.toml", ] ` - containerdv1AlreadyConfigured = fullContainerdConfigV2 + runtimeConfig + ` +) + +var containerdv1AlreadyConfigured = fullContainerdConfigV2 + fmt.Sprintf(runtimeConfig, "", "[]") + ` [plugins."io.containerd.internal.v1.opt"] - path = "/opt/zeropod" + path = "/opt/zeropod" ` -) type testConfig struct { containerdConfig string