diff --git a/api/v1alpha1/obproxy_types.go b/api/v1alpha1/obproxy_types.go new file mode 100644 index 000000000..6d5148c8f --- /dev/null +++ b/api/v1alpha1/obproxy_types.go @@ -0,0 +1,153 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + apitypes "github.com/oceanbase/ob-operator/api/types" + tasktypes "github.com/oceanbase/ob-operator/pkg/task/types" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// OBClusterReference identifies an OBCluster resource that this OBProxy connects to. +type OBClusterReference struct { + // Namespace of the referenced OBCluster. Defaults to the namespace of the OBProxy resource. + // +optional + Namespace string `json:"namespace,omitempty"` + // Name of the OBCluster resource. + Name string `json:"name"` +} + +// OBProxySpec defines the desired state of OBProxy +type OBProxySpec struct { + // Reference to the target OBCluster. + OBCluster OBClusterReference `json:"obCluster"` + + // ProxyClusterName is the OBProxy cluster (application) name used inside OceanBase proxy. + ProxyClusterName string `json:"proxyClusterName"` + + // ProxySysSecret is the name of a Secret in the same namespace as this OBProxy that holds + // the root@proxysys password. The Secret must contain a key named "password". + ProxySysSecret string `json:"proxySysSecret"` + + // Image is the OBProxy container image (e.g. oceanbase/obproxy-ce:tag). + Image string `json:"image"` + + // ServiceType is the Kubernetes Service type exposed for SQL access. + // +kubebuilder:default=ClusterIP + // +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer;ExternalName + ServiceType string `json:"serviceType,omitempty"` + + // Replicas is the number of OBProxy pods. + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=1 + Replicas int32 `json:"replicas"` + + // Resource limits and requests for each OBProxy pod. + Resource *apitypes.ResourceSpec `json:"resource"` + + // Parameters are optional OBProxy configuration key-value pairs (mapped to proxy startup config). + // +optional + Parameters []apitypes.Parameter `json:"parameters,omitempty"` + + // ServiceAccount is the name of the ServiceAccount to use for OBProxy pods. + // +kubebuilder:default=default + // +optional + ServiceAccount string `json:"serviceAccount,omitempty"` + + // NodeSelector constrains OBProxy pods to nodes matching these labels. + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Affinity defines scheduling affinity rules for OBProxy pods. + // +optional + Affinity *corev1.Affinity `json:"affinity,omitempty"` + + // Tolerations allow OBProxy pods to be scheduled on tainted nodes. + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` +} + +// OBProxyStatus defines the observed state of OBProxy +type OBProxyStatus struct { + // High-level phase, e.g. Running, Pending, Failed. + Status string `json:"status,omitempty"` + + // Image is the container image currently running in the Deployment (observed). + Image string `json:"image,omitempty"` + + // Replicas is the replica count currently set in the Deployment (observed). + Replicas int32 `json:"replicas,omitempty"` + + // ReadyReplicas is the number of ready OBProxy pods (observed). + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + + // RSList is the RS_LIST env value currently set in the Deployment (observed). + // +optional + RSList string `json:"rsList,omitempty"` + + // RSListSource indicates how RSList was last resolved: "k8s" or "sql". + // +optional + RSListSource string `json:"rsListSource,omitempty"` + + // ServiceIP is the ClusterIP of the managed Service (observed). + // +optional + ServiceIP string `json:"serviceIP,omitempty"` + + // Conditions holds granular state for diagnostic purposes. + // Known types: OBClusterAvailable, OBClusterReady, RSListAvailable. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // OperationContext carries long-running task progress when using the operator task flow. + // +optional + OperationContext *tasktypes.OperationContext `json:"operationContext,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:shortName=obp +//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status" +//+kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".status.replicas" +//+kubebuilder:printcolumn:name="Ready",type="integer",JSONPath=".status.readyReplicas" +//+kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".spec.obCluster.name" +//+kubebuilder:printcolumn:name="RSListSrc",type="string",JSONPath=".status.rsListSource" +//+kubebuilder:printcolumn:name="ServiceIP",type="string",JSONPath=".status.serviceIP",priority=1 +//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// OBProxy is the Schema for the obproxies API +type OBProxy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec OBProxySpec `json:"spec,omitempty"` + Status OBProxyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// OBProxyList contains a list of OBProxy +type OBProxyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []OBProxy `json:"items"` +} + +func init() { + SchemeBuilder.Register(&OBProxy{}, &OBProxyList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1302a44d9..ced1a0d9e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,6 +23,7 @@ package v1alpha1 import ( "github.com/oceanbase/ob-operator/api/types" "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -428,6 +429,21 @@ func (in *OBClusterOperationStatus) DeepCopy() *OBClusterOperationStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OBClusterReference) DeepCopyInto(out *OBClusterReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBClusterReference. +func (in *OBClusterReference) DeepCopy() *OBClusterReference { + if in == nil { + return nil + } + out := new(OBClusterReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OBClusterSnapshot) DeepCopyInto(out *OBClusterSnapshot) { *out = *in @@ -635,6 +651,137 @@ func (in *OBParameterStatus) DeepCopy() *OBParameterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OBProxy) DeepCopyInto(out *OBProxy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBProxy. +func (in *OBProxy) DeepCopy() *OBProxy { + if in == nil { + return nil + } + out := new(OBProxy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OBProxy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OBProxyList) DeepCopyInto(out *OBProxyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]OBProxy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBProxyList. +func (in *OBProxyList) DeepCopy() *OBProxyList { + if in == nil { + return nil + } + out := new(OBProxyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OBProxyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OBProxySpec) DeepCopyInto(out *OBProxySpec) { + *out = *in + out.OBCluster = in.OBCluster + if in.Resource != nil { + in, out := &in.Resource, &out.Resource + *out = (*in).DeepCopy() + } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make([]types.Parameter, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBProxySpec. +func (in *OBProxySpec) DeepCopy() *OBProxySpec { + if in == nil { + return nil + } + out := new(OBProxySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OBProxyStatus) DeepCopyInto(out *OBProxyStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.OperationContext != nil { + in, out := &in.OperationContext, &out.OperationContext + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBProxyStatus. +func (in *OBProxyStatus) DeepCopy() *OBProxyStatus { + if in == nil { + return nil + } + out := new(OBProxyStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OBResourceRescue) DeepCopyInto(out *OBResourceRescue) { *out = *in diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 969da75aa..0e5cd9d08 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -239,6 +239,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "OBClusterOperation") os.Exit(1) } + if err = (&controller.OBProxyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: telemetry.NewRecorder(ctx, mgr.GetEventRecorderFor(config.OBProxyControllerName)), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "Unable to create controller", "controller", "OBProxy") + os.Exit(1) + } if err = (&controller.K8sClusterReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/config/crd/bases/oceanbase.oceanbase.com_obproxies.yaml b/config/crd/bases/oceanbase.oceanbase.com_obproxies.yaml new file mode 100644 index 000000000..73a947139 --- /dev/null +++ b/config/crd/bases/oceanbase.oceanbase.com_obproxies.yaml @@ -0,0 +1,214 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: obproxies.oceanbase.oceanbase.com +spec: + group: oceanbase.oceanbase.com + names: + kind: OBProxy + listKind: OBProxyList + plural: obproxies + shortNames: + - obp + singular: obproxy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .spec.obCluster.name + name: Cluster + type: string + - jsonPath: .spec.proxyClusterName + name: ProxyName + type: string + - jsonPath: .spec.withConfigServer + name: CfgSrv + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: OBProxy is the Schema for the obproxies API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: OBProxySpec defines the desired state of OBProxy + properties: + image: + description: Image is the OBProxy container image (e.g. oceanbase/obproxy-ce:tag). + type: string + obCluster: + description: Reference to the target OBCluster. + properties: + name: + description: Name of the OBCluster resource. + type: string + namespace: + description: Namespace of the referenced OBCluster. Defaults to + the namespace of the OBProxy resource. + type: string + required: + - name + type: object + parameters: + description: Parameters are optional OBProxy configuration key-value + pairs (mapped to proxy startup config). + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + proxyClusterName: + description: ProxyClusterName is the OBProxy cluster (application) + name used inside OceanBase proxy. + type: string + proxySysSecret: + description: |- + ProxySysSecret is the name of a Secret in the same namespace as this OBProxy that holds + the root@proxysys password. The Secret must contain a key named "password". + type: string + replicas: + default: 1 + description: Replicas is the number of OBProxy pods. + format: int32 + minimum: 1 + type: integer + resource: + description: Resource limits and requests for each OBProxy pod. + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - memory + type: object + serviceType: + default: ClusterIP + description: ServiceType is the Kubernetes Service type exposed for + SQL access. + enum: + - ClusterIP + - NodePort + - LoadBalancer + - ExternalName + type: string + withConfigServer: + default: false + description: |- + WithConfigServer declares whether OBProxy is started with a config server. + false (default) means RS_LIST bootstrap: the operator may derive RS_LIST from cluster topology and roll pods when it changes. + true means config-server mode: the operator must not treat RS_LIST as operator-maintained for rolling updates. + type: boolean + required: + - image + - obCluster + - proxyClusterName + - proxySysSecret + - replicas + - resource + type: object + status: + description: OBProxyStatus defines the observed state of OBProxy + properties: + image: + description: Image currently used by the OBProxy workload (observed). + type: string + operationContext: + description: OperationContext carries long-running task progress when + using the operator task flow. + properties: + failureRule: + properties: + failureStatus: + type: string + failureStrategy: + type: string + maxRetry: + type: integer + retryCount: + type: integer + required: + - failureStatus + - failureStrategy + type: object + idx: + type: integer + name: + type: string + targetStatus: + type: string + task: + type: string + taskId: + type: string + taskStatus: + type: string + tasks: + items: + type: string + type: array + required: + - idx + - name + - targetStatus + - task + - taskId + - taskStatus + - tasks + type: object + readyReplicas: + description: ReadyReplicas is the number of ready OBProxy pods. + format: int32 + type: integer + replicas: + description: Replicas is the desired replica count observed from the + workload. + format: int32 + type: integer + status: + description: High-level phase, e.g. Running, Pending, Failed. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index e6ac223a1..8647c8a6e 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -13,6 +13,7 @@ resources: - bases/oceanbase.oceanbase.com_obtenantoperations.yaml - bases/oceanbase.oceanbase.com_obresourcerescues.yaml - bases/oceanbase.oceanbase.com_obclusteroperations.yaml +- bases/oceanbase.oceanbase.com_obproxies.yaml - bases/k8s.oceanbase.com_k8sclusters.yaml - bases/oceanbase.oceanbase.com_obtenantvariables.yaml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/rbac/obproxy_editor_role.yaml b/config/rbac/obproxy_editor_role.yaml new file mode 100644 index 000000000..1d9278e1f --- /dev/null +++ b/config/rbac/obproxy_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit obproxies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: obproxy-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: ob-operator-generate + app.kubernetes.io/part-of: ob-operator-generate + app.kubernetes.io/managed-by: kustomize + name: obproxy-editor-role +rules: +- apiGroups: + - oceanbase.oceanbase.com + resources: + - obproxies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - oceanbase.oceanbase.com + resources: + - obproxies/status + verbs: + - get diff --git a/config/rbac/obproxy_viewer_role.yaml b/config/rbac/obproxy_viewer_role.yaml new file mode 100644 index 000000000..8fff83d5b --- /dev/null +++ b/config/rbac/obproxy_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view obproxies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: obproxy-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: ob-operator-generate + app.kubernetes.io/part-of: ob-operator-generate + app.kubernetes.io/managed-by: kustomize + name: obproxy-viewer-role +rules: +- apiGroups: + - oceanbase.oceanbase.com + resources: + - obproxies + verbs: + - get + - list + - watch +- apiGroups: + - oceanbase.oceanbase.com + resources: + - obproxies/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 23b038938..3c5479059 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -7,34 +7,35 @@ rules: - apiGroups: - "" resources: - - events + - configmaps + - persistentvolumeclaims + - persistentvolumes + - pods + - secrets + - services verbs: - create + - delete + - get + - list - patch + - update + - watch - apiGroups: - "" resources: - - nodes - - serviceaccounts + - events verbs: - - get - - list - - watch + - create + - patch - apiGroups: - "" resources: - - persistentvolumeclaims - - persistentvolumes - - pods - - secrets - - services + - nodes + - serviceaccounts verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - "" @@ -62,6 +63,26 @@ rules: - pods/log verbs: - get +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments/status + verbs: + - get + - patch + - update - apiGroups: - batch resources: @@ -120,6 +141,7 @@ rules: - obclusteroperations - obclusters - obparameters + - obproxies - obresourcerescues - observers - obtenantbackuppolicies @@ -144,6 +166,7 @@ rules: - obclusteroperations/finalizers - obclusters/finalizers - obparameters/finalizers + - obproxies/finalizers - obresourcerescues/finalizers - observers/finalizers - obtenantbackuppolicies/finalizers @@ -161,6 +184,7 @@ rules: - obclusteroperations/status - obclusters/status - obparameters/status + - obproxies/status - obresourcerescues/status - observers/status - obtenantbackuppolicies/status diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 42a71c6b3..a96b64f06 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -16,7 +16,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: k8sclusters.k8s.oceanbase.com spec: conversion: @@ -102,7 +102,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obclusteroperations.oceanbase.oceanbase.com spec: conversion: @@ -1144,12 +1144,10 @@ spec: Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: - 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows. format: int64 @@ -1236,7 +1234,6 @@ spec: type indicates which kind of seccomp profile will be applied. Valid options are: - Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied. @@ -1476,7 +1473,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -1593,7 +1589,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -1632,7 +1627,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -1701,7 +1695,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the ConfigMap @@ -1737,7 +1730,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -1875,7 +1867,6 @@ spec: The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed. - Use this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity @@ -1886,17 +1877,14 @@ spec: information on the connection between this volume type and PersistentVolumeClaim). - Use PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod. - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information. - A pod can use both types of ephemeral volumes and persistent volumes at the same time. properties: @@ -1910,7 +1898,6 @@ spec: entry. Pod validation will reject the pod if the concatenated name is not valid for a PVC (for example, too long). - An existing PVC with that name that is not owned by the pod will *not* be used for the pod to avoid using an unrelated volume by mistake. Starting the pod is then blocked until @@ -1920,11 +1907,9 @@ spec: this should not be necessary, but it may be useful when manually reconstructing a broken cluster. - This field is read-only and no changes will be made by Kubernetes to the PVC after it has been created. - Required, must not be nil. properties: metadata: @@ -2150,7 +2135,6 @@ spec: fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem from compromising the machine type: string lun: description: 'lun is Optional: FC target lun number' @@ -2213,7 +2197,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -2247,7 +2230,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -2328,9 +2310,6 @@ spec: used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- - TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not - mount host directories as read/write. properties: path: description: |- @@ -2367,7 +2346,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising the machine type: string initiatorName: description: |- @@ -2407,7 +2385,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -2536,14 +2513,11 @@ spec: ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field of ClusterTrustBundle objects in an auto-updating file. - Alpha, gated by the ClusterTrustBundleProjection feature gate. - ClusterTrustBundle objects can either be selected by name, or by the combination of signer name and a label selector. - Kubelet performs aggressive normalization of the PEM contents written into the pod filesystem. Esoteric PEM features such as inter-block comments and block headers are stripped. Certificates are deduplicated. @@ -2672,7 +2646,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the @@ -2808,7 +2781,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional field specify whether @@ -2897,7 +2869,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising the machine type: string image: description: |- @@ -2940,7 +2911,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -2987,7 +2957,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3106,7 +3075,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3303,7 +3271,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -3421,7 +3388,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3460,7 +3426,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3529,7 +3494,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the ConfigMap @@ -3565,7 +3529,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3707,7 +3670,6 @@ spec: The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed. - Use this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity @@ -3718,17 +3680,14 @@ spec: information on the connection between this volume type and PersistentVolumeClaim). - Use PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod. - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information. - A pod can use both types of ephemeral volumes and persistent volumes at the same time. properties: @@ -3742,7 +3701,6 @@ spec: entry. Pod validation will reject the pod if the concatenated name is not valid for a PVC (for example, too long). - An existing PVC with that name that is not owned by the pod will *not* be used for the pod to avoid using an unrelated volume by mistake. Starting the pod is then blocked until @@ -3752,11 +3710,9 @@ spec: this should not be necessary, but it may be useful when manually reconstructing a broken cluster. - This field is read-only and no changes will be made by Kubernetes to the PVC after it has been created. - Required, must not be nil. properties: metadata: @@ -3984,7 +3940,6 @@ spec: fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem from compromising the machine type: string lun: description: 'lun is Optional: FC target lun number' @@ -4047,7 +4002,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -4081,7 +4035,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -4162,9 +4115,6 @@ spec: used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- - TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not - mount host directories as read/write. properties: path: description: |- @@ -4201,7 +4151,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising the machine type: string initiatorName: description: |- @@ -4242,7 +4191,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -4371,14 +4319,11 @@ spec: ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field of ClusterTrustBundle objects in an auto-updating file. - Alpha, gated by the ClusterTrustBundleProjection feature gate. - ClusterTrustBundle objects can either be selected by name, or by the combination of signer name and a label selector. - Kubelet performs aggressive normalization of the PEM contents written into the pod filesystem. Esoteric PEM features such as inter-block comments and block headers are stripped. Certificates are deduplicated. @@ -4508,7 +4453,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether @@ -4648,7 +4592,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional field specify @@ -4739,7 +4682,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising the machine type: string image: description: |- @@ -4782,7 +4724,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -4830,7 +4771,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -4949,7 +4889,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -5075,12 +5014,10 @@ spec: Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: - 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows. format: int64 @@ -5167,7 +5104,6 @@ spec: type indicates which kind of seccomp profile will be applied. Valid options are: - Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied. @@ -6320,12 +6256,10 @@ spec: Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: - 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows. format: int64 @@ -6412,7 +6346,6 @@ spec: type indicates which kind of seccomp profile will be applied. Valid options are: - Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied. @@ -6771,7 +6704,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obclusters.oceanbase.oceanbase.com spec: conversion: @@ -6865,7 +6798,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -6981,7 +6913,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -7020,7 +6951,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -7088,7 +7018,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the ConfigMap or @@ -7124,7 +7053,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -7260,7 +7188,6 @@ spec: The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed. - Use this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity @@ -7271,17 +7198,14 @@ spec: information on the connection between this volume type and PersistentVolumeClaim). - Use PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod. - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information. - A pod can use both types of ephemeral volumes and persistent volumes at the same time. properties: @@ -7295,7 +7219,6 @@ spec: entry. Pod validation will reject the pod if the concatenated name is not valid for a PVC (for example, too long). - An existing PVC with that name that is not owned by the pod will *not* be used for the pod to avoid using an unrelated volume by mistake. Starting the pod is then blocked until @@ -7305,11 +7228,9 @@ spec: this should not be necessary, but it may be useful when manually reconstructing a broken cluster. - This field is read-only and no changes will be made by Kubernetes to the PVC after it has been created. - Required, must not be nil. properties: metadata: @@ -7535,7 +7456,6 @@ spec: fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem from compromising the machine type: string lun: description: 'lun is Optional: FC target lun number' @@ -7598,7 +7518,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -7632,7 +7551,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -7713,9 +7631,6 @@ spec: used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- - TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not - mount host directories as read/write. properties: path: description: |- @@ -7752,7 +7667,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising the machine type: string initiatorName: description: |- @@ -7792,7 +7706,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -7919,14 +7832,11 @@ spec: ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field of ClusterTrustBundle objects in an auto-updating file. - Alpha, gated by the ClusterTrustBundleProjection feature gate. - ClusterTrustBundle objects can either be selected by name, or by the combination of signer name and a label selector. - Kubelet performs aggressive normalization of the PEM contents written into the pod filesystem. Esoteric PEM features such as inter-block comments and block headers are stripped. Certificates are deduplicated. @@ -8054,7 +7964,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the ConfigMap @@ -8187,7 +8096,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional field specify whether @@ -8276,7 +8184,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising the machine type: string image: description: |- @@ -8319,7 +8226,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -8366,7 +8272,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -8484,7 +8389,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -8608,12 +8512,10 @@ spec: Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: - 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows. format: int64 @@ -8700,7 +8602,6 @@ spec: type indicates which kind of seccomp profile will be applied. Valid options are: - Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied. @@ -9838,12 +9739,10 @@ spec: Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: - 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows. format: int64 @@ -9930,7 +9829,6 @@ spec: type indicates which kind of seccomp profile will be applied. Valid options are: - Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied. @@ -10236,7 +10134,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obparameters.oceanbase.oceanbase.com spec: group: oceanbase.oceanbase.com @@ -10397,7 +10295,222 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 + name: obproxies.oceanbase.oceanbase.com +spec: + group: oceanbase.oceanbase.com + names: + kind: OBProxy + listKind: OBProxyList + plural: obproxies + shortNames: + - obp + singular: obproxy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .spec.obCluster.name + name: Cluster + type: string + - jsonPath: .spec.proxyClusterName + name: ProxyName + type: string + - jsonPath: .spec.withConfigServer + name: CfgSrv + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: OBProxy is the Schema for the obproxies API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: OBProxySpec defines the desired state of OBProxy + properties: + image: + description: Image is the OBProxy container image (e.g. oceanbase/obproxy-ce:tag). + type: string + obCluster: + description: Reference to the target OBCluster. + properties: + name: + description: Name of the OBCluster resource. + type: string + namespace: + description: Namespace of the referenced OBCluster. Defaults to + the namespace of the OBProxy resource. + type: string + required: + - name + type: object + parameters: + description: Parameters are optional OBProxy configuration key-value + pairs (mapped to proxy startup config). + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + proxyClusterName: + description: ProxyClusterName is the OBProxy cluster (application) + name used inside OceanBase proxy. + type: string + proxySysSecret: + description: |- + ProxySysSecret is the name of a Secret in the same namespace as this OBProxy that holds + the root@proxysys password. The Secret must contain a key named "password". + type: string + replicas: + default: 1 + description: Replicas is the number of OBProxy pods. + format: int32 + minimum: 1 + type: integer + resource: + description: Resource limits and requests for each OBProxy pod. + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - memory + type: object + serviceType: + default: ClusterIP + description: ServiceType is the Kubernetes Service type exposed for + SQL access. + enum: + - ClusterIP + - NodePort + - LoadBalancer + - ExternalName + type: string + withConfigServer: + default: false + description: |- + WithConfigServer declares whether OBProxy is started with a config server. + false (default) means RS_LIST bootstrap: the operator may derive RS_LIST from cluster topology and roll pods when it changes. + true means config-server mode: the operator must not treat RS_LIST as operator-maintained for rolling updates. + type: boolean + required: + - image + - obCluster + - proxyClusterName + - proxySysSecret + - replicas + - resource + type: object + status: + description: OBProxyStatus defines the observed state of OBProxy + properties: + image: + description: Image currently used by the OBProxy workload (observed). + type: string + operationContext: + description: OperationContext carries long-running task progress when + using the operator task flow. + properties: + failureRule: + properties: + failureStatus: + type: string + failureStrategy: + type: string + maxRetry: + type: integer + retryCount: + type: integer + required: + - failureStatus + - failureStrategy + type: object + idx: + type: integer + name: + type: string + targetStatus: + type: string + task: + type: string + taskId: + type: string + taskStatus: + type: string + tasks: + items: + type: string + type: array + required: + - idx + - name + - targetStatus + - task + - taskId + - taskStatus + - tasks + type: object + readyReplicas: + description: ReadyReplicas is the number of ready OBProxy pods. + format: int32 + type: integer + replicas: + description: Replicas is the desired replica count observed from the + workload. + format: int32 + type: integer + status: + description: High-level phase, e.g. Running, Pending, Failed. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert + controller-gen.kubebuilder.io/version: v0.20.0 name: obresourcerescues.oceanbase.oceanbase.com spec: conversion: @@ -10489,7 +10602,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: observers.oceanbase.oceanbase.com spec: group: oceanbase.oceanbase.com @@ -11461,7 +11574,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -11577,7 +11689,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -11616,7 +11727,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -11684,7 +11794,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the ConfigMap or @@ -11720,7 +11829,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -11856,7 +11964,6 @@ spec: The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed. - Use this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity @@ -11867,17 +11974,14 @@ spec: information on the connection between this volume type and PersistentVolumeClaim). - Use PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod. - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information. - A pod can use both types of ephemeral volumes and persistent volumes at the same time. properties: @@ -11891,7 +11995,6 @@ spec: entry. Pod validation will reject the pod if the concatenated name is not valid for a PVC (for example, too long). - An existing PVC with that name that is not owned by the pod will *not* be used for the pod to avoid using an unrelated volume by mistake. Starting the pod is then blocked until @@ -11901,11 +12004,9 @@ spec: this should not be necessary, but it may be useful when manually reconstructing a broken cluster. - This field is read-only and no changes will be made by Kubernetes to the PVC after it has been created. - Required, must not be nil. properties: metadata: @@ -12131,7 +12232,6 @@ spec: fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem from compromising the machine type: string lun: description: 'lun is Optional: FC target lun number' @@ -12194,7 +12294,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -12228,7 +12327,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -12309,9 +12407,6 @@ spec: used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- - TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not - mount host directories as read/write. properties: path: description: |- @@ -12348,7 +12443,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising the machine type: string initiatorName: description: |- @@ -12388,7 +12482,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -12515,14 +12608,11 @@ spec: ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field of ClusterTrustBundle objects in an auto-updating file. - Alpha, gated by the ClusterTrustBundleProjection feature gate. - ClusterTrustBundle objects can either be selected by name, or by the combination of signer name and a label selector. - Kubelet performs aggressive normalization of the PEM contents written into the pod filesystem. Esoteric PEM features such as inter-block comments and block headers are stripped. Certificates are deduplicated. @@ -12650,7 +12740,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the ConfigMap @@ -12783,7 +12872,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional field specify whether @@ -12872,7 +12960,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising the machine type: string image: description: |- @@ -12915,7 +13002,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -12962,7 +13048,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -13080,7 +13165,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -13210,12 +13294,10 @@ spec: Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: - 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows. format: int64 @@ -13302,7 +13384,6 @@ spec: type indicates which kind of seccomp profile will be applied. Valid options are: - Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied. @@ -13588,7 +13669,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obtenantbackuppolicies.oceanbase.oceanbase.com spec: conversion: @@ -14102,8 +14183,6 @@ spec: default: 1 type: integer resource: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -14140,12 +14219,8 @@ spec: - memorySize type: object type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec - And ResourcePoolStatus type: boolean name: type: string @@ -14320,12 +14395,8 @@ spec: priority: type: integer type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec - And ResourcePoolStatus type: boolean name: type: string @@ -14337,8 +14408,6 @@ spec: - replica type: object unitConfig: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -14708,7 +14777,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obtenantbackups.oceanbase.oceanbase.com spec: group: oceanbase.oceanbase.com @@ -15043,7 +15112,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obtenantoperations.oceanbase.oceanbase.com spec: conversion: @@ -15128,8 +15197,6 @@ spec: default: 1 type: integer resource: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -15166,12 +15233,8 @@ spec: - memorySize type: object type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec And - ResourcePoolStatus type: boolean name: type: string @@ -15226,8 +15289,6 @@ spec: default: 1 type: integer resource: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -15264,12 +15325,8 @@ spec: - memorySize type: object type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec And - ResourcePoolStatus type: boolean name: type: string @@ -15421,8 +15478,6 @@ spec: default: 1 type: integer resource: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -15459,12 +15514,8 @@ spec: - memorySize type: object type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec - And ResourcePoolStatus type: boolean name: type: string @@ -15639,12 +15690,8 @@ spec: priority: type: integer type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec - And ResourcePoolStatus type: boolean name: type: string @@ -15656,8 +15703,6 @@ spec: - replica type: object unitConfig: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -15994,8 +16039,6 @@ spec: default: 1 type: integer resource: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -16032,12 +16075,8 @@ spec: - memorySize type: object type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec - And ResourcePoolStatus type: boolean name: type: string @@ -16212,12 +16251,8 @@ spec: priority: type: integer type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec - And ResourcePoolStatus type: boolean name: type: string @@ -16229,8 +16264,6 @@ spec: - replica type: object unitConfig: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -16524,7 +16557,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obtenantrestores.oceanbase.oceanbase.com spec: group: oceanbase.oceanbase.com @@ -16806,7 +16839,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obtenants.oceanbase.oceanbase.com spec: conversion: @@ -16935,8 +16968,6 @@ spec: default: 1 type: integer resource: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -16973,12 +17004,8 @@ spec: - memorySize type: object type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec And - ResourcePoolStatus type: boolean name: type: string @@ -17153,12 +17180,8 @@ spec: priority: type: integer type: - description: TODO Split LocalityType struct to SpecLocalityType - and StatusLocalityType properties: isActive: - description: TODO move isActive to ResourcePoolSpec And - ResourcePoolStatus type: boolean name: type: string @@ -17170,8 +17193,6 @@ spec: - replica type: object unitConfig: - description: TODO Split UnitConfig struct to SpecUnitConfig - and StatusUnitConfig properties: iopsWeight: type: integer @@ -17456,7 +17477,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obtenantvariables.oceanbase.oceanbase.com spec: group: oceanbase.oceanbase.com @@ -17600,7 +17621,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: oceanbase-system/oceanbase-serving-cert - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.20.0 name: obzones.oceanbase.oceanbase.com spec: group: oceanbase.oceanbase.com @@ -17679,7 +17700,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -17795,7 +17815,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -17834,7 +17853,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -17902,7 +17920,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the ConfigMap or @@ -17938,7 +17955,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -18074,7 +18090,6 @@ spec: The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed. - Use this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity @@ -18085,17 +18100,14 @@ spec: information on the connection between this volume type and PersistentVolumeClaim). - Use PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod. - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information. - A pod can use both types of ephemeral volumes and persistent volumes at the same time. properties: @@ -18109,7 +18121,6 @@ spec: entry. Pod validation will reject the pod if the concatenated name is not valid for a PVC (for example, too long). - An existing PVC with that name that is not owned by the pod will *not* be used for the pod to avoid using an unrelated volume by mistake. Starting the pod is then blocked until @@ -18119,11 +18130,9 @@ spec: this should not be necessary, but it may be useful when manually reconstructing a broken cluster. - This field is read-only and no changes will be made by Kubernetes to the PVC after it has been created. - Required, must not be nil. properties: metadata: @@ -18349,7 +18358,6 @@ spec: fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem from compromising the machine type: string lun: description: 'lun is Optional: FC target lun number' @@ -18412,7 +18420,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -18446,7 +18453,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising the machine type: string partition: description: |- @@ -18527,9 +18533,6 @@ spec: used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- - TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not - mount host directories as read/write. properties: path: description: |- @@ -18566,7 +18569,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising the machine type: string initiatorName: description: |- @@ -18606,7 +18608,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -18733,14 +18734,11 @@ spec: ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field of ClusterTrustBundle objects in an auto-updating file. - Alpha, gated by the ClusterTrustBundleProjection feature gate. - ClusterTrustBundle objects can either be selected by name, or by the combination of signer name and a label selector. - Kubelet performs aggressive normalization of the PEM contents written into the pod filesystem. Esoteric PEM features such as inter-block comments and block headers are stripped. Certificates are deduplicated. @@ -18868,7 +18866,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional specify whether the ConfigMap @@ -19001,7 +18998,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: optional field specify whether @@ -19090,7 +19086,6 @@ spec: Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising the machine type: string image: description: |- @@ -19133,7 +19128,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -19180,7 +19174,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -19298,7 +19291,6 @@ spec: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -19422,12 +19414,10 @@ spec: Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: - 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows. format: int64 @@ -19514,7 +19504,6 @@ spec: type indicates which kind of seccomp profile will be applied. Valid options are: - Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied. @@ -20635,12 +20624,10 @@ spec: Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: - 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows. format: int64 @@ -20727,7 +20714,6 @@ spec: type indicates which kind of seccomp profile will be applied. Valid options are: - Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied. @@ -21064,6 +21050,23 @@ kind: ClusterRole metadata: name: oceanbase-manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + - persistentvolumeclaims + - persistentvolumes + - pods + - secrets + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -21075,6 +21078,7 @@ rules: - "" resources: - nodes + - serviceaccounts verbs: - get - list @@ -21082,27 +21086,33 @@ rules: - apiGroups: - "" resources: - - persistentvolumeclaims + - persistentvolumeclaims/status + - persistentvolumes/status + - pods/status + - secrets/status + - services/status verbs: - - create - - delete - get - - list - patch - update - - watch - apiGroups: - "" resources: - - persistentvolumeclaims/status + - pods/finalizers + - secrets/finalizers + - services/finalizers verbs: - - get - - patch - update - apiGroups: - "" resources: - - persistentvolumes + - pods/log + verbs: + - get +- apiGroups: + - apps + resources: + - deployments verbs: - create - delete @@ -21112,109 +21122,17 @@ rules: - update - watch - apiGroups: - - "" + - apps resources: - - persistentvolumes/status + - deployments/status verbs: - get - patch - update - apiGroups: - - "" + - batch resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/finalizers - verbs: - - update -- apiGroups: - - "" - resources: - - pods/log - verbs: - - get -- apiGroups: - - "" - resources: - - pods/status - verbs: - - get - - patch - - update -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets/finalizers - verbs: - - update -- apiGroups: - - "" - resources: - - secrets/status - verbs: - - get - - patch - - update -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - services/finalizers - verbs: - - update -- apiGroups: - - "" - resources: - - services/status - verbs: - - get - - patch - - update -- apiGroups: - - batch - resources: - - jobs + - jobs verbs: - create - delete @@ -21267,311 +21185,18 @@ rules: - oceanbase.oceanbase.com resources: - obclusteroperations - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obclusteroperations/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obclusteroperations/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obclusters/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obclusters/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obparameters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obparameters/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obparameters/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: + - obproxies - obresourcerescues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obresourcerescues/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obresourcerescues/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - observers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - observers/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - observers/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obtenantbackuppolicies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantbackuppolicies/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantbackuppolicies/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obtenantbackups - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantbackups/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantbackups/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obtenantoperations - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantoperations/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantoperations/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obtenantrestore - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantrestore/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obtenantrestores - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantrestores/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantrestores/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obtenants - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenants/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenants/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obtenantvariables - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantvariables/finalizers - verbs: - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - - obtenantvariables/status - verbs: - - get - - patch - - update -- apiGroups: - - oceanbase.oceanbase.com - resources: - obzones verbs: - create @@ -21584,12 +21209,37 @@ rules: - apiGroups: - oceanbase.oceanbase.com resources: + - obclusteroperations/finalizers + - obclusters/finalizers + - obparameters/finalizers + - obproxies/finalizers + - obresourcerescues/finalizers + - observers/finalizers + - obtenantbackuppolicies/finalizers + - obtenantbackups/finalizers + - obtenantoperations/finalizers + - obtenantrestores/finalizers + - obtenants/finalizers + - obtenantvariables/finalizers - obzones/finalizers verbs: - update - apiGroups: - oceanbase.oceanbase.com resources: + - obclusteroperations/status + - obclusters/status + - obparameters/status + - obproxies/status + - obresourcerescues/status + - observers/status + - obtenantbackuppolicies/status + - obtenantbackups/status + - obtenantoperations/status + - obtenantrestore/status + - obtenantrestores/status + - obtenants/status + - obtenantvariables/status - obzones/status verbs: - get @@ -21864,7 +21514,7 @@ spec: env: - name: TELEMETRY_REPORTER value: ob-operator - image: quay.io/oceanbase/ob-operator:2.3.4 + image: quay.io/oceanbase/ob-operator:2.3.3 livenessProbe: httpGet: path: /healthz diff --git a/go.mod b/go.mod index 80890f86b..6c598dfa0 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ require ( github.com/duckdb/duckdb-go/mapping v0.0.27 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/fatih/color v1.18.0 // indirect diff --git a/internal/const/oceanbase/finalizers.go b/internal/const/oceanbase/finalizers.go index 6acf491b5..217523021 100644 --- a/internal/const/oceanbase/finalizers.go +++ b/internal/const/oceanbase/finalizers.go @@ -18,5 +18,6 @@ const ( FinalizerDeleteOBServer = "finalizers.oceanbase.com.deleteobserver" FinalizerOBServer = "observer.oceanbase.com.finalizers" FinalizerDeleteOBTenant = "finalizers.oceanbase.com.deleteobtenant" + FinalizerDeleteOBProxy = "finalizers.oceanbase.com.deleteobproxy" FinalizerBackupPolicy = "obtenantbackuppolicy.finalizers.oceanbase.com" ) diff --git a/internal/const/status/obproxy/obproxy_status.go b/internal/const/status/obproxy/obproxy_status.go new file mode 100644 index 000000000..7a729ca87 --- /dev/null +++ b/internal/const/status/obproxy/obproxy_status.go @@ -0,0 +1,25 @@ +/* +Copyright (c) 2024 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package obproxy + +const ( + New = "new" + Pending = "pending" + Creating = "creating" + Running = "running" + Failed = "failed" + Updating = "updating" + Scaling = "scaling" + Deleting = "deleting" + FinalizerFinished = "finalizer finished" +) diff --git a/internal/controller/config/const.go b/internal/controller/config/const.go index 1f153b0ce..ad33feda9 100644 --- a/internal/controller/config/const.go +++ b/internal/controller/config/const.go @@ -25,4 +25,5 @@ const ( OBTenantOperationControllerName = "obtenantoperation-controller" OBResourceRescueControllerName = "obresourcerescue-controller" OBClusterOperationControllerName = "obclusteroperation-controller" + OBProxyControllerName = "obproxy-controller" ) diff --git a/internal/controller/obproxy_controller.go b/internal/controller/obproxy_controller.go new file mode 100644 index 000000000..85e648ef5 --- /dev/null +++ b/internal/controller/obproxy_controller.go @@ -0,0 +1,247 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + kubeerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" + oceanbaseconst "github.com/oceanbase/ob-operator/internal/const/oceanbase" + resobproxy "github.com/oceanbase/ob-operator/internal/resource/obproxy" + "github.com/oceanbase/ob-operator/internal/telemetry" + "github.com/oceanbase/ob-operator/pkg/coordinator" +) + +// obServerWatchPredicates enqueues OBProxy when OBServer spec changes (generation) or +// status/address/labels change, so RS_LIST can refresh when observers become Ready without spec bump. +// Global GenerationChangedPredicate is not used for this watch (see SetupWithManager). +var obServerWatchPredicates = predicate.Or( + predicate.GenerationChangedPredicate{}, + predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldO, ok1 := e.ObjectOld.(*v1alpha1.OBServer) + newO, ok2 := e.ObjectNew.(*v1alpha1.OBServer) + if !ok1 || !ok2 { + return true + } + if oldO.Status.Status != newO.Status.Status || + oldO.Status.GetConnectAddr() != newO.Status.GetConnectAddr() { + return true + } + oldRef := "" + if oldO.Labels != nil { + oldRef = oldO.Labels[oceanbaseconst.LabelRefOBCluster] + } + newRef := "" + if newO.Labels != nil { + newRef = newO.Labels[oceanbaseconst.LabelRefOBCluster] + } + return oldRef != newRef + }, + }, +) + +// +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obproxies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obproxies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obproxies/finalizers,verbs=update +// +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obclusters,verbs=get;list;watch +// +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=observers,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch + +// OBProxyReconciler reconciles an OBProxy object. +type OBProxyReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +func (r *OBProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + obproxy := &v1alpha1.OBProxy{} + err := r.Client.Get(ctx, req.NamespacedName, obproxy) + if err != nil { + if kubeerrors.IsNotFound(err) { + // obproxy not found, just return + return ctrl.Result{}, nil + } + logger.Error(err, "Get obproxy error") + return ctrl.Result{}, err + } + + // Create obproxy manager + obproxyManager := &resobproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxy, + Client: r.Client, + Logger: &logger, + Recorder: telemetry.NewRecorder(ctx, r.Recorder), + } + coordinator := coordinator.NewCoordinator(obproxyManager, &logger) + return coordinator.Coordinate() +} + +// obClusterStatusPredicate triggers only when OBCluster status changes, +// so OBProxy can start its create flow once OBCluster becomes Running. +var obClusterStatusPredicate = predicate.Funcs{ + CreateFunc: func(_ event.CreateEvent) bool { return false }, + DeleteFunc: func(_ event.DeleteEvent) bool { return false }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldC, ok1 := e.ObjectOld.(*v1alpha1.OBCluster) + newC, ok2 := e.ObjectNew.(*v1alpha1.OBCluster) + if !ok1 || !ok2 { + return true + } + return oldC.Status.Status != newC.Status.Status + }, + GenericFunc: func(_ event.GenericEvent) bool { return false }, +} + +// SetupWithManager sets up the controller with the Manager. +func (r *OBProxyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.OBProxy{}, builder.WithPredicates(preds)). + Watches( + &v1alpha1.OBServer{}, + handler.EnqueueRequestsFromMapFunc(r.mapOBServerToOBProxy), + builder.WithPredicates(obServerWatchPredicates), + ). + Watches( + &v1alpha1.OBCluster{}, + handler.EnqueueRequestsFromMapFunc(r.mapOBClusterToOBProxy), + builder.WithPredicates(obClusterStatusPredicate), + ). + Complete(r) +} + +// mapOBClusterToOBProxy enqueues OBProxy reconcile requests when the referenced OBCluster status changes. +func (r *OBProxyReconciler) mapOBClusterToOBProxy(ctx context.Context, obj client.Object) []reconcile.Request { + obcluster, ok := obj.(*v1alpha1.OBCluster) + if !ok { + return nil + } + + obproxyList := &v1alpha1.OBProxyList{} + if err := r.Client.List(ctx, obproxyList); err != nil { + return nil + } + + var requests []reconcile.Request + for _, proxy := range obproxyList.Items { + ns := proxy.Spec.OBCluster.Namespace + if ns == "" { + ns = proxy.Namespace + } + if proxy.Spec.OBCluster.Name == obcluster.Name && ns == obcluster.Namespace { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: proxy.Name, + Namespace: proxy.Namespace, + }, + }) + } + } + return requests +} + +// mapOBServerToOBProxy maps OBServer changes to OBProxy reconcile requests. +// When an OBServer is added/deleted or its status changes, we need to update +// the RS_LIST in all OBProxies that reference the same OBCluster. +func (r *OBProxyReconciler) mapOBServerToOBProxy(ctx context.Context, obj client.Object) []reconcile.Request { + logger := log.FromContext(ctx) + observer, ok := obj.(*v1alpha1.OBServer) + if !ok { + return nil + } + + // Get the OBCluster name from OBServer labels + obclusterName, hasLabel := observer.Labels[oceanbaseconst.LabelRefOBCluster] + if !hasLabel || obclusterName == "" { + return nil + } + + logger.Info("OBServer event detected, will check related OBProxies", + "observer", observer.Name, + "namespace", observer.Namespace, + "obcluster", obclusterName, + "observerStatus", observer.Status.Status, + "observerIP", observer.Status.ServiceIp) + + // Find all OBProxies that reference this OBCluster + obproxyList := &v1alpha1.OBProxyList{} + err := r.Client.List(ctx, obproxyList) + if err != nil { + logger.Error(err, "Failed to list OBProxies") + return nil + } + + var requests []reconcile.Request + for _, obproxy := range obproxyList.Items { + // Check if this OBProxy references the same OBCluster + if obproxy.Spec.OBCluster.Name == obclusterName { + // Check namespace match - OBProxy can reference OBCluster in same or different namespace + obclusterNS := obproxy.Spec.OBCluster.Namespace + if obclusterNS == "" { + obclusterNS = obproxy.Namespace + } + // Only trigger if the OBServer is in the same namespace as the OBCluster + if obclusterNS == observer.Namespace { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: obproxy.Name, + Namespace: obproxy.Namespace, + }, + }) + logger.Info("OBServer change triggers OBProxy reconciliation", + "observer", observer.Name, + "observerStatus", observer.Status.Status, + "observerIP", observer.Status.ServiceIp, + "obcluster", obclusterName, + "obproxy", obproxy.Name, + "obproxyNamespace", obproxy.Namespace, + "reason", "RS_LIST may need update due to OBServer change") + } + } + } + + if len(requests) > 0 { + logger.Info("OBServer event will trigger reconciliation for OBProxies", + "observer", observer.Name, + "obcluster", obclusterName, + "obproxyCount", len(requests)) + } + + return requests +} diff --git a/internal/controller/rbac_marks.go b/internal/controller/rbac_marks.go index a440f00ec..5bcc889ae 100644 --- a/internal/controller/rbac_marks.go +++ b/internal/controller/rbac_marks.go @@ -52,6 +52,10 @@ package controller // +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obparameters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obparameters/finalizers,verbs=update +// +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obproxies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obproxies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obproxies/finalizers,verbs=update + // +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obresourcerescues,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obresourcerescues/status,verbs=get;update;patch // +kubebuilder:rbac:groups=oceanbase.oceanbase.com,resources=obresourcerescues/finalizers,verbs=update diff --git a/internal/resource/obproxy/obproxy_builder.go b/internal/resource/obproxy/obproxy_builder.go new file mode 100644 index 000000000..c02ed85fc --- /dev/null +++ b/internal/resource/obproxy/obproxy_builder.go @@ -0,0 +1,346 @@ +/* +Copyright (c) 2024 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package obproxy + +import ( + "fmt" + "strings" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + apitypes "github.com/oceanbase/ob-operator/api/types" +) + +const ( + envPrefix = "ODP_" +) + +// Resource name prefixes +const ( + cmPrefix = "cm-" + svcPrefix = "svc-" + proxyRoSecretPrefix = "sec-ro-" +) + +// Additional label constants +const ( + LabelWithConfigMap = "obproxy.oceanbase.com/with-config-map" + LabelProxyClusterName = "obproxy.oceanbase.com/proxy-cluster-name" +) + +// Annotation constants +const ( + AnnotationServiceType = "obproxy.oceanbase.com/service-type" + AnnotationServiceIP = "obproxy.oceanbase.com/service-ip" + AnnotationProxySysSecret = "obproxy.oceanbase.com/proxy-sys-secret" +) + +// Container ports +const ( + SqlPort = 2883 + PrometheusPort = 2884 +) + +func (m *OBProxyManager) buildOwnerReference() metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: m.OBProxy.APIVersion, + Kind: m.OBProxy.Kind, + Name: m.OBProxy.Name, + UID: m.OBProxy.GetUID(), + } +} + +func (m *OBProxyManager) buildOwnerReferenceList() []metav1.OwnerReference { + return []metav1.OwnerReference{m.buildOwnerReference()} +} + +func (m *OBProxyManager) buildCommonLabels() map[string]string { + clusterNS := m.OBProxy.Spec.OBCluster.Namespace + if clusterNS == "" { + clusterNS = m.OBProxy.Namespace + } + + labels := map[string]string{ + LabelOBProxyInstance: m.OBProxy.Name, + LabelRefOBCluster: m.OBProxy.Spec.OBCluster.Name, + LabelRefOBClusterNamespace: clusterNS, + } + return labels +} + +func (m *OBProxyManager) buildConfigMap() *corev1.ConfigMap { + cmName := cmPrefix + m.OBProxy.Name + labels := m.buildCommonLabels() + + data := make(map[string]string) + for _, param := range m.OBProxy.Spec.Parameters { + key := strings.ToUpper(envPrefix + param.Name) + data[key] = param.Value + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: m.OBProxy.Namespace, + Labels: labels, + OwnerReferences: m.buildOwnerReferenceList(), + }, + Data: data, + } + + return cm +} + +func (m *OBProxyManager) buildService() *corev1.Service { + svcName := svcPrefix + m.OBProxy.Name + labels := m.buildCommonLabels() + + svcType := corev1.ServiceTypeClusterIP + if m.OBProxy.Spec.ServiceType != "" { + svcType = corev1.ServiceType(m.OBProxy.Spec.ServiceType) + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: m.OBProxy.Namespace, + Labels: labels, + OwnerReferences: m.buildOwnerReferenceList(), + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "sql", + Port: SqlPort, + TargetPort: intstr.FromInt(SqlPort), + }, + { + Name: "prometheus", + Port: PrometheusPort, + TargetPort: intstr.FromInt(PrometheusPort), + }, + }, + Selector: map[string]string{ + LabelOBProxyInstance: m.OBProxy.Name, + }, + Type: svcType, + }, + } + + return svc +} + +func (m *OBProxyManager) getProxySysSecret() (*corev1.Secret, error) { + secret := &corev1.Secret{} + err := m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: m.OBProxy.Namespace, + Name: m.OBProxy.Spec.ProxySysSecret, + }, secret) + if err != nil { + return nil, errors.Wrap(err, "get proxy sys secret") + } + return secret, nil +} + +func (m *OBProxyManager) buildDeployment(rsList string, svc *corev1.Service, proxyRoSecret, proxySysSecret *corev1.Secret) *appsv1.Deployment { + labels := m.buildCommonLabels() + labels[LabelProxyClusterName] = m.OBProxy.Spec.ProxyClusterName + + podLabels := map[string]string{ + LabelOBProxyInstance: m.OBProxy.Name, + } + + resources := m.buildResourceRequirements() + + proxyClusterName := m.OBProxy.Spec.ProxyClusterName + if proxyClusterName == "" { + proxyClusterName = m.OBProxy.Name + } + + container := corev1.Container{ + Name: "obproxy", + Image: m.OBProxy.Spec.Image, + Ports: []corev1.ContainerPort{ + { + Name: "sql", + ContainerPort: SqlPort, + }, + { + Name: "prometheus", + ContainerPort: PrometheusPort, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "RS_LIST", + Value: rsList, + }, + { + Name: "APP_NAME", + Value: proxyClusterName, + }, + { + Name: "OB_CLUSTER", + Value: m.OBProxy.Spec.OBCluster.Name, + }, + { + Name: "PROXYRO_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: proxyRoSecret.Name, + }, + Key: "password", + }, + }, + }, + { + Name: "PROXYSYS_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: proxySysSecret.Name, + }, + Key: "password", + }, + }, + }, + }, + Resources: resources, + } + + cmName := m.getConfigMapName() + container.EnvFrom = []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmName, + }, + }, + }, + } + + if m.OBProxy.Spec.Resource != nil && !m.OBProxy.Spec.Resource.Memory.IsZero() { + memoryMB := m.OBProxy.Spec.Resource.Memory.Value() * 95 / 100 / (1 << 20) + container.Env = append(container.Env, corev1.EnvVar{ + Name: "ODP_PROXY_MEM_LIMITED", + Value: fmt.Sprintf("%dMB", memoryMB), + }) + } + + replicas := m.OBProxy.Spec.Replicas + if replicas == 0 { + replicas = 1 + } + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: m.OBProxy.Name, + Namespace: m.OBProxy.Namespace, + Labels: labels, + OwnerReferences: m.buildOwnerReferenceList(), + Annotations: map[string]string{ + AnnotationServiceType: string(svc.Spec.Type), + AnnotationServiceIP: svc.Spec.ClusterIP, + AnnotationProxySysSecret: proxySysSecret.Name, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: podLabels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{container}, + NodeSelector: m.OBProxy.Spec.NodeSelector, + Affinity: m.OBProxy.Spec.Affinity, + Tolerations: m.OBProxy.Spec.Tolerations, + ServiceAccountName: m.OBProxy.Spec.ServiceAccount, + }, + }, + }, + } + + return deploy +} + +func (m *OBProxyManager) buildResourceRequirements() corev1.ResourceRequirements { + resources := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{}, + } + + if m.OBProxy.Spec.Resource != nil { + if !m.OBProxy.Spec.Resource.Cpu.IsZero() { + resources.Limits[corev1.ResourceCPU] = m.OBProxy.Spec.Resource.Cpu + } + if !m.OBProxy.Spec.Resource.Memory.IsZero() { + resources.Limits[corev1.ResourceMemory] = m.OBProxy.Spec.Resource.Memory + } + } + + return resources +} + +func (m *OBProxyManager) buildCopiedProxyROSecret(sourceSecret *corev1.Secret) *corev1.Secret { + secretName := proxyRoSecretPrefix + m.OBProxy.Name + labels := m.buildCommonLabels() + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: m.OBProxy.Namespace, + Labels: labels, + OwnerReferences: m.buildOwnerReferenceList(), + }, + Data: sourceSecret.Data, + } + + return secret +} + +func (m *OBProxyManager) getConfigMapName() string { + return cmPrefix + m.OBProxy.Name +} + +func (m *OBProxyManager) getServiceName() string { + return svcPrefix + m.OBProxy.Name +} + +func (m *OBProxyManager) getProxyROSecretName() string { + return proxyRoSecretPrefix + m.OBProxy.Name +} + +func (m *OBProxyManager) updateConfigMapData(cm *corev1.ConfigMap, parameters []apitypes.Parameter) *corev1.ConfigMap { + cmCopy := cm.DeepCopy() + cmCopy.Data = make(map[string]string) + for _, param := range parameters { + key := strings.ToUpper(envPrefix + param.Name) + cmCopy.Data[key] = param.Value + } + return cmCopy +} + diff --git a/internal/resource/obproxy/obproxy_flow.go b/internal/resource/obproxy/obproxy_flow.go new file mode 100644 index 000000000..f5d1da0b0 --- /dev/null +++ b/internal/resource/obproxy/obproxy_flow.go @@ -0,0 +1,83 @@ +/* +Copyright (c) 2024 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package obproxy + +import ( + proxystatus "github.com/oceanbase/ob-operator/internal/const/status/obproxy" + "github.com/oceanbase/ob-operator/pkg/task/const/strategy" + tasktypes "github.com/oceanbase/ob-operator/pkg/task/types" +) + +func genCreateOBProxyFlow(_ *OBProxyManager) *tasktypes.TaskFlow { + return &tasktypes.TaskFlow{ + OperationContext: &tasktypes.OperationContext{ + Name: "create obproxy", + Tasks: []tasktypes.TaskName{ + tCopyProxyROSecret, + tCreateOBProxyConfigMap, + tCreateOBProxyService, + tCreateOBProxyDeployment, + tWaitOBProxyReady, + }, + TargetStatus: proxystatus.Running, + OnFailure: tasktypes.FailureRule{ + NextTryStatus: proxystatus.Failed, + }, + }, + } +} + +func genUpdateOBProxyFlow(_ *OBProxyManager) *tasktypes.TaskFlow { + return &tasktypes.TaskFlow{ + OperationContext: &tasktypes.OperationContext{ + Name: "update obproxy", + Tasks: []tasktypes.TaskName{ + tUpdateOBProxyConfigMap, + tUpdateOBProxyDeployment, + tWaitOBProxyReady, + }, + TargetStatus: proxystatus.Running, + }, + } +} + +func genScaleOBProxyFlow(_ *OBProxyManager) *tasktypes.TaskFlow { + return &tasktypes.TaskFlow{ + OperationContext: &tasktypes.OperationContext{ + Name: "scale obproxy", + Tasks: []tasktypes.TaskName{ + tScaleOBProxyDeployment, + tWaitOBProxyReady, + }, + TargetStatus: proxystatus.Running, + }, + } +} + +func genDeleteOBProxyFlow(_ *OBProxyManager) *tasktypes.TaskFlow { + return &tasktypes.TaskFlow{ + OperationContext: &tasktypes.OperationContext{ + Name: "delete obproxy", + Tasks: []tasktypes.TaskName{ + tDeleteOBProxyDeployment, + tDeleteOBProxyService, + tDeleteOBProxyConfigMap, + tDeleteOBProxySecrets, + }, + TargetStatus: proxystatus.FinalizerFinished, + OnFailure: tasktypes.FailureRule{ + Strategy: strategy.StartOver, + }, + }, + } +} diff --git a/internal/resource/obproxy/obproxy_manager.go b/internal/resource/obproxy/obproxy_manager.go new file mode 100644 index 000000000..7bd9146cf --- /dev/null +++ b/internal/resource/obproxy/obproxy_manager.go @@ -0,0 +1,366 @@ +/* +Copyright (c) 2024 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package obproxy + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + apitypes "github.com/oceanbase/ob-operator/api/types" + v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" + oceanbaseconst "github.com/oceanbase/ob-operator/internal/const/oceanbase" + clusterstatus "github.com/oceanbase/ob-operator/internal/const/status/obcluster" + proxystatus "github.com/oceanbase/ob-operator/internal/const/status/obproxy" + "github.com/oceanbase/ob-operator/internal/telemetry" + opresource "github.com/oceanbase/ob-operator/pkg/coordinator" + taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/status" + "github.com/oceanbase/ob-operator/pkg/task/const/strategy" + tasktypes "github.com/oceanbase/ob-operator/pkg/task/types" +) + +var _ opresource.ResourceManager = &OBProxyManager{} + +type OBProxyManager struct { + Ctx context.Context + OBProxy *v1alpha1.OBProxy + Client client.Client + Recorder telemetry.Recorder + Logger *logr.Logger +} + +func (m *OBProxyManager) GetMeta() metav1.Object { + return m.OBProxy.GetObjectMeta() +} + +func (m *OBProxyManager) GetStatus() string { + return m.OBProxy.Status.Status +} + +func (m *OBProxyManager) InitStatus() { + m.Logger.Info("Newly created obproxy, init status") + m.Recorder.Event(m.OBProxy, "Init", "", "Newly created obproxy, init status") + status := v1alpha1.OBProxyStatus{ + Status: proxystatus.New, + Image: m.OBProxy.Spec.Image, + Replicas: m.OBProxy.Spec.Replicas, + } + m.OBProxy.Status = status +} + +func (m *OBProxyManager) SetOperationContext(c *tasktypes.OperationContext) { + m.OBProxy.Status.OperationContext = c +} + +func (m *OBProxyManager) GetTaskFlow() (*tasktypes.TaskFlow, error) { + // exists unfinished task flow, return the last task flow + if m.OBProxy.Status.OperationContext != nil { + m.Logger.V(oceanbaseconst.LogLevelTrace).Info("Get task flow from obproxy status") + return tasktypes.NewTaskFlow(m.OBProxy.Status.OperationContext), nil + } + + // return task flow depends on status + var taskFlow *tasktypes.TaskFlow + m.Logger.V(oceanbaseconst.LogLevelTrace).Info("Create task flow according to obproxy status") + + switch m.OBProxy.Status.Status { + case proxystatus.New: + obcluster, err := m.getOBCluster() + if err != nil { + m.Logger.Info("OBCluster not found, waiting", "cluster", m.OBProxy.Spec.OBCluster.Name) + return nil, nil + } + if obcluster.Status.Status != clusterstatus.Running { + m.Logger.Info("OBCluster not ready, waiting", "cluster", obcluster.Name, "status", obcluster.Status.Status) + return nil, nil + } + taskFlow = genCreateOBProxyFlow(m) + case proxystatus.Updating: + taskFlow = genUpdateOBProxyFlow(m) + case proxystatus.Scaling: + taskFlow = genScaleOBProxyFlow(m) + case proxystatus.Deleting: + taskFlow = genDeleteOBProxyFlow(m) + default: + m.Logger.V(oceanbaseconst.LogLevelTrace).Info("No need to run anything for obproxy", "obproxy", m.OBProxy.Name) + return nil, nil + } + + if taskFlow.OperationContext.OnFailure.Strategy == "" { + taskFlow.OperationContext.OnFailure.Strategy = strategy.StartOver + if taskFlow.OperationContext.OnFailure.NextTryStatus == "" { + taskFlow.OperationContext.OnFailure.NextTryStatus = proxystatus.Running + } + } + + return taskFlow, nil +} + +func (m *OBProxyManager) CheckAndUpdateFinalizers() error { + if m.OBProxy.Status.Status == proxystatus.FinalizerFinished { + m.OBProxy.ObjectMeta.Finalizers = make([]string, 0) + return m.Client.Update(m.Ctx, m.OBProxy) + } + return nil +} + +func (m *OBProxyManager) UpdateStatus() error { + deployment, err := m.getOBProxyDeployment() + if err != nil { + m.Logger.Error(err, "get obproxy deployment error") + return errors.Wrap(err, "get obproxy deployment") + } + if deployment != nil { + m.OBProxy.Status.Image = deployment.Spec.Template.Spec.Containers[0].Image + if deployment.Spec.Replicas != nil { + m.OBProxy.Status.Replicas = *deployment.Spec.Replicas + } + m.OBProxy.Status.ReadyReplicas = deployment.Status.ReadyReplicas + // Read RS_LIST env from the running Deployment (observed, not desired). + for _, c := range deployment.Spec.Template.Spec.Containers { + if c.Name == "obproxy" { + for _, env := range c.Env { + if env.Name == "RS_LIST" { + m.OBProxy.Status.RSList = env.Value + break + } + } + break + } + } + } + + if svc, svcErr := m.getOBProxyService(); svcErr == nil && svc != nil { + m.OBProxy.Status.ServiceIP = svc.Spec.ClusterIP + } + + obcluster, clusterErr := m.getOBCluster() + if clusterErr != nil { + m.setCondition("OBClusterAvailable", false, "NotFound", clusterErr.Error()) + m.setCondition("OBClusterReady", false, "Unknown", "OBCluster not reachable") + m.Logger.V(1).Info("cannot get obcluster", "error", clusterErr) + } else { + m.setCondition("OBClusterAvailable", true, "Found", "") + if obcluster.Status.Status == clusterstatus.Running && obcluster.Status.OperationContext == nil { + m.setCondition("OBClusterReady", true, "Running", "") + } else { + m.setCondition("OBClusterReady", false, "Transitioning", obcluster.Status.Status) + } + } + + if deployment != nil && m.OBProxy.Status.Status == proxystatus.Running { + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == "obproxy" && container.Image != m.OBProxy.Spec.Image { + m.Logger.Info("OBProxy image changed, need update") + m.OBProxy.Status.Status = proxystatus.Updating + break + } + } + + if m.OBProxy.Status.Status == proxystatus.Running { + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != m.OBProxy.Spec.Replicas { + m.Logger.Info("OBProxy replicas changed, need scale") + m.OBProxy.Status.Status = proxystatus.Scaling + } + } + + if m.OBProxy.Status.Status == proxystatus.Running && m.OBProxy.Spec.Resource != nil { + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == "obproxy" { + if !m.isResourceEqual(container.Resources, *m.OBProxy.Spec.Resource) { + m.Logger.Info("OBProxy resource changed, need update") + m.OBProxy.Status.Status = proxystatus.Updating + break + } + } + } + } + + if m.OBProxy.Status.Status == proxystatus.Running { + podSpec := deployment.Spec.Template.Spec + if !reflect.DeepEqual(podSpec.NodeSelector, m.OBProxy.Spec.NodeSelector) || + !reflect.DeepEqual(podSpec.Affinity, m.OBProxy.Spec.Affinity) || + !reflect.DeepEqual(podSpec.Tolerations, m.OBProxy.Spec.Tolerations) { + m.Logger.Info("OBProxy scheduling constraints changed, need update") + m.OBProxy.Status.Status = proxystatus.Updating + } + } + + if m.OBProxy.Status.Status == proxystatus.Running { + if clusterErr != nil { + m.setCondition("RSListAvailable", false, "OBClusterUnavailable", "cannot get obcluster") + m.Logger.V(1).Info("skip RS_LIST drift check, cannot get obcluster", "error", clusterErr) + } else if obcluster.Status.Status != clusterstatus.Running || obcluster.Status.OperationContext != nil { + m.setCondition("RSListAvailable", false, "OBClusterNotStable", obcluster.Status.Status) + m.Logger.V(1).Info("skip RS_LIST drift check, obcluster is not stable", + "clusterStatus", obcluster.Status.Status, + "hasOperationContext", obcluster.Status.OperationContext != nil) + } else { + desiredRS, rsSrc, rsErr := m.getRootServiceList() + if rsErr != nil { + m.setCondition("RSListAvailable", false, "ResolveFailed", rsErr.Error()) + m.Logger.V(1).Info("skip RS_LIST drift check", "error", rsErr) + } else { + m.setCondition("RSListAvailable", true, "Resolved", "") + m.OBProxy.Status.RSListSource = rsSrc + currentRS := m.OBProxy.Status.RSList + if desiredRS != currentRS { + m.Logger.Info("RS_LIST drift detected, OBProxy will restart to update RS_LIST", + "obproxy", m.OBProxy.Name, + "namespace", m.OBProxy.Namespace, + "desiredRS", desiredRS, + "currentRS", currentRS) + m.Recorder.Event(m.OBProxy, "RSListDrift", "Update", + fmt.Sprintf("RS_LIST drift detected: %s -> %s", currentRS, desiredRS)) + m.OBProxy.Status.Status = proxystatus.Updating + } + } + } + } + + if m.OBProxy.Status.Status == proxystatus.Running { + cm, cmErr := m.getOBProxyConfigMap() + if cmErr == nil && cm != nil && !m.isParametersEqual(cm.Data) { + m.Logger.Info("OBProxy parameters changed, need update") + m.OBProxy.Status.Status = proxystatus.Updating + } + } + } + + m.Logger.V(oceanbaseconst.LogLevelTrace).Info("Update obproxy status", "status", m.OBProxy.Status) + err = m.retryUpdateStatus() + if err != nil { + m.Logger.Error(err, "Got error when update obproxy status") + } + return err +} + +func (m *OBProxyManager) ClearTaskInfo() { + m.OBProxy.Status.Status = proxystatus.Running + m.OBProxy.Status.OperationContext = nil +} + +func (m *OBProxyManager) FinishTask() { + m.OBProxy.Status.Status = m.OBProxy.Status.OperationContext.TargetStatus + m.OBProxy.Status.OperationContext = nil +} + +func (m *OBProxyManager) HandleFailure() { + operationContext := m.OBProxy.Status.OperationContext + failureRule := operationContext.OnFailure + switch failureRule.Strategy { + case strategy.StartOver: + if m.OBProxy.Status.Status != failureRule.NextTryStatus { + m.OBProxy.Status.Status = failureRule.NextTryStatus + m.OBProxy.Status.OperationContext = nil + } else { + m.OBProxy.Status.OperationContext.Idx = 0 + m.OBProxy.Status.OperationContext.TaskStatus = "" + m.OBProxy.Status.OperationContext.TaskId = "" + m.OBProxy.Status.OperationContext.Task = "" + } + case strategy.RetryFromCurrent: + operationContext.TaskStatus = taskstatus.Pending + case strategy.Pause: + } +} + +func (m *OBProxyManager) GetTaskFunc(name tasktypes.TaskName) (tasktypes.TaskFunc, error) { + return taskMap.GetTask(name, m) +} + +func (m *OBProxyManager) PrintErrEvent(err error) { + m.Recorder.Event(m.OBProxy, corev1.EventTypeWarning, "Task failed", err.Error()) +} + +func (m *OBProxyManager) ArchiveResource() { + m.Logger.Info("Archive obproxy", "obproxy", m.OBProxy.Name) + m.Recorder.Event(m.OBProxy, "Archive", "", "Archive obproxy") + m.OBProxy.Status.Status = proxystatus.Failed + m.OBProxy.Status.OperationContext = nil +} + +// Helper functions for manager + +func (m *OBProxyManager) retryUpdateStatus() error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + return m.Client.Status().Update(m.Ctx, m.OBProxy) + }) +} + +func (m *OBProxyManager) isResourceEqual(resources corev1.ResourceRequirements, spec apitypes.ResourceSpec) bool { + // Compare CPU + if !spec.Cpu.IsZero() { + if resources.Limits.Cpu().Cmp(spec.Cpu) != 0 { + return false + } + } + // Compare Memory + if !spec.Memory.IsZero() { + if resources.Limits.Memory().Cmp(spec.Memory) != 0 { + return false + } + } + return true +} + +func (m *OBProxyManager) isParametersEqual(cmData map[string]string) bool { + if len(cmData) != len(m.OBProxy.Spec.Parameters) { + return false + } + for _, param := range m.OBProxy.Spec.Parameters { + key := strings.ToUpper(envPrefix + param.Name) + if cmData[key] != param.Value { + return false + } + } + return true +} + +// setCondition upserts a Condition on the OBProxy status. +// It preserves LastTransitionTime when the status value has not changed. +func (m *OBProxyManager) setCondition(condType string, ok bool, reason, message string) { + desired := metav1.ConditionFalse + if ok { + desired = metav1.ConditionTrue + } + now := metav1.Now() + for i := range m.OBProxy.Status.Conditions { + c := &m.OBProxy.Status.Conditions[i] + if c.Type != condType { + continue + } + if c.Status != desired { + c.LastTransitionTime = now + } + c.Status = desired + c.Reason = reason + c.Message = message + return + } + m.OBProxy.Status.Conditions = append(m.OBProxy.Status.Conditions, metav1.Condition{ + Type: condType, + Status: desired, + LastTransitionTime: now, + Reason: reason, + Message: message, + }) +} diff --git a/internal/resource/obproxy/obproxy_task.go b/internal/resource/obproxy/obproxy_task.go new file mode 100644 index 000000000..11567bc1b --- /dev/null +++ b/internal/resource/obproxy/obproxy_task.go @@ -0,0 +1,541 @@ +/* +Copyright (c) 2024 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ +//go:generate task_register $GOFILE + +package obproxy + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kubeerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" + oceanbaseconst "github.com/oceanbase/ob-operator/internal/const/oceanbase" + observerstatus "github.com/oceanbase/ob-operator/internal/const/status/observer" + "github.com/oceanbase/ob-operator/pkg/oceanbase-sdk/connector" + "github.com/oceanbase/ob-operator/pkg/oceanbase-sdk/operation" + "github.com/oceanbase/ob-operator/pkg/task/builder" + tasktypes "github.com/oceanbase/ob-operator/pkg/task/types" +) + +var taskMap = builder.NewTaskHub[*OBProxyManager]() + +func CopyProxyROSecret(m *OBProxyManager) tasktypes.TaskError { + obcluster, err := m.getOBCluster() + if err != nil { + m.Logger.Error(err, "Failed to get obcluster") + return errors.Wrap(err, "get obcluster") + } + + clusterNS := m.OBProxy.Spec.OBCluster.Namespace + if clusterNS == "" { + clusterNS = m.OBProxy.Namespace + } + + proxyROSecretName := obcluster.Spec.UserSecrets.ProxyRO + if proxyROSecretName == "" { + return errors.New("obcluster does not have proxyRO secret configured") + } + + sourceSecret := &corev1.Secret{} + err = m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: clusterNS, + Name: proxyROSecretName, + }, sourceSecret) + if err != nil { + m.Logger.Error(err, "Failed to get proxyRO secret from obcluster namespace", + "namespace", clusterNS, "secret", proxyROSecretName) + return errors.Wrap(err, "get proxyRO secret from obcluster namespace") + } + + targetSecretName := m.getProxyROSecretName() + existingSecret := &corev1.Secret{} + err = m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: m.OBProxy.Namespace, + Name: targetSecretName, + }, existingSecret) + if err == nil { + return nil + } + if !kubeerrors.IsNotFound(err) { + return errors.Wrap(err, "check existing proxyRO secret") + } + + newSecret := m.buildCopiedProxyROSecret(sourceSecret) + err = m.Client.Create(m.Ctx, newSecret) + if err != nil { + m.Logger.Error(err, "Failed to create proxyRO secret copy") + return errors.Wrap(err, "create proxyRO secret copy") + } + + m.Logger.Info("Created proxyRO secret copy", "secret", targetSecretName) + m.Recorder.Event(m.OBProxy, "CreateSecret", "", fmt.Sprintf("Created proxyRO secret %s", targetSecretName)) + return nil +} + +func CreateOBProxyConfigMap(m *OBProxyManager) tasktypes.TaskError { + cmName := m.getConfigMapName() + + existingCM := &corev1.ConfigMap{} + err := m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: m.OBProxy.Namespace, + Name: cmName, + }, existingCM) + if err == nil { + m.Logger.Info("ConfigMap already exists", "configmap", cmName) + return nil + } + if !kubeerrors.IsNotFound(err) { + return errors.Wrap(err, "check existing configmap") + } + + cm := m.buildConfigMap() + err = m.Client.Create(m.Ctx, cm) + if err != nil { + m.Logger.Error(err, "Failed to create configmap") + return errors.Wrap(err, "create configmap") + } + + m.Logger.Info("Created ConfigMap", "configmap", cmName) + m.Recorder.Event(m.OBProxy, "CreateConfigMap", "", fmt.Sprintf("Created ConfigMap %s", cmName)) + return nil +} + +func CreateOBProxyService(m *OBProxyManager) tasktypes.TaskError { + svcName := m.getServiceName() + + existingSvc := &corev1.Service{} + err := m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: m.OBProxy.Namespace, + Name: svcName, + }, existingSvc) + if err == nil { + m.Logger.Info("Service already exists", "service", svcName) + return nil + } + if !kubeerrors.IsNotFound(err) { + return errors.Wrap(err, "check existing service") + } + + svc := m.buildService() + err = m.Client.Create(m.Ctx, svc) + if err != nil { + m.Logger.Error(err, "Failed to create service") + return errors.Wrap(err, "create service") + } + + m.Logger.Info("Created Service", "service", svcName) + m.Recorder.Event(m.OBProxy, "CreateService", "", fmt.Sprintf("Created Service %s", svcName)) + return nil +} + +func CreateOBProxyDeployment(m *OBProxyManager) tasktypes.TaskError { + rsList, _, err := m.getRootServiceList() + if err != nil { + m.Logger.Error(err, "Failed to get rootservice list") + return errors.Wrap(err, "get rootservice list") + } + + svcName := m.getServiceName() + svc := &corev1.Service{} + err = m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: m.OBProxy.Namespace, + Name: svcName, + }, svc) + if err != nil { + m.Logger.Error(err, "Failed to get service", "service", svcName) + return errors.Wrap(err, "get service") + } + + proxySysSecret, err := m.getProxySysSecret() + if err != nil { + m.Logger.Error(err, "Failed to get proxySys secret") + return errors.Wrap(err, "get proxySys secret") + } + + proxyROSecretName := m.getProxyROSecretName() + proxyROSecret := &corev1.Secret{} + err = m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: m.OBProxy.Namespace, + Name: proxyROSecretName, + }, proxyROSecret) + if err != nil { + m.Logger.Error(err, "Failed to get proxyRO secret", "secret", proxyROSecretName) + return errors.Wrap(err, "get proxyRO secret") + } + + existingDeploy := &appsv1.Deployment{} + err = m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: m.OBProxy.Namespace, + Name: m.OBProxy.Name, + }, existingDeploy) + if err == nil { + m.Logger.Info("Deployment already exists", "deployment", m.OBProxy.Name) + return nil + } + if !kubeerrors.IsNotFound(err) { + return errors.Wrap(err, "check existing deployment") + } + + deploy := m.buildDeployment(rsList, svc, proxyROSecret, proxySysSecret) + err = m.Client.Create(m.Ctx, deploy) + if err != nil { + m.Logger.Error(err, "Failed to create deployment") + return errors.Wrap(err, "create deployment") + } + + m.Logger.Info("Created Deployment", "deployment", m.OBProxy.Name, "rsList", rsList) + m.Recorder.Event(m.OBProxy, "CreateDeployment", "", fmt.Sprintf("Created Deployment %s", m.OBProxy.Name)) + return nil +} + +func WaitOBProxyReady(m *OBProxyManager) tasktypes.TaskError { + timeout := 300 // 5 minutes + for range timeout { + deploy, err := m.getOBProxyDeployment() + if err != nil { + return errors.Wrap(err, "get obproxy deployment during wait") + } + if deploy == nil { + return errors.New("deployment not found during wait") + } + + if m.isOBProxyReady(deploy) { + m.Logger.Info("OBProxy is ready", + "obproxy", m.OBProxy.Name, + "replicas", deploy.Status.ReadyReplicas) + m.Recorder.Event(m.OBProxy, "Ready", "", + fmt.Sprintf("OBProxy ready: %d/%d replicas", deploy.Status.ReadyReplicas, deploy.Status.Replicas)) + return nil + } + + m.Logger.V(oceanbaseconst.LogLevelDebug).Info("Waiting for OBProxy to be ready", + "ready", deploy.Status.ReadyReplicas, "desired", deploy.Status.Replicas) + time.Sleep(time.Second) + } + + m.Logger.Error(errors.New("timeout"), "OBProxy wait timeout", "obproxy", m.OBProxy.Name) + return errors.New("timeout waiting for obproxy to be ready") +} + +func UpdateOBProxyConfigMap(m *OBProxyManager) tasktypes.TaskError { + m.Logger.Info("Updating OBProxy ConfigMap", + "obproxy", m.OBProxy.Name, + "namespace", m.OBProxy.Namespace) + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + cm, err := m.getOBProxyConfigMap() + if err != nil { + return errors.Wrap(err, "get obproxy configmap") + } + if cm == nil { + newCM := m.buildConfigMap() + return m.Client.Create(m.Ctx, newCM) + } + + updatedCM := m.updateConfigMapData(cm, m.OBProxy.Spec.Parameters) + return m.Client.Update(m.Ctx, updatedCM) + }) +} + +func UpdateOBProxyDeployment(m *OBProxyManager) tasktypes.TaskError { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + deploy, err := m.getOBProxyDeployment() + if err != nil { + return errors.Wrap(err, "get obproxy deployment") + } + if deploy == nil { + return errors.New("deployment not found") + } + + m.Logger.Info("Updating OBProxy Deployment", + "obproxy", m.OBProxy.Name, + "namespace", m.OBProxy.Namespace, + "deployment", deploy.Name, + "replicas", m.OBProxy.Spec.Replicas, + "image", m.OBProxy.Spec.Image) + + if deploy.Spec.Template.Spec.Containers[0].Image != m.OBProxy.Spec.Image { + deploy.Spec.Template.Spec.Containers[0].Image = m.OBProxy.Spec.Image + } + + if deploy.Spec.Replicas == nil || *deploy.Spec.Replicas != m.OBProxy.Spec.Replicas { + replicas := m.OBProxy.Spec.Replicas + deploy.Spec.Replicas = &replicas + } + + rsList, _, err := m.getRootServiceList() + if err != nil { + return errors.Wrap(err, "get rootservice list") + } + for i := range deploy.Spec.Template.Spec.Containers { + container := &deploy.Spec.Template.Spec.Containers[i] + if container.Name == "obproxy" { + for j := range container.Env { + if container.Env[j].Name == "RS_LIST" { + if container.Env[j].Value != rsList { + container.Env[j].Value = rsList + } + break + } + } + + container.Resources = m.buildResourceRequirements() + + var newMemLimited string + if m.OBProxy.Spec.Resource != nil && !m.OBProxy.Spec.Resource.Memory.IsZero() { + memoryMB := m.OBProxy.Spec.Resource.Memory.Value() * 95 / 100 / (1 << 20) + newMemLimited = fmt.Sprintf("%dMB", memoryMB) + } + memIdx := -1 + for j := range container.Env { + if container.Env[j].Name == "ODP_PROXY_MEM_LIMITED" { + memIdx = j + break + } + } + if memIdx >= 0 { + if newMemLimited == "" { + container.Env = append(container.Env[:memIdx], container.Env[memIdx+1:]...) + } else { + container.Env[memIdx].Value = newMemLimited + } + } else if newMemLimited != "" { + container.Env = append(container.Env, corev1.EnvVar{Name: "ODP_PROXY_MEM_LIMITED", Value: newMemLimited}) + } + + break + } + } + + deploy.Spec.Template.Spec.NodeSelector = m.OBProxy.Spec.NodeSelector + deploy.Spec.Template.Spec.Affinity = m.OBProxy.Spec.Affinity + deploy.Spec.Template.Spec.Tolerations = m.OBProxy.Spec.Tolerations + + return m.Client.Update(m.Ctx, deploy) + }) +} + +func ScaleOBProxyDeployment(m *OBProxyManager) tasktypes.TaskError { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + deploy, err := m.getOBProxyDeployment() + if err != nil { + return errors.Wrap(err, "get obproxy deployment") + } + if deploy == nil { + return errors.New("deployment not found") + } + + replicas := m.OBProxy.Spec.Replicas + if deploy.Spec.Replicas != nil && *deploy.Spec.Replicas == replicas { + return nil + } + + deploy.Spec.Replicas = &replicas + return m.Client.Update(m.Ctx, deploy) + }) +} + +func DeleteOBProxyDeployment(m *OBProxyManager) tasktypes.TaskError { + deploy, err := m.getOBProxyDeployment() + if err != nil { + return errors.Wrap(err, "get obproxy deployment") + } + if deploy == nil { + m.Logger.Info("Deployment already deleted") + return nil + } + + err = m.Client.Delete(m.Ctx, deploy) + if err != nil { + if kubeerrors.IsNotFound(err) { + m.Logger.Info("Deployment already deleted") + return nil + } + return errors.Wrap(err, "delete deployment") + } + + m.Logger.Info("Deleted Deployment", "deployment", deploy.Name) + m.Recorder.Event(m.OBProxy, "DeleteDeployment", "", fmt.Sprintf("Deleted Deployment %s", deploy.Name)) + return nil +} + +func DeleteOBProxyService(m *OBProxyManager) tasktypes.TaskError { + svc, err := m.getOBProxyService() + if err != nil { + return errors.Wrap(err, "get obproxy service") + } + if svc == nil { + m.Logger.Info("Service already deleted") + return nil + } + + err = m.Client.Delete(m.Ctx, svc) + if err != nil { + if kubeerrors.IsNotFound(err) { + m.Logger.Info("Service already deleted") + return nil + } + return errors.Wrap(err, "delete service") + } + + m.Logger.Info("Deleted Service", "service", svc.Name) + m.Recorder.Event(m.OBProxy, "DeleteService", "", fmt.Sprintf("Deleted Service %s", svc.Name)) + return nil +} + +func DeleteOBProxyConfigMap(m *OBProxyManager) tasktypes.TaskError { + cm, err := m.getOBProxyConfigMap() + if err != nil { + return errors.Wrap(err, "get obproxy configmap") + } + if cm == nil { + m.Logger.Info("ConfigMap already deleted") + return nil + } + + err = m.Client.Delete(m.Ctx, cm) + if err != nil { + if kubeerrors.IsNotFound(err) { + m.Logger.Info("ConfigMap already deleted") + return nil + } + return errors.Wrap(err, "delete configmap") + } + + m.Logger.Info("Deleted ConfigMap", "configmap", cm.Name) + m.Recorder.Event(m.OBProxy, "DeleteConfigMap", "", fmt.Sprintf("Deleted ConfigMap %s", cm.Name)) + return nil +} + +func DeleteOBProxySecrets(m *OBProxyManager) tasktypes.TaskError { + proxyROSecretName := m.getProxyROSecretName() + proxyROSecret := &corev1.Secret{} + err := m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: m.OBProxy.Namespace, + Name: proxyROSecretName, + }, proxyROSecret) + if err == nil { + err = m.Client.Delete(m.Ctx, proxyROSecret) + if err != nil && !kubeerrors.IsNotFound(err) { + return errors.Wrap(err, "delete proxyRO secret") + } + m.Logger.Info("Deleted proxyRO secret", "secret", proxyROSecretName) + } + + m.Recorder.Event(m.OBProxy, "DeleteSecrets", "", "Deleted OBProxy secrets") + return nil +} + +func (m *OBProxyManager) getRootServiceList() (string, string, error) { + obcluster, err := m.getOBCluster() + if err != nil { + return "", "", errors.Wrap(err, "get obcluster") + } + + observerList := &v1alpha1.OBServerList{} + err = m.Client.List(m.Ctx, observerList, + client.MatchingLabels{oceanbaseconst.LabelRefOBCluster: obcluster.Name}, + client.InNamespace(obcluster.Namespace), + ) + if err != nil { + return "", "", errors.Wrap(err, "list observers") + } + + var rsList []string + for _, observer := range observerList.Items { + if observer.DeletionTimestamp != nil { + continue + } + if observer.Status.Status == observerstatus.Running { + rsList = append(rsList, fmt.Sprintf("%s:%d", observer.Status.GetConnectAddr(), oceanbaseconst.SqlPort)) + } + } + + if len(rsList) > 0 { + sort.Strings(rsList) + return strings.Join(rsList, ";"), "k8s", nil + } + + // Fallback: try to get from OceanBase parameter via SQL + rs, err := m.getRootServiceListFromDB(obcluster) + if err != nil { + return "", "", err + } + return rs, "sql", nil +} + +func (m *OBProxyManager) getRootServiceListFromDB(obcluster *v1alpha1.OBCluster) (string, error) { + secretName := obcluster.Spec.UserSecrets.Root + if secretName == "" { + return "", errors.New("root secret not configured in obcluster") + } + + secret := &corev1.Secret{} + err := m.Client.Get(m.Ctx, types.NamespacedName{ + Namespace: obcluster.Namespace, + Name: secretName, + }, secret) + if err != nil { + return "", errors.Wrap(err, "get root secret") + } + + password := string(secret.Data["password"]) + + observerList := &v1alpha1.OBServerList{} + err = m.Client.List(m.Ctx, observerList, + client.MatchingLabels{oceanbaseconst.LabelRefOBCluster: obcluster.Name}, + client.InNamespace(obcluster.Namespace), + ) + if err != nil { + return "", errors.Wrap(err, "list observers") + } + + if len(observerList.Items) == 0 { + return "", errors.New("no observers found") + } + + for _, observer := range observerList.Items { + if observer.Status.Status != observerstatus.Running { + continue + } + + address := observer.Status.GetConnectAddr() + dataSource := connector.NewOceanBaseDataSource(address, oceanbaseconst.SqlPort, "root", "sys", password, oceanbaseconst.DefaultDatabase) + manager, err := operation.GetOceanbaseOperationManager(dataSource) + if err != nil { + continue + } + + parameters, err := manager.GetParameter(m.Ctx, "rootservice_list", nil) + if err != nil { + continue + } + + if len(parameters) > 0 { + rsList := parameters[0].Value + rsList = strings.ReplaceAll(rsList, ":2882", "") // strip OB internal port suffix + return rsList, nil + } + } + + return "", errors.New("failed to get rootservice_list from any observer") +} diff --git a/internal/resource/obproxy/obproxy_task_gen.go b/internal/resource/obproxy/obproxy_task_gen.go new file mode 100644 index 000000000..58386df3a --- /dev/null +++ b/internal/resource/obproxy/obproxy_task_gen.go @@ -0,0 +1,17 @@ +// Code generated by go generate; DO NOT EDIT. +package obproxy + +func init() { + taskMap.Register(tCopyProxyROSecret, CopyProxyROSecret) + taskMap.Register(tCreateOBProxyConfigMap, CreateOBProxyConfigMap) + taskMap.Register(tCreateOBProxyService, CreateOBProxyService) + taskMap.Register(tCreateOBProxyDeployment, CreateOBProxyDeployment) + taskMap.Register(tWaitOBProxyReady, WaitOBProxyReady) + taskMap.Register(tUpdateOBProxyConfigMap, UpdateOBProxyConfigMap) + taskMap.Register(tUpdateOBProxyDeployment, UpdateOBProxyDeployment) +taskMap.Register(tScaleOBProxyDeployment, ScaleOBProxyDeployment) + taskMap.Register(tDeleteOBProxyDeployment, DeleteOBProxyDeployment) + taskMap.Register(tDeleteOBProxyService, DeleteOBProxyService) + taskMap.Register(tDeleteOBProxyConfigMap, DeleteOBProxyConfigMap) + taskMap.Register(tDeleteOBProxySecrets, DeleteOBProxySecrets) +} diff --git a/internal/resource/obproxy/obproxy_taskname_gen.go b/internal/resource/obproxy/obproxy_taskname_gen.go new file mode 100644 index 000000000..eb78e2fae --- /dev/null +++ b/internal/resource/obproxy/obproxy_taskname_gen.go @@ -0,0 +1,19 @@ +// Code generated by go generate; DO NOT EDIT. +package obproxy + +import ttypes "github.com/oceanbase/ob-operator/pkg/task/types" + +const ( + tCopyProxyROSecret ttypes.TaskName = "copy proxy rosecret" + tCreateOBProxyConfigMap ttypes.TaskName = "create obproxy config map" + tCreateOBProxyService ttypes.TaskName = "create obproxy service" + tCreateOBProxyDeployment ttypes.TaskName = "create obproxy deployment" + tWaitOBProxyReady ttypes.TaskName = "wait obproxy ready" + tUpdateOBProxyConfigMap ttypes.TaskName = "update obproxy config map" + tUpdateOBProxyDeployment ttypes.TaskName = "update obproxy deployment" +tScaleOBProxyDeployment ttypes.TaskName = "scale obproxy deployment" + tDeleteOBProxyDeployment ttypes.TaskName = "delete obproxy deployment" + tDeleteOBProxyService ttypes.TaskName = "delete obproxy service" + tDeleteOBProxyConfigMap ttypes.TaskName = "delete obproxy config map" + tDeleteOBProxySecrets ttypes.TaskName = "delete obproxy secrets" +) diff --git a/internal/resource/obproxy/utils.go b/internal/resource/obproxy/utils.go new file mode 100644 index 000000000..53f7594e5 --- /dev/null +++ b/internal/resource/obproxy/utils.go @@ -0,0 +1,133 @@ +/* +Copyright (c) 2024 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package obproxy + +import ( + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" +) + +const ( + // LabelOBProxyInstance is the label key for OBProxy instance + LabelOBProxyInstance = "obproxy.oceanbase.com/obproxy" + // LabelRefOBCluster is the label key for referencing OBCluster + LabelRefOBCluster = "obproxy.oceanbase.com/obcluster" + // LabelRefOBClusterNamespace is the label key for referencing OBCluster namespace + LabelRefOBClusterNamespace = "obproxy.oceanbase.com/obcluster-namespace" +) + +// getOBProxySelector returns label selector for OBProxy resources +func (m *OBProxyManager) getOBProxySelector() labels.Selector { + return labels.SelectorFromSet(labels.Set{ + LabelOBProxyInstance: m.OBProxy.Name, + }) +} + +// getOBProxyDeployment gets the Deployment owned by OBProxy +func (m *OBProxyManager) getOBProxyDeployment() (*appsv1.Deployment, error) { + deploymentList := &appsv1.DeploymentList{} + err := m.Client.List(m.Ctx, deploymentList, + client.MatchingLabelsSelector{Selector: m.getOBProxySelector()}, + client.InNamespace(m.OBProxy.Namespace), + ) + if err != nil { + return nil, errors.Wrap(err, "list obproxy deployments") + } + + if len(deploymentList.Items) == 0 { + return nil, nil + } + + // Return the first deployment (should only be one) + return &deploymentList.Items[0], nil +} + +// getOBProxyService gets the Service owned by OBProxy +func (m *OBProxyManager) getOBProxyService() (*corev1.Service, error) { + serviceList := &corev1.ServiceList{} + err := m.Client.List(m.Ctx, serviceList, + client.MatchingLabelsSelector{Selector: m.getOBProxySelector()}, + client.InNamespace(m.OBProxy.Namespace), + ) + if err != nil { + return nil, errors.Wrap(err, "list obproxy services") + } + + if len(serviceList.Items) == 0 { + return nil, nil + } + + return &serviceList.Items[0], nil +} + +// getOBProxyConfigMap gets the ConfigMap owned by OBProxy +func (m *OBProxyManager) getOBProxyConfigMap() (*corev1.ConfigMap, error) { + cmList := &corev1.ConfigMapList{} + err := m.Client.List(m.Ctx, cmList, + client.MatchingLabelsSelector{Selector: m.getOBProxySelector()}, + client.InNamespace(m.OBProxy.Namespace), + ) + if err != nil { + return nil, errors.Wrap(err, "list obproxy configmaps") + } + + if len(cmList.Items) == 0 { + return nil, nil + } + + return &cmList.Items[0], nil +} + +// getOBCluster gets the OBCluster referenced by OBProxy +func (m *OBProxyManager) getOBCluster() (*v1alpha1.OBCluster, error) { + clusterNS := m.OBProxy.Spec.OBCluster.Namespace + if clusterNS == "" { + clusterNS = m.OBProxy.Namespace + } + + clusterKey := types.NamespacedName{ + Namespace: clusterNS, + Name: m.OBProxy.Spec.OBCluster.Name, + } + + obcluster := &v1alpha1.OBCluster{} + err := m.Client.Get(m.Ctx, clusterKey, obcluster) + if err != nil { + return nil, errors.Wrap(err, "get obcluster") + } + + return obcluster, nil +} + +// isOBProxyReady checks if OBProxy Deployment is ready +func (m *OBProxyManager) isOBProxyReady(deployment *appsv1.Deployment) bool { + if deployment == nil { + return false + } + + // Check if deployment has replicas + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas == 0 { + return false + } + + // Check if all replicas are ready + return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas && + deployment.Status.AvailableReplicas == *deployment.Spec.Replicas +} + diff --git a/internal/resource/obproxy_test.go b/internal/resource/obproxy_test.go new file mode 100644 index 000000000..34b0ee252 --- /dev/null +++ b/internal/resource/obproxy_test.go @@ -0,0 +1,1530 @@ +/* +Copyright (c) 2024 OceanBase +ob-operator is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + +package resource_test + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + apitypes "github.com/oceanbase/ob-operator/api/types" + v1alpha1 "github.com/oceanbase/ob-operator/api/v1alpha1" + oceanbaseconst "github.com/oceanbase/ob-operator/internal/const/oceanbase" + clusterstatus "github.com/oceanbase/ob-operator/internal/const/status/obcluster" + proxystatus "github.com/oceanbase/ob-operator/internal/const/status/obproxy" + observerstatus "github.com/oceanbase/ob-operator/internal/const/status/observer" + "github.com/oceanbase/ob-operator/internal/resource/obproxy" + "github.com/oceanbase/ob-operator/internal/telemetry" + strategy "github.com/oceanbase/ob-operator/pkg/task/const/strategy" + taskstatus "github.com/oceanbase/ob-operator/pkg/task/const/status" + tasktypes "github.com/oceanbase/ob-operator/pkg/task/types" +) + +func newNopRecorder() telemetry.Recorder { + fakeRecorder := record.NewFakeRecorder(10) + return telemetry.NewRecorder(context.Background(), fakeRecorder) +} + +var _ = Describe("OBProxy Manager", Label("obproxy"), func() { + + var ( + scheme *runtime.Scheme + ctx context.Context + logger logr.Logger + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + Expect(corev1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + logger = logr.Discard() + }) + + Context("Test GetTaskFlow", func() { + It("should return create flow for New status", func() { + obcluster := &v1alpha1.OBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + Spec: v1alpha1.OBClusterSpec{ + ClusterName: "test-cluster", + }, + Status: v1alpha1.OBClusterStatus{ + Status: clusterstatus.Running, + }, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + OBCluster: v1alpha1.OBClusterReference{ + Name: "test-cluster", + Namespace: "default", + }, + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.New, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obcluster, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).ShouldNot(BeNil()) + Expect(string(flow.OperationContext.Name)).Should(Equal("create obproxy")) + }) + + It("should return update flow for Updating status", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{}, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Updating, + }, + } + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).ShouldNot(BeNil()) + Expect(string(flow.OperationContext.Name)).Should(Equal("update obproxy")) + }) + + It("should return scale flow for Scaling status", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Scaling, + }, + } + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).ShouldNot(BeNil()) + Expect(string(flow.OperationContext.Name)).Should(Equal("scale obproxy")) + }) + + It("should return delete flow for Deleting status", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Deleting, + }, + } + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).ShouldNot(BeNil()) + Expect(string(flow.OperationContext.Name)).Should(Equal("delete obproxy")) + }) + + It("should return existing flow when OperationContext is set", func() { + existingContext := &tasktypes.OperationContext{ + Name: "existing operation", + Idx: 2, + Tasks: []tasktypes.TaskName{"CheckOBClusterReady", "CreateProxySysSecret", "CopyProxyROSecret"}, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.New, + OperationContext: existingContext, + }, + } + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).ShouldNot(BeNil()) + Expect(string(flow.OperationContext.Name)).Should(Equal("existing operation")) + Expect(flow.OperationContext.Idx).Should(Equal(2)) + }) + + It("should return nil for Running status", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Running, + }, + } + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).Should(BeNil()) + }) + }) + + Context("Test InitStatus", func() { + It("should initialize status correctly", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + Image: "oceanbase/obproxy:latest", + Replicas: 3, + }, + } + + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + Logger: &logger, + Recorder: newNopRecorder(), + } + m.InitStatus() + + Expect(obproxyCR.Status.Status).Should(Equal(proxystatus.New)) + Expect(obproxyCR.Status.Image).Should(Equal("oceanbase/obproxy:latest")) + Expect(obproxyCR.Status.Replicas).Should(Equal(int32(3))) + }) + }) + + Context("Test FinishTask", func() { + It("should set status to target status and clear operation context", func() { + obproxyCR := &v1alpha1.OBProxy{ + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Updating, + OperationContext: &tasktypes.OperationContext{ + Name: "update obproxy", + TargetStatus: proxystatus.Running, + Idx: 1, + TaskStatus: "running", + Task: "UpdateOBProxyConfigMap", + }, + }, + } + + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + } + m.FinishTask() + + Expect(obproxyCR.Status.Status).Should(Equal(proxystatus.Running)) + Expect(obproxyCR.Status.OperationContext).Should(BeNil()) + }) + }) + + Context("Test ClearTaskInfo", func() { + It("should clear task info and set status to running", func() { + obproxyCR := &v1alpha1.OBProxy{ + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Failed, + OperationContext: &tasktypes.OperationContext{ + Name: "failed operation", + }, + }, + } + + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + } + m.ClearTaskInfo() + + Expect(obproxyCR.Status.Status).Should(Equal(proxystatus.Running)) + Expect(obproxyCR.Status.OperationContext).Should(BeNil()) + }) + }) + + Context("Test GetStatus", func() { + It("should return current status", func() { + obproxyCR := &v1alpha1.OBProxy{ + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Running, + }, + } + + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + } + Expect(m.GetStatus()).Should(Equal(proxystatus.Running)) + }) + }) + + Context("Test GetMeta", func() { + It("should return object meta", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "test-ns", + }, + } + + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + } + meta := m.GetMeta() + Expect(meta.GetName()).Should(Equal("test-obproxy")) + Expect(meta.GetNamespace()).Should(Equal("test-ns")) + }) + }) + + Context("Test SetOperationContext", func() { + It("should set operation context", func() { + obproxyCR := &v1alpha1.OBProxy{} + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + } + + opCtx := &tasktypes.OperationContext{ + Name: "test-operation", + Idx: 0, + Tasks: []tasktypes.TaskName{"Task1", "Task2"}, + } + m.SetOperationContext(opCtx) + + Expect(obproxyCR.Status.OperationContext).ShouldNot(BeNil()) + Expect(string(obproxyCR.Status.OperationContext.Name)).Should(Equal("test-operation")) + }) + }) + + Context("Test GetTaskFunc", func() { + It("should return task function for valid task name", func() { + obproxyCR := &v1alpha1.OBProxy{} + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + } + + taskFunc, err := m.GetTaskFunc(tasktypes.TaskName("create proxy sys secret")) + Expect(err).ShouldNot(HaveOccurred()) + Expect(taskFunc).ShouldNot(BeNil()) + }) + + It("should return error for invalid task name", func() { + obproxyCR := &v1alpha1.OBProxy{} + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + } + + _, err := m.GetTaskFunc(tasktypes.TaskName("InvalidTask")) + Expect(err).Should(HaveOccurred()) + }) + }) +}) + +var _ = Describe("OBProxy cross-namespace secret copy", Label("obproxy"), func() { + + var ( + scheme *runtime.Scheme + ctx context.Context + logger logr.Logger + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + Expect(corev1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + logger = logr.Discard() + }) + + Context("Test CopyProxyROSecret across namespaces", func() { + It("should copy secret from cluster namespace to obproxy namespace", func() { + obcluster := &v1alpha1.OBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "cluster-ns", + }, + Spec: v1alpha1.OBClusterSpec{ + ClusterName: "test-cluster", + UserSecrets: &apitypes.OBUserSecrets{ + ProxyRO: "proxyro-secret", + }, + }, + Status: v1alpha1.OBClusterStatus{ + Status: clusterstatus.Running, + }, + } + + sourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxyro-secret", + Namespace: "cluster-ns", + }, + Data: map[string][]byte{ + "password": []byte("proxyro-password"), + }, + } + + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "obproxy-ns", + UID: types.UID("test-uid"), + }, + Spec: v1alpha1.OBProxySpec{ + OBCluster: v1alpha1.OBClusterReference{ + Name: "test-cluster", + Namespace: "cluster-ns", + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obcluster, sourceSecret, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + + err := obproxy.CopyProxyROSecret(m) + Expect(err).ShouldNot(HaveOccurred()) + + copiedSecret := &corev1.Secret{} + err = fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "obproxy-ns", + Name: "sec-ro-test-obproxy", + }, copiedSecret) + Expect(err).ShouldNot(HaveOccurred()) + Expect(copiedSecret.Data["password"]).Should(Equal([]byte("proxyro-password"))) + }) + + It("should use obproxy namespace when cluster namespace is empty", func() { + obcluster := &v1alpha1.OBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + Spec: v1alpha1.OBClusterSpec{ + ClusterName: "test-cluster", + UserSecrets: &apitypes.OBUserSecrets{ + ProxyRO: "proxyro-secret", + }, + }, + Status: v1alpha1.OBClusterStatus{ + Status: clusterstatus.Running, + }, + } + + sourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxyro-secret", + Namespace: "default", + }, + Data: map[string][]byte{ + "password": []byte("proxyro-password"), + }, + } + + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + UID: types.UID("test-uid"), + }, + Spec: v1alpha1.OBProxySpec{ + OBCluster: v1alpha1.OBClusterReference{ + Name: "test-cluster", + Namespace: "", + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obcluster, sourceSecret, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + + err := obproxy.CopyProxyROSecret(m) + Expect(err).ShouldNot(HaveOccurred()) + + copiedSecret := &corev1.Secret{} + err = fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", + Name: "sec-ro-test-obproxy", + }, copiedSecret) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("should return error when obcluster does not have proxyRO secret configured", func() { + obcluster := &v1alpha1.OBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "cluster-ns", + }, + Spec: v1alpha1.OBClusterSpec{ + ClusterName: "test-cluster", + UserSecrets: &apitypes.OBUserSecrets{ + ProxyRO: "", + }, + }, + Status: v1alpha1.OBClusterStatus{ + Status: clusterstatus.Running, + }, + } + + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "obproxy-ns", + }, + Spec: v1alpha1.OBProxySpec{ + OBCluster: v1alpha1.OBClusterReference{ + Name: "test-cluster", + Namespace: "cluster-ns", + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obcluster, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + + err := obproxy.CopyProxyROSecret(m) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("does not have proxyRO secret configured")) + }) + }) +}) + +var _ = Describe("OBProxy RS_LIST calculation", Label("obproxy"), func() { + + var ( + scheme *runtime.Scheme + ctx context.Context + logger logr.Logger + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + Expect(corev1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + logger = logr.Discard() + }) + + Context("Test RS_LIST with multiple observers", func() { + It("should include all running observers in RS_LIST", func() { + obcluster := &v1alpha1.OBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + Spec: v1alpha1.OBClusterSpec{ + ClusterName: "test-cluster", + }, + Status: v1alpha1.OBClusterStatus{ + Status: clusterstatus.Running, + }, + } + + observers := make([]runtime.Object, 3) + for i := 0; i < 3; i++ { + observers[i] = &v1alpha1.OBServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("observer-%d", i+1), + Namespace: "default", + Labels: map[string]string{ + oceanbaseconst.LabelRefOBCluster: "test-cluster", + }, + }, + Status: v1alpha1.OBServerStatus{ + Status: observerstatus.Running, + PodIp: fmt.Sprintf("192.168.1.%d", i+1), + }, + } + } + + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + OBCluster: v1alpha1.OBClusterReference{ + Name: "test-cluster", + Namespace: "default", + }, + }, + } + + objs := append([]runtime.Object{obcluster, obproxyCR}, observers...) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objs...).Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + + err := obproxy.CreateOBProxyConfigMap(m) + _ = err + }) + + It("should only include running observers in RS_LIST", func() { + obcluster := &v1alpha1.OBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + Spec: v1alpha1.OBClusterSpec{ + ClusterName: "test-cluster", + }, + Status: v1alpha1.OBClusterStatus{ + Status: clusterstatus.Running, + }, + } + + runningObserver1 := &v1alpha1.OBServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "observer-1", + Namespace: "default", + Labels: map[string]string{ + oceanbaseconst.LabelRefOBCluster: "test-cluster", + }, + }, + Status: v1alpha1.OBServerStatus{ + Status: observerstatus.Running, + PodIp: "192.168.1.1", + }, + } + + runningObserver2 := &v1alpha1.OBServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "observer-2", + Namespace: "default", + Labels: map[string]string{ + oceanbaseconst.LabelRefOBCluster: "test-cluster", + }, + }, + Status: v1alpha1.OBServerStatus{ + Status: observerstatus.Running, + PodIp: "192.168.1.2", + }, + } + + notRunningObserver := &v1alpha1.OBServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "observer-3", + Namespace: "default", + Labels: map[string]string{ + oceanbaseconst.LabelRefOBCluster: "test-cluster", + }, + }, + Status: v1alpha1.OBServerStatus{ + Status: "NotRunning", + PodIp: "192.168.1.3", + }, + } + + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + OBCluster: v1alpha1.OBClusterReference{ + Name: "test-cluster", + Namespace: "default", + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obcluster, obproxyCR, runningObserver1, runningObserver2, notRunningObserver). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + + err := obproxy.CreateOBProxyConfigMap(m) + _ = err + }) + }) +}) + +var _ = Describe("OBProxy GetTaskFlow edge cases", Label("obproxy"), func() { + var ( + scheme *runtime.Scheme + ctx context.Context + logger logr.Logger + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + Expect(corev1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + logger = logr.Discard() + }) + + It("should return nil when OBCluster is not found for New status", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + OBCluster: v1alpha1.OBClusterReference{ + Name: "missing-cluster", + Namespace: "default", + }, + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.New, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).Should(BeNil()) + }) + + It("should return nil when OBCluster is not Running for New status", func() { + obcluster := &v1alpha1.OBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + Status: v1alpha1.OBClusterStatus{ + Status: "Bootstrapping", + }, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + OBCluster: v1alpha1.OBClusterReference{ + Name: "test-cluster", + Namespace: "default", + }, + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.New, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obcluster, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).Should(BeNil()) + }) + + It("should return nil for unknown status", func() { + obproxyCR := &v1alpha1.OBProxy{ + Status: v1alpha1.OBProxyStatus{ + Status: "SomeUnknownStatus", + }, + } + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Logger: &logger, + } + flow, err := m.GetTaskFlow() + Expect(err).ShouldNot(HaveOccurred()) + Expect(flow).Should(BeNil()) + }) +}) + +var _ = Describe("OBProxy HandleFailure", Label("obproxy"), func() { + It("StartOver with different status clears context and sets new status", func() { + obproxyCR := &v1alpha1.OBProxy{ + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Failed, + OperationContext: &tasktypes.OperationContext{ + Name: "create obproxy", + Idx: 2, + TaskStatus: taskstatus.Failed, + OnFailure: tasktypes.FailureRule{ + Strategy: strategy.StartOver, + NextTryStatus: proxystatus.New, + }, + }, + }, + } + m := &obproxy.OBProxyManager{OBProxy: obproxyCR} + m.HandleFailure() + + Expect(obproxyCR.Status.Status).Should(Equal(proxystatus.New)) + Expect(obproxyCR.Status.OperationContext).Should(BeNil()) + }) + + It("StartOver with same status resets task progress without clearing context", func() { + obproxyCR := &v1alpha1.OBProxy{ + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.New, + OperationContext: &tasktypes.OperationContext{ + Name: "create obproxy", + Idx: 3, + TaskStatus: taskstatus.Failed, + Task: "CreateOBProxyDeployment", + TaskId: "some-task-id", + OnFailure: tasktypes.FailureRule{ + Strategy: strategy.StartOver, + NextTryStatus: proxystatus.New, + }, + }, + }, + } + m := &obproxy.OBProxyManager{OBProxy: obproxyCR} + m.HandleFailure() + + Expect(obproxyCR.Status.Status).Should(Equal(proxystatus.New)) + Expect(obproxyCR.Status.OperationContext).ShouldNot(BeNil()) + Expect(obproxyCR.Status.OperationContext.Idx).Should(Equal(0)) + Expect(string(obproxyCR.Status.OperationContext.TaskStatus)).Should(BeEmpty()) + Expect(string(obproxyCR.Status.OperationContext.TaskId)).Should(BeEmpty()) + Expect(string(obproxyCR.Status.OperationContext.Task)).Should(BeEmpty()) + }) + + It("RetryFromCurrent sets task status to Pending", func() { + obproxyCR := &v1alpha1.OBProxy{ + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Updating, + OperationContext: &tasktypes.OperationContext{ + Name: "update obproxy", + Idx: 1, + TaskStatus: taskstatus.Failed, + OnFailure: tasktypes.FailureRule{ + Strategy: strategy.RetryFromCurrent, + }, + }, + }, + } + m := &obproxy.OBProxyManager{OBProxy: obproxyCR} + m.HandleFailure() + + Expect(obproxyCR.Status.Status).Should(Equal(proxystatus.Updating)) + Expect(obproxyCR.Status.OperationContext).ShouldNot(BeNil()) + Expect(string(obproxyCR.Status.OperationContext.TaskStatus)).Should(Equal(taskstatus.Pending)) + }) + + It("Pause leaves status and context unchanged", func() { + obproxyCR := &v1alpha1.OBProxy{ + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Failed, + OperationContext: &tasktypes.OperationContext{ + Name: "some operation", + Idx: 2, + TaskStatus: taskstatus.Failed, + OnFailure: tasktypes.FailureRule{ + Strategy: strategy.Pause, + }, + }, + }, + } + m := &obproxy.OBProxyManager{OBProxy: obproxyCR} + m.HandleFailure() + + Expect(obproxyCR.Status.Status).Should(Equal(proxystatus.Failed)) + Expect(obproxyCR.Status.OperationContext).ShouldNot(BeNil()) + Expect(obproxyCR.Status.OperationContext.Idx).Should(Equal(2)) + Expect(string(obproxyCR.Status.OperationContext.TaskStatus)).Should(Equal(taskstatus.Failed)) + }) +}) + +var _ = Describe("OBProxy ArchiveResource", Label("obproxy"), func() { + It("sets status to Failed and clears OperationContext", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Updating, + OperationContext: &tasktypes.OperationContext{ + Name: "some operation", + }, + }, + } + logger := logr.Discard() + m := &obproxy.OBProxyManager{ + OBProxy: obproxyCR, + Logger: &logger, + Recorder: newNopRecorder(), + } + m.ArchiveResource() + + Expect(obproxyCR.Status.Status).Should(Equal(proxystatus.Failed)) + Expect(obproxyCR.Status.OperationContext).Should(BeNil()) + }) +}) + +var _ = Describe("OBProxy CheckAndUpdateFinalizers", Label("obproxy"), func() { + var ( + scheme *runtime.Scheme + ctx context.Context + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + }) + + It("clears finalizers and updates when status is FinalizerFinished", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + Finalizers: []string{"oceanbase.com/finalizer"}, + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.FinalizerFinished, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + } + Expect(m.CheckAndUpdateFinalizers()).Should(Succeed()) + Expect(obproxyCR.ObjectMeta.Finalizers).Should(BeEmpty()) + }) + + It("does nothing for non-FinalizerFinished status", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + Finalizers: []string{"oceanbase.com/finalizer"}, + }, + Status: v1alpha1.OBProxyStatus{ + Status: proxystatus.Running, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + } + Expect(m.CheckAndUpdateFinalizers()).Should(Succeed()) + Expect(obproxyCR.ObjectMeta.Finalizers).Should(HaveLen(1)) + }) +}) + +var _ = Describe("OBProxy CreateOBProxyConfigMap task", Label("obproxy"), func() { + var ( + scheme *runtime.Scheme + ctx context.Context + logger logr.Logger + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + Expect(corev1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + logger = logr.Discard() + }) + + It("creates ConfigMap with uppercased ODP_ parameter keys", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + Parameters: []apitypes.Parameter{ + {Name: "max_connections", Value: "100"}, + {Name: "enable_transaction_internal_routing", Value: "true"}, + }, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + Expect(obproxy.CreateOBProxyConfigMap(m)).Should(Succeed()) + + cm := &corev1.ConfigMap{} + Expect(fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", + Name: "cm-test-obproxy", + }, cm)).Should(Succeed()) + Expect(cm.Data["ODP_MAX_CONNECTIONS"]).Should(Equal("100")) + Expect(cm.Data["ODP_ENABLE_TRANSACTION_INTERNAL_ROUTING"]).Should(Equal("true")) + }) + + It("skips creation and preserves existing ConfigMap", func() { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-test-obproxy", + Namespace: "default", + }, + Data: map[string]string{"ODP_MAX_CONNECTIONS": "50"}, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + Parameters: []apitypes.Parameter{ + {Name: "max_connections", Value: "100"}, + }, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(existingCM, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + Expect(obproxy.CreateOBProxyConfigMap(m)).Should(Succeed()) + + cm := &corev1.ConfigMap{} + Expect(fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", + Name: "cm-test-obproxy", + }, cm)).Should(Succeed()) + Expect(cm.Data["ODP_MAX_CONNECTIONS"]).Should(Equal("50")) + }) +}) + +var _ = Describe("OBProxy CreateOBProxyService task", Label("obproxy"), func() { + var ( + scheme *runtime.Scheme + ctx context.Context + logger logr.Logger + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + Expect(corev1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + logger = logr.Discard() + }) + + It("creates Service with correct ports and default ClusterIP type", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + Expect(obproxy.CreateOBProxyService(m)).Should(Succeed()) + + svc := &corev1.Service{} + Expect(fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", + Name: "svc-test-obproxy", + }, svc)).Should(Succeed()) + Expect(svc.Spec.Type).Should(Equal(corev1.ServiceTypeClusterIP)) + + portMap := map[string]int32{} + for _, p := range svc.Spec.Ports { + portMap[p.Name] = p.Port + } + Expect(portMap["sql"]).Should(Equal(int32(2883))) + Expect(portMap["prometheus"]).Should(Equal(int32(2884))) + }) + + It("creates Service with LoadBalancer type when specified", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + ServiceType: "LoadBalancer", + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + Expect(obproxy.CreateOBProxyService(m)).Should(Succeed()) + + svc := &corev1.Service{} + Expect(fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", + Name: "svc-test-obproxy", + }, svc)).Should(Succeed()) + Expect(svc.Spec.Type).Should(Equal(corev1.ServiceTypeLoadBalancer)) + }) + + It("skips creation and preserves existing Service", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-test-obproxy", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeNodePort}, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(existingSvc, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Recorder: newNopRecorder(), + Logger: &logger, + } + Expect(obproxy.CreateOBProxyService(m)).Should(Succeed()) + + svc := &corev1.Service{} + Expect(fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", + Name: "svc-test-obproxy", + }, svc)).Should(Succeed()) + Expect(svc.Spec.Type).Should(Equal(corev1.ServiceTypeNodePort)) + }) +}) + +var _ = Describe("OBProxy UpdateOBProxyConfigMap task", Label("obproxy"), func() { + var ( + scheme *runtime.Scheme + ctx context.Context + logger logr.Logger + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + Expect(corev1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + logger = logr.Discard() + }) + + It("updates existing ConfigMap with new parameters and removes stale keys", func() { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-test-obproxy", + Namespace: "default", + Labels: map[string]string{ + obproxy.LabelOBProxyInstance: "test-obproxy", + }, + }, + Data: map[string]string{"ODP_OLD_PARAM": "old-value"}, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + Parameters: []apitypes.Parameter{ + {Name: "max_connections", Value: "200"}, + }, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(existingCM, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Logger: &logger, + } + Expect(obproxy.UpdateOBProxyConfigMap(m)).Should(Succeed()) + + cm := &corev1.ConfigMap{} + Expect(fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", + Name: "cm-test-obproxy", + }, cm)).Should(Succeed()) + Expect(cm.Data["ODP_MAX_CONNECTIONS"]).Should(Equal("200")) + Expect(cm.Data).ShouldNot(HaveKey("ODP_OLD_PARAM")) + }) + + It("creates ConfigMap when it does not exist", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + }, + Spec: v1alpha1.OBProxySpec{ + Parameters: []apitypes.Parameter{ + {Name: "max_connections", Value: "50"}, + }, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, + OBProxy: obproxyCR, + Client: fakeClient, + Logger: &logger, + } + Expect(obproxy.UpdateOBProxyConfigMap(m)).Should(Succeed()) + + cm := &corev1.ConfigMap{} + Expect(fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", + Name: "cm-test-obproxy", + }, cm)).Should(Succeed()) + Expect(cm.Data["ODP_MAX_CONNECTIONS"]).Should(Equal("50")) + }) +}) + +var _ = Describe("OBProxy delete tasks", Label("obproxy"), func() { + var ( + scheme *runtime.Scheme + ctx context.Context + logger logr.Logger + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1alpha1.AddToScheme(scheme)).Should(Succeed()) + Expect(corev1.AddToScheme(scheme)).Should(Succeed()) + Expect(appsv1.AddToScheme(scheme)).Should(Succeed()) + ctx = context.Background() + logger = logr.Discard() + }) + + Context("DeleteOBProxyDeployment", func() { + It("deletes an existing deployment", func() { + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-obproxy", + Namespace: "default", + Labels: map[string]string{ + obproxy.LabelOBProxyInstance: "test-obproxy", + }, + }, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-obproxy", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(deploy, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, OBProxy: obproxyCR, Client: fakeClient, + Recorder: newNopRecorder(), Logger: &logger, + } + Expect(obproxy.DeleteOBProxyDeployment(m)).Should(Succeed()) + + list := &appsv1.DeploymentList{} + Expect(fakeClient.List(ctx, list)).Should(Succeed()) + Expect(list.Items).Should(BeEmpty()) + }) + + It("succeeds when deployment does not exist", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-obproxy", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme).WithRuntimeObjects(obproxyCR).Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, OBProxy: obproxyCR, Client: fakeClient, + Recorder: newNopRecorder(), Logger: &logger, + } + Expect(obproxy.DeleteOBProxyDeployment(m)).Should(Succeed()) + }) + }) + + Context("DeleteOBProxyService", func() { + It("deletes an existing service", func() { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-test-obproxy", + Namespace: "default", + Labels: map[string]string{ + obproxy.LabelOBProxyInstance: "test-obproxy", + }, + }, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-obproxy", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(svc, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, OBProxy: obproxyCR, Client: fakeClient, + Recorder: newNopRecorder(), Logger: &logger, + } + Expect(obproxy.DeleteOBProxyService(m)).Should(Succeed()) + + list := &corev1.ServiceList{} + Expect(fakeClient.List(ctx, list)).Should(Succeed()) + Expect(list.Items).Should(BeEmpty()) + }) + + It("succeeds when service does not exist", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-obproxy", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme).WithRuntimeObjects(obproxyCR).Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, OBProxy: obproxyCR, Client: fakeClient, + Recorder: newNopRecorder(), Logger: &logger, + } + Expect(obproxy.DeleteOBProxyService(m)).Should(Succeed()) + }) + }) + + Context("DeleteOBProxyConfigMap", func() { + It("deletes an existing configmap", func() { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-test-obproxy", + Namespace: "default", + Labels: map[string]string{ + obproxy.LabelOBProxyInstance: "test-obproxy", + }, + }, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-obproxy", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(cm, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, OBProxy: obproxyCR, Client: fakeClient, + Recorder: newNopRecorder(), Logger: &logger, + } + Expect(obproxy.DeleteOBProxyConfigMap(m)).Should(Succeed()) + + list := &corev1.ConfigMapList{} + Expect(fakeClient.List(ctx, list)).Should(Succeed()) + Expect(list.Items).Should(BeEmpty()) + }) + + It("succeeds when configmap does not exist", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-obproxy", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme).WithRuntimeObjects(obproxyCR).Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, OBProxy: obproxyCR, Client: fakeClient, + Recorder: newNopRecorder(), Logger: &logger, + } + Expect(obproxy.DeleteOBProxyConfigMap(m)).Should(Succeed()) + }) + }) + + Context("DeleteOBProxySecrets", func() { + It("deletes the proxyRO secret", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sec-ro-test-obproxy", + Namespace: "default", + }, + } + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-obproxy", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(secret, obproxyCR). + Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, OBProxy: obproxyCR, Client: fakeClient, + Recorder: newNopRecorder(), Logger: &logger, + } + Expect(obproxy.DeleteOBProxySecrets(m)).Should(Succeed()) + + deleted := &corev1.Secret{} + err := fakeClient.Get(ctx, types.NamespacedName{ + Namespace: "default", Name: "sec-ro-test-obproxy", + }, deleted) + Expect(err).Should(HaveOccurred()) + }) + + It("succeeds when proxyRO secret does not exist", func() { + obproxyCR := &v1alpha1.OBProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-obproxy", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme).WithRuntimeObjects(obproxyCR).Build() + + m := &obproxy.OBProxyManager{ + Ctx: ctx, OBProxy: obproxyCR, Client: fakeClient, + Recorder: newNopRecorder(), Logger: &logger, + } + Expect(obproxy.DeleteOBProxySecrets(m)).Should(Succeed()) + }) + }) +}) diff --git a/tests/case_p8_OBProxy/check_P8_obproxy01_create.sh b/tests/case_p8_OBProxy/check_P8_obproxy01_create.sh new file mode 100755 index 000000000..420599dca --- /dev/null +++ b/tests/case_p8_OBProxy/check_P8_obproxy01_create.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +TESTS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +source "$TESTS_DIR/setup.sh" +source "$TESTS_DIR/util.sh" +source "$TESTS_DIR/env.sh" + +prepare() { + export PASSWORD=$(generate_random_str) + export SUFFIX=$(generate_random_str | tr '[:upper:]' '[:lower:]') + export NAMESPACE="oceanbase-${SUFFIX}" + export OBCLUSTER_NAME="test${SUFFIX}" + export OB_ROOT_SECRET="sc-root-${SUFFIX}" + export OBPROXY_IMAGE=${OBPROXY_IMAGE:-oceanbase/obproxy-ce:4.3.3.0-5} + export OBPROXY_NAME="obproxy-${SUFFIX}" + export PROXY_SYS_SECRET="sc-proxyro-${SUFFIX}" + export OBPROXY_REPLICAS=2 + + kubectl create namespace "$NAMESPACE" + create_pass_secret "$NAMESPACE" "$OB_ROOT_SECRET" "$PASSWORD" + create_pass_secret "$NAMESPACE" "$PROXY_SYS_SECRET" "$PASSWORD" + echo "Prepared environment: SUFFIX=$SUFFIX NAMESPACE=$NAMESPACE" +} + +create_obcluster() { + echo "Creating OBCluster: $OBCLUSTER_NAME in $NAMESPACE" + envsubst < "$TESTS_DIR/config/clusterManage/obcluster_template_1-1-1.yaml" | kubectl apply -f - +} + +wait_obcluster_running() { + local counter=0 + local timeout=200 + echo "Waiting for OBCluster to reach running status..." + while true; do + counter=$((counter + 1)) + local status + status=$(kubectl get obcluster "$OBCLUSTER_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.status}' 2>/dev/null) + local ready_pods + ready_pods=$(kubectl get pod -n "$NAMESPACE" --no-headers 2>/dev/null \ + | awk '$2=="1/1" && $3=="Running"' | wc -l | tr -d ' ') + echo " ($counter/$timeout) status=$status ready_pods=$ready_pods" + if [[ "$status" == "running" && "$ready_pods" -ge 3 ]]; then + echo "OBCluster is running." + return 0 + fi + if [[ $counter -ge $timeout ]]; then + echo "ERROR: OBCluster not running after timeout" + kubectl describe obcluster "$OBCLUSTER_NAME" -n "$NAMESPACE" | tail -20 + return 1 + fi + sleep 5 + done +} + +create_obproxy() { + echo "Creating OBProxy: $OBPROXY_NAME" + envsubst < "$TESTS_DIR/config/obproxyManage/obproxy_template.yaml" | kubectl apply -f - +} + +wait_obproxy_running() { + local counter=0 + local timeout=100 + echo "Waiting for OBProxy to be ready..." + while true; do + counter=$((counter + 1)) + local status ready desired + status=$(kubectl get obproxy "$OBPROXY_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.status}' 2>/dev/null) + ready=$(kubectl get obproxy "$OBPROXY_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.readyReplicas}' 2>/dev/null) + desired=$(kubectl get obproxy "$OBPROXY_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.replicas}' 2>/dev/null) + echo " ($counter/$timeout) status=$status ready=$ready/$desired" + if [[ "$status" == "running" && -n "$ready" && "$ready" == "$desired" ]]; then + echo "OBProxy is running: $ready/$desired ready" + return 0 + fi + if [[ $counter -ge $timeout ]]; then + echo "ERROR: OBProxy not running after timeout" + kubectl describe obproxy "$OBPROXY_NAME" -n "$NAMESPACE" | tail -20 + return 1 + fi + sleep 3 + done +} + +check_rs_list() { + echo "=== Checking RS_LIST configuration ===" + local obproxy_label="obproxy.oceanbase.com/obproxy=$OBPROXY_NAME" + local pod + pod=$(kubectl get pod -n "$NAMESPACE" -l "$obproxy_label" -o name 2>/dev/null | head -1) + if [[ -n "$pod" ]]; then + echo "Checking RS_LIST in $pod:" + kubectl exec -n "$NAMESPACE" "$pod" -- env 2>/dev/null \ + | grep -E "^RS_LIST=" || echo " RS_LIST env not found" + else + echo "No OBProxy pod found" + fi +} + +check_connectivity() { + echo "=== Checking OBProxy connectivity ===" + local obproxy_label="obproxy.oceanbase.com/obproxy=$OBPROXY_NAME" + local svc_ip + svc_ip=$(kubectl get svc -n "$NAMESPACE" -l "$obproxy_label" \ + -o jsonpath='{.items[0].spec.clusterIP}' 2>/dev/null) + if [[ -n "$svc_ip" ]]; then + echo "OBProxy Service IP: $svc_ip" + local proxy_sys_password + proxy_sys_password=$(kubectl get secret "$PROXY_SYS_SECRET" -n "$NAMESPACE" \ + -o jsonpath='{.data.password}' | base64 -d) + mysql -h "$svc_ip" -P 2883 -uroot@proxysys -p"$proxy_sys_password" \ + -e "SHOW PROXYCONFIG;" 2>&1 | head -5 \ + || echo "Connection test completed (mysql client may not be available)" + else + echo "OBProxy service not found" + fi +} + +export_to_file() { + local output_file="$SCRIPT_DIR/env_vars.sh" + cat < "$output_file" +export PASSWORD="$PASSWORD" +export SUFFIX="$SUFFIX" +export NAMESPACE="$NAMESPACE" +export OBCLUSTER_NAME="$OBCLUSTER_NAME" +export OB_ROOT_SECRET="$OB_ROOT_SECRET" +export OBPROXY_NAME="$OBPROXY_NAME" +export OBPROXY_IMAGE="$OBPROXY_IMAGE" +export PROXY_SYS_SECRET="$PROXY_SYS_SECRET" +export OBPROXY_REPLICAS="$OBPROXY_REPLICAS" +EOF + echo "Environment exported to $output_file" +} + +cleanup() { + echo "Cleaning up namespace $NAMESPACE..." + kubectl delete namespace "$NAMESPACE" --ignore-not-found=true +} + +echo "=== OBProxy P8 Environment Setup ===" + +prepare +export_to_file + +create_obcluster +if ! wait_obcluster_running; then + echo "FAILED: OBCluster did not reach running state" + exit 1 +fi + +create_obproxy +if ! wait_obproxy_running; then + echo "FAILED: OBProxy did not reach running state" + exit 1 +fi + +check_rs_list +check_connectivity + +echo "" +echo "=== Environment ready ===" +echo "NAMESPACE: $NAMESPACE" +echo "OBCLUSTER_NAME: $OBCLUSTER_NAME" +echo "OBPROXY_NAME: $OBPROXY_NAME" +echo "case passed" diff --git a/tests/case_p8_OBProxy/check_P8_obproxy02_rs_change.sh b/tests/case_p8_OBProxy/check_P8_obproxy02_rs_change.sh new file mode 100755 index 000000000..a2b05462c --- /dev/null +++ b/tests/case_p8_OBProxy/check_P8_obproxy02_rs_change.sh @@ -0,0 +1,455 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +TESTS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +source "$TESTS_DIR/setup.sh" +source "$TESTS_DIR/util.sh" +source "$TESTS_DIR/env.sh" +source "$TESTS_DIR/case_p8_OBProxy/env_vars.sh" + +# Define labels after sourcing env files to avoid being overwritten +OBPROXY_LABEL="obproxy.oceanbase.com/obproxy" +REF_CLUSTER_LABEL="ref-obcluster" + +get_obproxy_deployment_name() { + kubectl get deployment -n "$NAMESPACE" -l "${OBPROXY_LABEL}=${OBPROXY_NAME}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null +} + +get_rs_list() { + local pod + pod=$(kubectl get pod -n "$NAMESPACE" -l "${OBPROXY_LABEL}=${OBPROXY_NAME}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + if [[ -n "$pod" ]]; then + kubectl exec -n "$NAMESPACE" "pod/$pod" -- env 2>/dev/null | grep "^RS_LIST=" | cut -d'=' -f2- + fi +} + +verify_all_obproxy_pods_rs_list() { + local expected="$1" + local ok=0 + while read -r pod; do + [[ -z "$pod" ]] && continue + local phase + phase=$(kubectl get pod -n "$NAMESPACE" "$pod" -o jsonpath='{.status.phase}' 2>/dev/null) + [[ "$phase" != "Running" ]] && continue + local v + v=$(kubectl exec -n "$NAMESPACE" "pod/$pod" -- env 2>/dev/null | grep "^RS_LIST=" | cut -d'=' -f2-) + if [[ "$v" != "$expected" ]]; then + echo " RS_LIST mismatch on pod $pod" + ok=1 + fi + done < <(kubectl get pod -n "$NAMESPACE" -l "${OBPROXY_LABEL}=${OBPROXY_NAME}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null) + return $ok +} + +get_deployment_generation() { + kubectl get deployment -n "$NAMESPACE" -l "${OBPROXY_LABEL}=${OBPROXY_NAME}" -o jsonpath='{.items[0].metadata.generation}' 2>/dev/null +} + +# Debug: Get OBProxy status +get_obproxy_status() { + kubectl get obproxy -n "$NAMESPACE" "$OBPROXY_NAME" -o jsonpath='{.status.status}{"\t"}{.status.operationContext}' 2>/dev/null +} + +# Debug: Get controller logs for obproxy +get_obproxy_controller_logs() { + local since="${1:-2m}" + kubectl logs -n oceanbase-system -l control-plane=controller-manager -c manager --since="$since" 2>/dev/null | grep -i "obproxy" | tail -20 +} + +MONITOR_PID="" +MONITOR_LOG="" + +# Sample OBProxy conditions and Deployment generation every 2s into a temp file. +# Run in background; call stop_condition_monitor to terminate. +start_condition_monitor() { + MONITOR_LOG=$(mktemp /tmp/obproxy_monitor.XXXXXX) + ( + while true; do + local ts gen rs_avail cluster_ready + ts=$(date +%T) + gen=$(kubectl get deployment -n "$NAMESPACE" -l "${OBPROXY_LABEL}=${OBPROXY_NAME}" \ + -o jsonpath='{.items[0].metadata.generation}' 2>/dev/null) + rs_avail=$(kubectl get obproxy "$OBPROXY_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.conditions[?(@.type=="RSListAvailable")].reason}' 2>/dev/null) + cluster_ready=$(kubectl get obproxy "$OBPROXY_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.conditions[?(@.type=="OBClusterReady")].reason}' 2>/dev/null) + echo "$ts gen=$gen RSListAvail=$rs_avail ClusterReady=$cluster_ready" >> "$MONITOR_LOG" + sleep 2 + done + ) & + MONITOR_PID=$! +} + +stop_condition_monitor() { + if [[ -n "${MONITOR_PID:-}" ]]; then + kill "$MONITOR_PID" 2>/dev/null + wait "$MONITOR_PID" 2>/dev/null + MONITOR_PID="" + fi +} + +# Assert debounce correctness from the monitor log. +# $1: gen_before $2: label +analyze_debounce() { + local gen_before="$1" label="$2" + echo "" + echo "=== Debounce Analysis: $label ===" + if [[ ! -f "$MONITOR_LOG" ]]; then + echo " No monitor data." + return + fi + + echo "Condition timeline (sampled every 2s):" + cat "$MONITOR_LOG" + echo "" + + # Guard trigger: was OBClusterNotStable seen at least once? + local not_stable_count + not_stable_count=$(grep -c "OBClusterNotStable" "$MONITOR_LOG" 2>/dev/null || echo 0) + if [[ "$not_stable_count" -gt 0 ]]; then + echo "PASS (guard): OBClusterNotStable observed ${not_stable_count}x — controller skipped drift check while OBCluster was operating" + else + echo "NOTE (guard): OBClusterNotStable not observed — OBCluster may have finished before the first drift check (timing window missed)" + fi + + # Single-restart: generation must advance by exactly 1 + local max_gen + max_gen=$(grep -o "gen=[0-9]*" "$MONITOR_LOG" | sed 's/gen=//' | sort -n | tail -1) + local expected_gen=$(( gen_before + 1 )) + echo "" + if [[ -n "$max_gen" && "$max_gen" -le "$expected_gen" ]]; then + echo "PASS (single-restart): generation ${gen_before} → ${max_gen} — no spurious extra restarts" + else + echo "FAIL (single-restart): generation ${gen_before} → ${max_gen:-?} — expected at most ${expected_gen}" + fi + + rm -f "$MONITOR_LOG" + MONITOR_LOG="" +} + +count_observer_crs() { + # Only count observer CRs that are not in deleting state + # Use grep -v to filter out observers with "deleting" status + sudo kubectl get observer -n "$NAMESPACE" -l "${REF_CLUSTER_LABEL}=${OBCLUSTER_NAME}" --no-headers 2>/dev/null | grep -v "deleting" | wc -l | tr -d ' ' +} + +count_observer_crs_running() { + sudo kubectl get observer -n "$NAMESPACE" -l "${REF_CLUSTER_LABEL}=${OBCLUSTER_NAME}" \ + -o jsonpath='{.items[*].status.status}' 2>/dev/null | grep -o running | wc -l | tr -d ' ' +} + +get_observer_count() { + local ip + ip=$(kubectl get pod -o wide -n "$NAMESPACE" | grep "${OBCLUSTER_NAME}-1-zone1" | awk '{print $6}' | head -1) + if [[ -n "$ip" ]]; then + mysql -uroot -h "$ip" -P 2881 -Doceanbase -p"$PASSWORD" -e 'select count(*) from __all_server where status="active";' -N 2>/dev/null + else + echo "0" + fi +} + +wait_for_no_deleting_observers() { + local timeout="${1:-180}" + local counter=0 + local max_counter=$((timeout / 5)) + echo "Waiting for all deleting observers to be cleaned up..." + while true; do + counter=$((counter + 1)) + local deleting_count + # Count observers with "deleting" status + deleting_count=$(sudo kubectl get observer -n "$NAMESPACE" -l "${REF_CLUSTER_LABEL}=${OBCLUSTER_NAME}" --no-headers 2>/dev/null | grep "deleting" | wc -l | tr -d ' ') + + if [[ "$deleting_count" -eq 0 ]]; then + echo "No deleting observers found." + return 0 + fi + + echo " Still have $deleting_count observers in deleting state... ($counter/$max_counter)" + + if [[ $counter -ge $max_counter ]]; then + echo "WARNING: Timeout waiting for deleting observers to clean up" + return 1 + fi + sleep 5 + done +} + +get_current_observer_count() { + count_observer_crs +} + +reset_cluster_to_1_1_1() { + local current_count + current_count=$(get_current_observer_count) + + echo "Current observer count: $current_count" + + if [[ "$current_count" -eq 3 ]]; then + echo "Cluster is already in 1-1-1 state (3 observers)." + return 0 + fi + + echo "Cluster is in $current_count observers state, resetting to 1-1-1..." + + # Apply 1-1-1 configuration + envsubst < "$TESTS_DIR/config/clusterManage/obcluster_template_1-1-1.yaml" | kubectl apply -f - + + # Wait for scale in to complete + echo "Waiting for cluster to scale in to 1-1-1..." + wait_observer_targets 3 3 300 || { + echo "ERROR: Failed to scale in to 1-1-1" + return 1 + } + + # Wait for deleting observers to be cleaned up + wait_for_no_deleting_observers 180 || { + echo "WARNING: Some observers still in deleting state after reset" + } + + echo "Cluster successfully reset to 1-1-1 state." + return 0 +} + +wait_observer_targets() { + local want_total="$1" + local want_db="$2" + local timeout="${3:-180}" + local counter=0 + echo "Waiting for OBServer CRs total=$want_total running=$want_total (DB active=$want_db when reachable)..." + while true; do + counter=$((counter + 1)) + local total running db + total=$(count_observer_crs) + running=$(count_observer_crs_running) + db=$(get_observer_count) + echo " CR total=$total running=$running (want $want_total); DB active=$db (want $want_db)" + if [[ "$total" == "$want_total" && "$running" == "$want_total" && "$db" == "$want_db" ]]; then + echo "Observer targets satisfied." + return 0 + fi + if [[ $counter -ge $timeout ]]; then + echo "Timeout waiting for observer targets" + return 1 + fi + sleep 5 + done +} + +wait_obproxy_rollout() { + local deploy="$1" + local timeout="${2:-300s}" + if [[ -z "$deploy" ]]; then + echo "ERROR: OBProxy Deployment not found" + return 1 + fi + echo "Waiting for Deployment rollout: $deploy" + kubectl rollout status "deployment/$deploy" -n "$NAMESPACE" --timeout="$timeout" +} + +test_scale_out() { + echo "=== Testing RS_LIST change on scale out ===" + + local deploy rs_before observers_before gen_before + deploy=$(get_obproxy_deployment_name) + rs_before=$(get_rs_list) + observers_before=$(get_observer_count) + gen_before=$(get_deployment_generation) + + echo "Before scale out:" + echo " RS_LIST (from Pod): $rs_before" + echo " Observer count (DB): $observers_before" + echo " Deployment: $deploy generation=$gen_before" + echo " Debug - OBProxy status: $(get_obproxy_status)" + + echo "Scaling cluster to 2-2-2..." + start_condition_monitor + envsubst < "$TESTS_DIR/config/clusterManage/obcluster_template_2-2-2.yaml" | kubectl apply -f - + + wait_observer_targets 6 6 180 || true + + # Debug: Show Observer CRs + echo "" + echo "Observer CRs after scale out:" + kubectl get observer -n "$NAMESPACE" -l "${REF_CLUSTER_LABEL}=${OBCLUSTER_NAME}" -o custom-columns=NAME:.metadata.name,STATUS:.status.status 2>/dev/null || echo " (unable to get observers)" + + echo "Waiting for RS_LIST change AND rolling update (generation must change)..." + local counter=0 timeout=120 rs_after gen_after + while true; do + counter=$((counter + 1)) + deploy=$(get_obproxy_deployment_name) + rs_after=$(get_rs_list) + gen_after=$(get_deployment_generation) + + echo "Checking... ($counter/$timeout)" + echo " RS_LIST (Pod): $rs_after" + echo " generation: $gen_after (was $gen_before)" + + local rs_changed=0 rollout_signal=0 + [[ -n "$rs_after" && "$rs_before" != "$rs_after" ]] && rs_changed=1 + [[ -n "$gen_after" && "$gen_after" != "$gen_before" ]] && rollout_signal=1 + + if [[ "$rs_changed" -eq 1 && "$rollout_signal" -eq 1 ]]; then + echo "" + echo "SUCCESS: RS_LIST changed and Deployment spec changed (rolling update triggered)." + echo " RS before: $rs_before" + echo " RS after: $rs_after" + break + fi + + if [[ $counter -ge $timeout ]]; then + echo "WARNING: RS_LIST or rollout signal did not match within timeout (need RS change AND generation change)." + echo "" + echo "=== Debug Info on Timeout ===" + echo "Deployment env vars:" + kubectl get deployment -n "$NAMESPACE" -l "${OBPROXY_LABEL}=${OBPROXY_NAME}" -o jsonpath='{.items[0].spec.template.spec.containers[0].env}' 2>/dev/null || echo " Deployment not found" + echo "" + echo "Recent controller logs:" + get_obproxy_controller_logs "3m" + break + fi + sleep 5 + done + + if [[ -n "$deploy" ]]; then + wait_obproxy_rollout "$deploy" 300s || echo "WARNING: rollout status did not succeed in time" + fi + + stop_condition_monitor + + rs_after=$(get_rs_list) + if verify_all_obproxy_pods_rs_list "$rs_after"; then + echo "All Running OBProxy pods share the same RS_LIST." + else + echo "WARNING: not all OBProxy pods agree on RS_LIST after rollout." + fi + + analyze_debounce "$gen_before" "scale-out" +} + +test_scale_in() { + echo "" + echo "=== Testing RS_LIST change on scale in ===" + + local deploy rs_before gen_before + deploy=$(get_obproxy_deployment_name) + rs_before=$(get_rs_list) + gen_before=$(get_deployment_generation) + + echo "Before scale in:" + echo " RS_LIST (Pod): $rs_before" + echo " Deployment: $deploy generation=$gen_before" + + echo "Scaling cluster back to 1-1-1..." + start_condition_monitor + envsubst < "$TESTS_DIR/config/clusterManage/obcluster_template_1-1-1.yaml" | kubectl apply -f - + + wait_observer_targets 3 3 180 || true + + # Debug: Show Observer CRs + echo "" + echo "Observer CRs after scale in:" + kubectl get observer -n "$NAMESPACE" -l "${REF_CLUSTER_LABEL}=${OBCLUSTER_NAME}" -o custom-columns=NAME:.metadata.name,STATUS:.status.status 2>/dev/null || echo " (unable to get observers)" + + echo "Waiting for RS_LIST change AND rolling update..." + local counter=0 timeout=120 rs_after gen_after + while true; do + counter=$((counter + 1)) + deploy=$(get_obproxy_deployment_name) + rs_after=$(get_rs_list) + gen_after=$(get_deployment_generation) + + echo "Checking... ($counter/$timeout)" + echo " RS_LIST (Pod): $rs_after" + echo " generation: $gen_after (was $gen_before)" + + local rs_changed=0 rollout_signal=0 + [[ -n "$rs_after" && "$rs_before" != "$rs_after" ]] && rs_changed=1 + [[ -n "$gen_after" && "$gen_after" != "$gen_before" ]] && rollout_signal=1 + + if [[ "$rs_changed" -eq 1 && "$rollout_signal" -eq 1 ]]; then + echo "" + echo "SUCCESS: RS_LIST changed and Deployment spec changed (rolling update triggered)." + echo " RS before: $rs_before" + echo " RS after: $rs_after" + break + fi + + if [[ $counter -ge $timeout ]]; then + echo "WARNING: RS_LIST or rollout signal did not match within timeout." + echo "" + echo "=== Debug Info on Timeout ===" + echo "Deployment env vars:" + kubectl get deployment -n "$NAMESPACE" -l "${OBPROXY_LABEL}=${OBPROXY_NAME}" -o jsonpath='{.items[0].spec.template.spec.containers[0].env}' 2>/dev/null || echo " Deployment not found" + echo "" + echo "Recent controller logs:" + get_obproxy_controller_logs "3m" + break + fi + sleep 5 + done + + if [[ -n "$deploy" ]]; then + wait_obproxy_rollout "$deploy" 300s || echo "WARNING: rollout status did not succeed in time" + fi + + stop_condition_monitor + + rs_after=$(get_rs_list) + if verify_all_obproxy_pods_rs_list "$rs_after"; then + echo "All Running OBProxy pods agree on RS_LIST." + else + echo "WARNING: not all OBProxy pods agree on RS_LIST after rollout." + fi + + analyze_debounce "$gen_before" "scale-in" +} + +echo "=== OBProxy RS Change Test ===" +echo "NAMESPACE: $NAMESPACE" +echo "OBCLUSTER_NAME: $OBCLUSTER_NAME" +echo "OBPROXY_NAME: $OBPROXY_NAME" + +# Pre-check 1: Ensure no observers in deleting state +echo "" +echo "=== Pre-check 1: Ensuring no observers in deleting state ===" +wait_for_no_deleting_observers 60 || echo "WARNING: Some observers still in deleting state, continuing anyway..." + +# Pre-check 2: Check current cluster state and reset to 1-1-1 if needed +echo "" +echo "=== Pre-check 2: Checking initial cluster state ===" +current_observer_count=$(get_current_observer_count) +echo "Current observer count: $current_observer_count" + +if [[ "$current_observer_count" -eq 3 ]]; then + echo "Cluster is already in 1-1-1 state, ready to start test." +elif [[ "$current_observer_count" -eq 6 ]]; then + echo "Cluster is in 2-2-2 state, resetting to 1-1-1..." + if ! reset_cluster_to_1_1_1; then + echo "ERROR: Failed to reset cluster to 1-1-1 state. Aborting test." + exit 1 + fi +else + echo "WARNING: Cluster is in unexpected state with $current_observer_count observers." + echo "Attempting to reset to 1-1-1..." + if ! reset_cluster_to_1_1_1; then + echo "ERROR: Failed to reset cluster to 1-1-1 state. Aborting test." + exit 1 + fi +fi + +# Final check: Verify cluster is in 1-1-1 state +echo "" +echo "=== Final verification: Ensuring cluster is in 1-1-1 state ===" +final_count=$(get_current_observer_count) +if [[ "$final_count" -ne 3 ]]; then + echo "ERROR: Cluster is not in 1-1-1 state (has $final_count observers). Aborting test." + exit 1 +fi +echo "Cluster is ready in 1-1-1 state with 3 observers." + +test_scale_out +test_scale_in + +echo "" +echo "=== RS Change Test Completed ===" \ No newline at end of file diff --git a/tests/case_p8_OBProxy/check_P8_obproxy03_restart.sh b/tests/case_p8_OBProxy/check_P8_obproxy03_restart.sh new file mode 100755 index 000000000..f09981c4a --- /dev/null +++ b/tests/case_p8_OBProxy/check_P8_obproxy03_restart.sh @@ -0,0 +1,188 @@ +#!/bin/bash + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +TESTS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# load all the parameters in setup.sh +source "$TESTS_DIR/setup.sh" +source "$TESTS_DIR/util.sh" +source "$TESTS_DIR/env.sh" +source "$TESTS_DIR/case_p0_OBProxy/env_vars.sh" + +# OBProxy label selector +OBPROXY_LABEL="obproxy.oceanbase.com/obproxy" + +get_deployment_generation() { + kubectl get deployment -n $NAMESPACE -l ${OBPROXY_LABEL}=${OBPROXY_NAME} -o jsonpath='{.items[0].metadata.generation}' 2>/dev/null +} + +get_pod_restarts() { + kubectl get pods -n $NAMESPACE -l ${OBPROXY_LABEL}=${OBPROXY_NAME} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}' 2>/dev/null | tr ' ' '\n' | awk '{sum+=$1} END {print sum}' +} + +get_ready_replicas() { + kubectl get obproxy $OBPROXY_NAME -n $NAMESPACE -o jsonpath='{.status.readyReplicas}' 2>/dev/null +} + +get_desired_replicas() { + kubectl get obproxy $OBPROXY_NAME -n $NAMESPACE -o jsonpath='{.status.replicas}' 2>/dev/null +} + +# Test rolling update when image changes +test_image_update() { + echo "=== Testing OBProxy rolling update on image change ===" + + local gen_before=$(get_deployment_generation) + local image_before=$(kubectl get obproxy $OBPROXY_NAME -n $NAMESPACE -o jsonpath='{.spec.image}') + local pods_before=$(kubectl get pods -n $NAMESPACE -l ${OBPROXY_LABEL}=${OBPROXY_NAME} -o name | wc -l) + + echo "Before image update:" + echo " Image: $image_before" + echo " Generation: $gen_before" + echo " Pod count: $pods_before" + + # Update to a different image + local new_image="oceanbase/obproxy-ce:4.3.2.0-1" + echo "Updating image to: $new_image" + kubectl patch obproxy $OBPROXY_NAME -n $NAMESPACE --type=merge -p "{\"spec\":{\"image\":\"$new_image\"}}" + + # Wait for rolling update + local counter=0 + local timeout=180 + while true; do + counter=$((counter+1)) + local gen_after=$(get_deployment_generation) + local ready=$(get_ready_replicas) + local desired=$(get_desired_replicas) + local current_image=$(kubectl get obproxy $OBPROXY_NAME -n $NAMESPACE -o jsonpath='{.status.image}') + + echo "Waiting for rolling update... ($counter/$timeout)" + echo " Generation: $gen_after (was $gen_before)" + echo " Ready: $ready/$desired" + echo " Current image: $current_image" + + if [[ "$gen_after" != "$gen_before" && "$ready" == "$desired" && -n "$ready" ]]; then + echo "" + echo "SUCCESS: Rolling update completed!" + echo " Generation changed: $gen_before -> $gen_after" + echo " All pods ready: $ready/$desired" + break + fi + + if [ $counter -eq $timeout ]; then + echo "Rolling update did not complete within timeout" + kubectl describe obproxy $OBPROXY_NAME -n $NAMESPACE + kubectl get pods -n $NAMESPACE -l ${OBPROXY_LABEL}=${OBPROXY_NAME} + break + fi + sleep 5s + done +} + +# Test replica scaling +test_replica_scale() { + echo "" + echo "=== Testing OBProxy replica scaling ===" + + local replicas_before=$(kubectl get obproxy $OBPROXY_NAME -n $NAMESPACE -o jsonpath='{.spec.replicas}') + echo "Replicas before: $replicas_before" + + # Scale to 3 + local new_replicas=3 + echo "Scaling to $new_replicas replicas..." + kubectl patch obproxy $OBPROXY_NAME -n $NAMESPACE --type=merge -p "{\"spec\":{\"replicas\":$new_replicas}}" + + local counter=0 + local timeout=120 + while true; do + counter=$((counter+1)) + local ready=$(get_ready_replicas) + local desired=$(get_desired_replicas) + + echo "Waiting for scale... ready: $ready/$desired" + + if [[ "$ready" == "$new_replicas" && "$desired" == "$new_replicas" ]]; then + echo "" + echo "SUCCESS: Scale to $new_replicas completed!" + break + fi + + if [ $counter -eq $timeout ]; then + echo "Scale did not complete within timeout" + break + fi + sleep 3s + done + + # Scale back to original + echo "" + echo "Scaling back to $replicas_before replicas..." + kubectl patch obproxy $OBPROXY_NAME -n $NAMESPACE --type=merge -p "{\"spec\":{\"replicas\":$replicas_before}}" + + counter=0 + while true; do + counter=$((counter+1)) + local ready=$(get_ready_replicas) + local desired=$(get_desired_replicas) + + echo "Waiting for scale back... ready: $ready/$desired" + + if [[ "$ready" == "$replicas_before" && "$desired" == "$replicas_before" ]]; then + echo "" + echo "SUCCESS: Scale back to $replicas_before completed!" + break + fi + + if [ $counter -eq $timeout ]; then + echo "Scale back did not complete within timeout" + break + fi + sleep 3s + done +} + +# Test pod restart (delete pod and verify it comes back) +test_pod_restart() { + echo "" + echo "=== Testing OBProxy pod restart ===" + + local pods=$(kubectl get pods -n $NAMESPACE -l ${OBPROXY_LABEL}=${OBPROXY_NAME} -o name) + local pod_to_delete=$(echo "$pods" | head -1) + + echo "Deleting pod: $pod_to_delete" + kubectl delete $pod_to_delete -n $NAMESPACE + + local counter=0 + local timeout=60 + while true; do + counter=$((counter+1)) + local ready=$(get_ready_replicas) + local desired=$(get_desired_replicas) + + echo "Waiting for pod recovery... ready: $ready/$desired" + + if [[ "$ready" == "$desired" && -n "$ready" ]]; then + echo "" + echo "SUCCESS: Pod recovered successfully!" + break + fi + + if [ $counter -eq $timeout ]; then + echo "Pod did not recover within timeout" + break + fi + sleep 3s + done +} + +echo "=== OBProxy Restart/Rolling Update Test ===" +echo "NAMESPACE: $NAMESPACE" +echo "OBPROXY_NAME: $OBPROXY_NAME" + +test_image_update +test_replica_scale +test_pod_restart + +echo "" +echo "=== Restart/Rolling Update Test Completed ===" \ No newline at end of file diff --git a/tests/config/obproxyManage/obproxy_template.yaml b/tests/config/obproxyManage/obproxy_template.yaml new file mode 100644 index 000000000..71187b40b --- /dev/null +++ b/tests/config/obproxyManage/obproxy_template.yaml @@ -0,0 +1,17 @@ +apiVersion: oceanbase.oceanbase.com/v1alpha1 +kind: OBProxy +metadata: + name: ${OBPROXY_NAME} + namespace: ${NAMESPACE} +spec: + obCluster: + name: ${OBCLUSTER_NAME} + namespace: ${NAMESPACE} + proxyClusterName: ${OBPROXY_NAME} + proxySysSecret: ${PROXY_SYS_SECRET} + image: ${OBPROXY_IMAGE} + serviceType: ClusterIP + replicas: ${OBPROXY_REPLICAS} + resource: + memory: 1Gi + cpu: 500m \ No newline at end of file