PoC: Single MC

assets/performanceprofile/tuned/openshift-node-performance

@@ -1,3 +1,5 @@
+# PerformanceProfile.Name: {{.PerformanceProfileName}}
+# PerformanceProfile.Generation: {{.ProfileGeneration}}
 [main]
 summary=Openshift node optimized for deterministic performance at the cost of increased power consumption, focused on low latency network performance. Based on Tuned 2.11 and Cluster node tuning (oc 4.5)
 

assets/performanceprofile/tuned/openshift-node-performance-amd-x86

@@ -1,3 +1,5 @@
+# PerformanceProfile.Name: {{.PerformanceProfileName}}
+# PerformanceProfile.Generation: {{.ProfileGeneration}}
 [main]
 summary=Platform specific tuning for AMD x86
 

assets/performanceprofile/tuned/openshift-node-performance-arm-aarch64

@@ -1,3 +1,5 @@
+# PerformanceProfile.Name: {{.PerformanceProfileName}}
+# PerformanceProfile.Generation: {{.ProfileGeneration}}
 [main]
 summary=Platform specific tuning for aarch64
 

assets/performanceprofile/tuned/openshift-node-performance-intel-x86

@@ -1,3 +1,5 @@
+# PerformanceProfile.Name: {{.PerformanceProfileName}}
+# PerformanceProfile.Generation: {{.ProfileGeneration}}
 [main]
 summary=Platform specific tuning for Intel x86
 

assets/performanceprofile/tuned/openshift-node-performance-rt

@@ -1,3 +1,5 @@
+# PerformanceProfile.Name: {{.PerformanceProfileName}}
+# PerformanceProfile.Generation: {{.ProfileGeneration}}
 [main]
 summary=Real time profile to override unsupported settings
 

pkg/apis/tuned/v1/tuned_types.go

@@ -26,6 +26,9 @@ const (
 	// calculated by TuneD for the current profile applied to that Node.
 	TunedBootcmdlineAnnotationKey string = "tuned.openshift.io/bootcmdline"
 
+	// TunedBootcmdlineDepsAnnotationKey is used to determine Node's bootcmdline dependencies.
+	TunedBootcmdlineDepsAnnotationKey string = "tuned.openshift.io/bootcmdline-deps"
+
 	// TunedDeferredUpdate request the tuned daemons to defer the update of the rendered profile
 	// until the next restart.
 	TunedDeferredUpdate string = "tuned.openshift.io/deferred"

pkg/operator/controller.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"strings"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -38,6 +39,7 @@ import (
 	tunedinformers "github.com/openshift/cluster-node-tuning-operator/pkg/generated/informers/externalversions"
 	ntomf "github.com/openshift/cluster-node-tuning-operator/pkg/manifests"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/metrics"
+	"github.com/openshift/cluster-node-tuning-operator/pkg/tuned"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/util"
 	"github.com/openshift/cluster-node-tuning-operator/version"
 
@@ -692,11 +694,19 @@ func (c *Controller) syncProfile(tuned *tunedv1.Tuned, nodeName string) error {
 		// MachineConfigs.
 		if computed.NodePoolName != "" {
 			if profile.Status.TunedProfile == computed.TunedProfileName && profileApplied(profile) {
-				// Synchronize MachineConfig only once the (calculated) TuneD profile 'tunedProfileName'
-				// has been successfully applied.
-				err := c.syncMachineConfigHyperShift(computed.NodePoolName, profile)
-				if err != nil {
-					return fmt.Errorf("failed to update Profile %s: %v", profile.Name, err)
+				// Skip MachineConfig creation if this profile is managed by PerformanceProfile controller.
+				// PerformanceProfile controller creates its own "50-performance-*" MachineConfigs with
+				// boot parameters from tuned, so we don't need the operator controller to also create
+				// "50-nto-*" MachineConfigs for the same nodes (which would cause race conditions).
+				if c.isManagedByPerformanceProfile(profile) {
+					klog.V(2).Infof("skipping MachineConfig creation for profile %s: managed by PerformanceProfile controller", profile.Name)
+				} else {
+					// Synchronize MachineConfig only once the (calculated) TuneD profile 'tunedProfileName'
+					// has been successfully applied.
+					err := c.syncMachineConfigHyperShift(computed.NodePoolName, profile)
+					if err != nil {
+						return fmt.Errorf("failed to update Profile %s: %v", profile.Name, err)
+					}
 				}
 			}
 		}
@@ -706,11 +716,19 @@ func (c *Controller) syncProfile(tuned *tunedv1.Tuned, nodeName string) error {
 			// labels 'mcLabels' set for additional machine configuration.  Sync the operator-created
 			// MachineConfig based on 'mcLabels'.
 			if profile.Status.TunedProfile == computed.TunedProfileName && profileApplied(profile) {
-				// Synchronize MachineConfig only once the (calculated) TuneD profile 'tunedProfileName'
-				// has been successfully applied.
-				err := c.syncMachineConfig(computed.MCLabels, profile)
-				if err != nil {
-					return fmt.Errorf("failed to update Profile %s: %v", profile.Name, err)
+				// Skip MachineConfig creation if this profile is managed by PerformanceProfile controller.
+				// PerformanceProfile controller creates its own "50-performance-*" MachineConfigs with
+				// boot parameters from tuned, so we don't need the operator controller to also create
+				// "50-nto-*" MachineConfigs for the same nodes (which would cause race conditions).
+				if c.isManagedByPerformanceProfile(profile) {
+					klog.V(2).Infof("skipping MachineConfig creation for profile %s: managed by PerformanceProfile controller", profile.Name)
+				} else {
+					// Synchronize MachineConfig only once the (calculated) TuneD profile 'tunedProfileName'
+					// has been successfully applied.
+					err := c.syncMachineConfig(computed.MCLabels, profile)
+					if err != nil {
+						return fmt.Errorf("failed to update Profile %s: %v", profile.Name, err)
+					}
 				}
 			}
 		} else {
@@ -767,6 +785,81 @@ func (c *Controller) getProviderName(nodeName string) (string, error) {
 	return util.GetProviderName(node.Spec.ProviderID), nil
 }
 
+// getProfileDependencies recursively finds all profiles that the given profile depends on
+// by following include= directives. This is similar to profileDepends() but works with
+// in-memory Profile data (from tunedv1.TunedProfile) instead of reading from disk.
+func getProfileDependencies(profileName string, profileMap map[string]*tunedv1.TunedProfile, seen map[string]bool) map[string]bool {
+	if seen[profileName] {
+		return seen
+	}
+
+	tunedProfile, exists := profileMap[profileName]
+	if !exists || tunedProfile.Data == nil {
+		return seen
+	}
+
+	// Parse the profile data to get include= directives
+	includes := tuned.GetIniFileSectionSlice(tunedProfile.Data, "main", "include", ",")
+	for _, includedProfile := range includes {
+		// Remove optional leading '-' for conditional includes
+		includedProfile = strings.TrimPrefix(includedProfile, "-")
+		includedProfile = strings.TrimSpace(includedProfile)
+
+		if includedProfile == "" {
+			continue
+		}
+
+		seen[includedProfile] = true
+		// Recursively get dependencies of the included profile
+		seen = getProfileDependencies(includedProfile, profileMap, seen)
+	}
+
+	return seen
+}
+
+// isManagedByPerformanceProfile checks if a tuned profile is or includes/inherits a tuned profile managed by
+// the PerformanceProfile controller. PerformanceProfile controller creates tuned profiles with names starting
+// with "openshift-node-performance". These profiles should not have MachineConfigs created by the operator
+// controller as the PerformanceProfile controller creates its own MachineConfigs with boot parameters from tuned.
+func (c *Controller) isManagedByPerformanceProfile(profile *tunedv1.Profile) bool {
+	const (
+		tunedPerformanceProfilePrefix = "openshift-node-performance-"
+	)
+
+	tunedProfileName := profile.Status.TunedProfile
+
+	// Check if the profile name itself starts with "openshift-node-performance-".
+	// PerformanceProfile creates profiles with names like:
+	// - openshift-node-performance-<profile-name>
+	// - openshift-node-performance-<profile-name>-rt
+	// - openshift-node-performance-<profile-name>-amd-x86
+	// - openshift-node-performance-<profile-name>-arm-aarch64
+	// - openshift-node-performance-<profile-name>-intel-x86
+	if strings.HasPrefix(tunedProfileName, tunedPerformanceProfilePrefix) {
+		return true
+	}
+
+	// Build a map of all profiles in the Profile object for quick lookup.
+	profileMap := make(map[string]*tunedv1.TunedProfile)
+	for i := range profile.Spec.Profile {
+		tp := &profile.Spec.Profile[i]
+		if tp.Name != nil {
+			profileMap[*tp.Name] = tp
+		}
+	}
+
+	// Recursively check if the profile depends on a PerformanceProfile-managed profile.
+	deps := getProfileDependencies(tunedProfileName, profileMap, make(map[string]bool))
+	for depProfileName := range deps {
+		if strings.HasPrefix(depProfileName, tunedPerformanceProfilePrefix) {
+			klog.V(2).Infof("profile %s depends on PerformanceProfile-managed profile %s", tunedProfileName, depProfileName)
+			return true
+		}
+	}
+
+	return false
+}
+
 func (c *Controller) syncMachineConfig(labels map[string]string, profile *tunedv1.Profile) error {
 	var (
 		bootcmdline     string

pkg/performanceprofile/controller/performanceprofile/components/components.go

@@ -43,6 +43,7 @@ type Options struct {
 type MachineConfigOptions struct {
 	PinningMode      *apiconfigv1.CPUPartitioningMode
 	MixedCPUsEnabled bool
+	KernelArguments  []string
 }
 
 type KubeletConfigOptions struct {

pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go

@@ -31,6 +31,30 @@ func NewHandler(cli client.Client, scheme *runtime.Scheme) components.Handler {
 	return &handler{Client: cli, scheme: scheme}
 }
 
+func (h *handler) applyTuned(ctx context.Context, profile *performancev2.PerformanceProfile, mfs *manifestset.ManifestResultSet) ([]string, error) {
+	// Get mutated performance tuned.
+	performanceTunedMutated, err := resources.GetMutatedTuned(ctx, h.Client, mfs.Tuned)
+	if err != nil {
+		return nil, err
+	}
+
+	// The Tuned CR must be created first so the tuned operand can calculate bootcmdline.
+	if performanceTunedMutated != nil {
+		if err := resources.CreateOrUpdateTuned(ctx, h.Client, performanceTunedMutated, profile.Name); err != nil {
+			return nil, err
+		}
+	}
+
+	// Get kernel arguments from tuned bootcmdline. This will wait for tuned to calculate and agree on them.
+	// In other words, if bootcmdline is not ready or nodes disagree, an error is returned.
+	kernelArguments, err := resources.GetKernelArgumentsFromTunedBootcmdline(ctx, h.Client, profile)
+	if err != nil {
+		return nil, err
+	}
+
+	return kernelArguments, nil
+}
+
 func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record.EventRecorder, opts *components.Options) error {
 	profile, ok := obj.(*performancev2.PerformanceProfile)
 	if !ok {
@@ -45,43 +69,52 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record.
 	// set missing options
 	opts.MachineConfig.MixedCPUsEnabled = opts.MixedCPUsFeatureGateEnabled && profileutil.IsMixedCPUsEnabled(profile)
 
-	components, err := manifestset.GetNewComponents(profile, opts)
+	mfs, err := manifestset.GetNewComponents(profile, opts)
 	if err != nil {
 		return err
 	}
-	for _, componentObj := range components.ToObjects() {
+	for _, componentObj := range mfs.ToObjects() {
 		if err := controllerutil.SetControllerReference(profile, componentObj, h.scheme); err != nil {
 			return err
 		}
 	}
 
-	// get mutated machine config
-	mcMutated, err := resources.GetMutatedMachineConfig(ctx, h.Client, components.MachineConfig)
+	// The Tuned CR must be created first so the tuned operand can calculate bootcmdline.
+	kernelArguments, err := h.applyTuned(ctx, profile, mfs)
 	if err != nil {
 		return err
 	}
 
-	// get mutated kubelet config
-	kcMutated, err := resources.GetMutatedKubeletConfig(ctx, h.Client, components.KubeletConfig)
+	// Generate MachineConfig with kernel arguments from tuned bootcmdline
+	opts.MachineConfig.KernelArguments = kernelArguments
+	mc, err := machineconfig.New(profile, &opts.MachineConfig)
 	if err != nil {
 		return err
 	}
+	if err := controllerutil.SetControllerReference(profile, mc, h.scheme); err != nil {
+		return err
+	}
 
-	// get mutated performance tuned
-	performanceTunedMutated, err := resources.GetMutatedTuned(ctx, h.Client, components.Tuned)
+	// get mutated machine config WITH kernel arguments
+	mcMutated, err := resources.GetMutatedMachineConfig(ctx, h.Client, mc)
+	if err != nil {
+		return err
+	}
+
+	// get mutated kubelet config
+	kcMutated, err := resources.GetMutatedKubeletConfig(ctx, h.Client, mfs.KubeletConfig)
 	if err != nil {
 		return err
 	}
 
 	// get mutated RuntimeClass
-	runtimeClassMutated, err := resources.GetMutatedRuntimeClass(ctx, h.Client, components.RuntimeClass)
+	runtimeClassMutated, err := resources.GetMutatedRuntimeClass(ctx, h.Client, mfs.RuntimeClass)
 	if err != nil {
 		return err
 	}
 
 	updated := mcMutated != nil ||
 		kcMutated != nil ||
-		performanceTunedMutated != nil ||
 		runtimeClassMutated != nil
 
 	// does not update any resources if it no changes to relevant objects and continue to the status update
@@ -95,12 +128,6 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record.
 		}
 	}
 
-	if performanceTunedMutated != nil {
-		if err := resources.CreateOrUpdateTuned(ctx, h.Client, performanceTunedMutated, profile.Name); err != nil {
-			return err
-		}
-	}
-
 	if kcMutated != nil {
 		if err := resources.CreateOrUpdateKubeletConfig(ctx, h.Client, kcMutated); err != nil {
 			return err
@@ -113,6 +140,7 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record.
 		}
 	}
 	recorder.Eventf(profile, corev1.EventTypeNormal, "Creation succeeded", "Succeeded to create all components")
+
 	return nil
 }
 

pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go

@@ -132,7 +132,9 @@ func New(profile *performancev2.PerformanceProfile, opts *components.MachineConf
 			Name:   name,
 			Labels: profilecomponent.GetMachineConfigLabel(profile),
 		},
-		Spec: machineconfigv1.MachineConfigSpec{},
+		Spec: machineconfigv1.MachineConfigSpec{
+			KernelArguments: opts.KernelArguments,
+		},
 	}
 
 	ignitionConfig, err := getIgnitionConfig(profile, opts)

pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go

@@ -38,6 +38,7 @@ const (
 	templateIsolatedCpuList                 = "IsolatedCpuList"
 	templateReservedCpuList                 = "ReservedCpuList"
 	templatePerformanceProfileName          = "PerformanceProfileName"
+	templateProfileGeneration               = "ProfileGeneration"
 )
 
 func new(name string, profiles []tunedv1.TunedProfile, recommends []tunedv1.TunedRecommend) *tunedv1.Tuned {
@@ -62,6 +63,7 @@ func NewNodePerformance(profile *performancev2.PerformanceProfile) (*tunedv1.Tun
 	templateArgs := make(map[string]interface{})
 
 	templateArgs[templatePerformanceProfileName] = profile.Name
+	templateArgs[templateProfileGeneration] = profile.Generation
 
 	if profile.Spec.CPU.Isolated != nil {
 		minifiedCpuSet, err := cpuset.Parse(string(*profile.Spec.CPU.Isolated))