OCPBUGS-98219: Move RBAC manifests to the services that own them

pkg/cmd/run.go

@@ -228,8 +228,8 @@ func RunMicroshift(cfg *config.Config) error {
 	util.Must(m.AddService(controllers.NewRouteControllerManager(cfg)))
 	util.Must(m.AddService(controllers.NewOpenShiftDefaultSCCManager(cfg)))
 	util.Must(m.AddService(mdns.NewMicroShiftmDNSController(cfg)))
-	util.Must(m.AddService(controllers.NewInfrastructureServices(cfg)))
 	util.Must(m.AddService(controllers.NewClusterPolicyController(cfg)))
+	util.Must(m.AddService(controllers.NewInfrastructureServices(cfg)))
 	util.Must(m.AddService(controllers.NewVersionManager(cfg)))
 	util.Must(m.AddService(controllers.NewKubeletCAManager(cfg)))
 	util.Must(m.AddService(node.NewKubeletServer(cfg)))

pkg/controllers/cluster-policy-controller.go

@@ -17,21 +17,30 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"time"
 
 	openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1"
+	securityv1 "github.com/openshift/api/security/v1"
 	clusterpolicycontroller "github.com/openshift/cluster-policy-controller/pkg/cmd/cluster-policy-controller"
 	"github.com/openshift/library-go/pkg/controller/controllercmd"
 	"github.com/openshift/library-go/pkg/operator/events"
+	"github.com/openshift/microshift/pkg/assets"
 	"github.com/openshift/microshift/pkg/config"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	unstructuredv1 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	klog "k8s.io/klog/v2"
 	"k8s.io/utils/clock"
 )
 
 type ClusterPolicyController struct {
 	run        func(context.Context) error
+	applyRBAC  func(context.Context) error
 	kubeconfig string
 
 	configErr error
@@ -45,11 +54,31 @@ func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController {
 
 func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" }
 func (s *ClusterPolicyController) Dependencies() []string {
-	return []string{"kube-apiserver", "infrastructure-services-manager"}
+	return []string{"kube-apiserver"}
 }
 
 func (s *ClusterPolicyController) configure(cfg *config.Config) error {
 	s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController)
+	s.applyRBAC = func(ctx context.Context) error {
+		kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin)
+		cr := []string{
+			"controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml",
+			"controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml",
+			"controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml",
+		}
+		crb := []string{
+			"controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml",
+			"controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml",
+			"controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml",
+		}
+		if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil {
+			return fmt.Errorf("failed to apply cluster-policy-controller RBAC: %w", err)
+		}
+		if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil {
+			return fmt.Errorf("failed to apply cluster-policy-controller RBAC: %w", err)
+		}
+		return nil
+	}
 
 	scheme := runtime.NewScheme()
 	if err := openshiftcontrolplanev1.AddToScheme(scheme); err != nil {
@@ -101,6 +130,44 @@ func (s *ClusterPolicyController) Run(ctx context.Context, ready chan<- struct{}
 		return fmt.Errorf("configuration failed: %w", s.configErr)
 	}
 
-	close(ready) // todo
-	return s.run(ctx)
+	if err := s.applyRBAC(ctx); err != nil {
+		return err
+	}
+
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- s.run(ctx)
+	}()
+
+	if err := waitForNamespaceSecurityAllocation(ctx, s.kubeconfig); err != nil {
+		return err
+	}
+	klog.Infof("%s is ready", s.Name())
+	close(ready)
+
+	return <-errCh
+}
+
+// waitForNamespaceSecurityAllocation waits until the namespace-security-allocation-controller
+// has annotated the default namespace with the UID range annotation, proving it is running
+// and processing namespaces. This must happen before infrastructure-services-manager starts
+// creating deployments, because the SCC admission plugin blocks until these annotations exist.
+func waitForNamespaceSecurityAllocation(ctx context.Context, kubeconfigPath string) error {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to build kubeconfig for readiness check: %w", err)
+	}
+	kubeClient, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return fmt.Errorf("failed to create kube client for readiness check: %w", err)
+	}
+
+	return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
+		ns, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{})
+		if err != nil {
+			return false, nil //nolint:nilerr // retry on transient errors
+		}
+		_, ok := ns.Annotations[securityv1.UIDRangeAnnotation]
+		return ok, nil
+	})
 }

pkg/controllers/infra-services-controller.go

@@ -37,55 +37,22 @@ func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManage
 
 func (s *InfrastructureServicesManager) Name() string { return "infrastructure-services-manager" }
 func (s *InfrastructureServicesManager) Dependencies() []string {
-	return []string{"kube-apiserver", "openshift-crd-manager", "route-controller-manager"}
+	return []string{"kube-apiserver", "cluster-policy-controller", "openshift-crd-manager", "route-controller-manager"}
 }
 
 func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error {
 	defer close(stopped)
 	defer close(ready)
 
-	if err := applyDefaultRBACs(ctx, s.cfg); err != nil {
-		klog.Errorf("%s unable to apply default RBACs: %v", s.Name(), err)
-		return err
-	}
-
 	priorityClasses := []string{"core/priority-class-openshift-user-critical.yaml"}
 	if err := assets.ApplyPriorityClasses(ctx, priorityClasses, s.cfg.KubeConfigPath(config.KubeAdmin)); err != nil {
 		klog.Errorf("%s unable to apply PriorityClasses: %v", s.Name(), err)
 		return err
 	}
 
-	// TO-DO add readiness check
 	if err := components.StartComponents(s.cfg, ctx); err != nil {
 		return err
 	}
 	klog.Infof("%s launched ocp componets", s.Name())
 	return ctx.Err()
 }
-
-func applyDefaultRBACs(ctx context.Context, cfg *config.Config) error {
-	kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin)
-	var (
-		cr = []string{
-			"controllers/kube-controller-manager/csr_approver_clusterrole.yaml",
-			"controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml",
-			"controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml",
-			"controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml",
-		}
-		crb = []string{
-			"controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml",
-			"controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml",
-			"controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml",
-			"controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml",
-		}
-	)
-	if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil {
-		klog.Warningf("failed to apply cluster roles %v", err)
-		return err
-	}
-	if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil {
-		klog.Warningf("failed to apply cluster roles %v", err)
-		return err
-	}
-	return nil
-}

pkg/controllers/kube-controller-manager.go

@@ -111,10 +111,24 @@ func configure(ctx context.Context, cfg *config.Config) (args []string, applyFn
 
 	args, err = mergeAndConvertToArgs(overrides)
 	applyFn = func() error {
-		return assets.ApplyNamespaces(ctx, []string{
+		kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin)
+		if err := assets.ApplyNamespaces(ctx, []string{
 			"controllers/kube-controller-manager/namespace-openshift-kube-controller-manager.yaml",
 			"core/namespace-openshift-infra.yaml",
-		}, cfg.KubeConfigPath(config.KubeAdmin))
+		}, kubeconfigPath); err != nil {
+			return fmt.Errorf("failed to apply kube-controller-manager namespaces: %w", err)
+		}
+		if err := assets.ApplyClusterRoles(ctx, []string{
+			"controllers/kube-controller-manager/csr_approver_clusterrole.yaml",
+		}, kubeconfigPath); err != nil {
+			return fmt.Errorf("failed to apply kube-controller-manager RBAC: %w", err)
+		}
+		if err := assets.ApplyClusterRoleBindings(ctx, []string{
+			"controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml",
+		}, kubeconfigPath); err != nil {
+			return fmt.Errorf("failed to apply kube-controller-manager RBAC: %w", err)
+		}
+		return nil
 	}
 	return args, applyFn, err
 }
@@ -157,7 +171,7 @@ func (s *KubeControllerManager) Run(ctx context.Context, ready chan<- struct{},
 	}()
 
 	if err := s.applyFn(); err != nil {
-		return fmt.Errorf("failed to apply openshift namespaces: %w", err)
+		return err
 	}
 
 	select {