diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 7ea654c2ea..ca2b132f91 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -228,8 +228,8 @@ func RunMicroshift(cfg *config.Config) error { util.Must(m.AddService(controllers.NewRouteControllerManager(cfg))) util.Must(m.AddService(controllers.NewOpenShiftDefaultSCCManager(cfg))) util.Must(m.AddService(mdns.NewMicroShiftmDNSController(cfg))) - util.Must(m.AddService(controllers.NewInfrastructureServices(cfg))) util.Must(m.AddService(controllers.NewClusterPolicyController(cfg))) + util.Must(m.AddService(controllers.NewInfrastructureServices(cfg))) util.Must(m.AddService(controllers.NewVersionManager(cfg))) util.Must(m.AddService(controllers.NewKubeletCAManager(cfg))) util.Must(m.AddService(node.NewKubeletServer(cfg))) diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index f074f71b8d..95623505fd 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -17,21 +17,30 @@ package controllers import ( "context" "fmt" + "time" openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1" + securityv1 "github.com/openshift/api/security/v1" clusterpolicycontroller "github.com/openshift/cluster-policy-controller/pkg/cmd/cluster-policy-controller" "github.com/openshift/library-go/pkg/controller/controllercmd" "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/microshift/pkg/assets" "github.com/openshift/microshift/pkg/config" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unstructuredv1 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + klog "k8s.io/klog/v2" "k8s.io/utils/clock" ) type ClusterPolicyController struct { run func(context.Context) error + applyRBAC func(context.Context) error kubeconfig string configErr error @@ -45,11 +54,31 @@ func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController { func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" } func (s *ClusterPolicyController) Dependencies() []string { - return []string{"kube-apiserver", "infrastructure-services-manager"} + return []string{"kube-apiserver"} } func (s *ClusterPolicyController) configure(cfg *config.Config) error { s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController) + s.applyRBAC = func(ctx context.Context) error { + kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) + cr := []string{ + "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml", + } + crb := []string{ + "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml", + } + if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply cluster-policy-controller RBAC: %w", err) + } + if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply cluster-policy-controller RBAC: %w", err) + } + return nil + } scheme := runtime.NewScheme() if err := openshiftcontrolplanev1.AddToScheme(scheme); err != nil { @@ -101,6 +130,44 @@ func (s *ClusterPolicyController) Run(ctx context.Context, ready chan<- struct{} return fmt.Errorf("configuration failed: %w", s.configErr) } - close(ready) // todo - return s.run(ctx) + if err := s.applyRBAC(ctx); err != nil { + return err + } + + errCh := make(chan error, 1) + go func() { + errCh <- s.run(ctx) + }() + + if err := waitForNamespaceSecurityAllocation(ctx, s.kubeconfig); err != nil { + return err + } + klog.Infof("%s is ready", s.Name()) + close(ready) + + return <-errCh +} + +// waitForNamespaceSecurityAllocation waits until the namespace-security-allocation-controller +// has annotated the default namespace with the UID range annotation, proving it is running +// and processing namespaces. This must happen before infrastructure-services-manager starts +// creating deployments, because the SCC admission plugin blocks until these annotations exist. +func waitForNamespaceSecurityAllocation(ctx context.Context, kubeconfigPath string) error { + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return fmt.Errorf("failed to build kubeconfig for readiness check: %w", err) + } + kubeClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to create kube client for readiness check: %w", err) + } + + return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + ns, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}) + if err != nil { + return false, nil //nolint:nilerr // retry on transient errors + } + _, ok := ns.Annotations[securityv1.UIDRangeAnnotation] + return ok, nil + }) } diff --git a/pkg/controllers/infra-services-controller.go b/pkg/controllers/infra-services-controller.go index 628a6c7076..3f0207ec38 100644 --- a/pkg/controllers/infra-services-controller.go +++ b/pkg/controllers/infra-services-controller.go @@ -37,55 +37,22 @@ func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManage func (s *InfrastructureServicesManager) Name() string { return "infrastructure-services-manager" } func (s *InfrastructureServicesManager) Dependencies() []string { - return []string{"kube-apiserver", "openshift-crd-manager", "route-controller-manager"} + return []string{"kube-apiserver", "cluster-policy-controller", "openshift-crd-manager", "route-controller-manager"} } func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { defer close(stopped) defer close(ready) - if err := applyDefaultRBACs(ctx, s.cfg); err != nil { - klog.Errorf("%s unable to apply default RBACs: %v", s.Name(), err) - return err - } - priorityClasses := []string{"core/priority-class-openshift-user-critical.yaml"} if err := assets.ApplyPriorityClasses(ctx, priorityClasses, s.cfg.KubeConfigPath(config.KubeAdmin)); err != nil { klog.Errorf("%s unable to apply PriorityClasses: %v", s.Name(), err) return err } - // TO-DO add readiness check if err := components.StartComponents(s.cfg, ctx); err != nil { return err } klog.Infof("%s launched ocp componets", s.Name()) return ctx.Err() } - -func applyDefaultRBACs(ctx context.Context, cfg *config.Config) error { - kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) - var ( - cr = []string{ - "controllers/kube-controller-manager/csr_approver_clusterrole.yaml", - "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml", - } - crb = []string{ - "controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml", - "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml", - } - ) - if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil { - klog.Warningf("failed to apply cluster roles %v", err) - return err - } - if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil { - klog.Warningf("failed to apply cluster roles %v", err) - return err - } - return nil -} diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 409f87e694..a5f2ea662e 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -111,10 +111,24 @@ func configure(ctx context.Context, cfg *config.Config) (args []string, applyFn args, err = mergeAndConvertToArgs(overrides) applyFn = func() error { - return assets.ApplyNamespaces(ctx, []string{ + kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) + if err := assets.ApplyNamespaces(ctx, []string{ "controllers/kube-controller-manager/namespace-openshift-kube-controller-manager.yaml", "core/namespace-openshift-infra.yaml", - }, cfg.KubeConfigPath(config.KubeAdmin)) + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager namespaces: %w", err) + } + if err := assets.ApplyClusterRoles(ctx, []string{ + "controllers/kube-controller-manager/csr_approver_clusterrole.yaml", + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager RBAC: %w", err) + } + if err := assets.ApplyClusterRoleBindings(ctx, []string{ + "controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml", + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager RBAC: %w", err) + } + return nil } return args, applyFn, err } @@ -157,7 +171,7 @@ func (s *KubeControllerManager) Run(ctx context.Context, ready chan<- struct{}, }() if err := s.applyFn(); err != nil { - return fmt.Errorf("failed to apply openshift namespaces: %w", err) + return err } select {