// SPDX-License-Identifier: BSD-3-Clause package main import ( _ "k8s.io/client-go/plugin/pkg/client/auth" "crypto/tls" "errors" "flag" "fmt" "net/url" "os" "path/filepath" "runtime" "strings" corev1 "k8s.io/api/core/v1" kruntime "k8s.io/apimachinery/pkg/runtime" kruntimeutil "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" netbird "github.com/netbirdio/netbird/shared/management/client/rest" nbv1 "github.com/netbirdio/kubernetes-operator/api/v1" nbv1alpha1 "github.com/netbirdio/kubernetes-operator/api/v1alpha1" "github.com/netbirdio/kubernetes-operator/internal/controller" "github.com/netbirdio/kubernetes-operator/internal/version" nbwebhookv1 "github.com/netbirdio/kubernetes-operator/internal/webhook/v1" ) var ( scheme = kruntime.NewScheme() setupLog = ctrl.Log.WithName("setup") ) func init() { kruntimeutil.Must(clientgoscheme.AddToScheme(scheme)) kruntimeutil.Must(nbv1.AddToScheme(scheme)) kruntimeutil.Must(corev1.AddToScheme(scheme)) kruntimeutil.Must(gwv1.Install(scheme)) kruntimeutil.Must(gwv1alpha2.Install(scheme)) kruntimeutil.Must(nbv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } // nolint:gocyclo func main() { // NB Specific flags var ( runtimeNamespace string managementURL string clientImage string clusterName string namespacedNetworks bool clusterDNS string netbirdAPIKey string allowAutomaticPolicyCreation bool defaultLabels string gatewayAPIEnabled bool ) flag.StringVar(&runtimeNamespace, "runtime-namespace", "", "Namespace the controller is running in") flag.StringVar(&managementURL, "netbird-management-url", "https://api.netbird.io", "Management service URL") flag.StringVar(&clientImage, "netbird-client-image", "", "Image for netbird client container") flag.StringVar( &clusterName, "cluster-name", "kubernetes", "User-friendly name for kubernetes cluster for NetBird resource creation", ) flag.BoolVar( &namespacedNetworks, "namespaced-networks", false, "Create NetBird Network per namespace, set to true if a NetworkPolicy exists that would require this", ) flag.StringVar(&clusterDNS, "cluster-dns", "svc.cluster.local", "Cluster DNS name") flag.StringVar(&netbirdAPIKey, "netbird-api-key", "", "API key for NetBird API operations") flag.BoolVar( &allowAutomaticPolicyCreation, "allow-automatic-policy-creation", false, "Allow creating NBPolicy resources from annotations on Services", ) flag.StringVar( &defaultLabels, "default-labels", "", "Default labels used for all resources, in format key=value,key=value", ) flag.BoolVar(&gatewayAPIEnabled, "gateway-api-enabled", false, "When true Gateway API resources will be reconciled.") // Controller generic flags var ( metricsAddr string webhookCertPath string webhookCertName string webhookCertKey string enableLeaderElection bool probeAddr string enableWebhooks bool ) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") flag.BoolVar(&enableWebhooks, "enable-webhooks", true, "If set, enable Mutating and Validating webhooks.") opts := zap.Options{ Development: true, } opts.BindFlags(flag.CommandLine) flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) _, err := url.Parse(managementURL) if err != nil { setupLog.Error(err, "invalid management url") os.Exit(1) } runtimeNamespace, err = getRuntimeNamespace(runtimeNamespace) if err != nil { setupLog.Error(err, "unable to get runtime namespace") os.Exit(1) } if clientImage == "" { clientImage = version.ClientImage() } defaultLabelsMap := make(map[string]string) if defaultLabels != "" { for s := range strings.SplitSeq(defaultLabels, ",") { kv := strings.Split(s, "=") if len(kv) != 2 { panic(fmt.Errorf("invalid label format: %s", s)) } defaultLabelsMap[kv[0]] = kv[1] } } // Setup webhook server. type TLSOption = func(*tls.Config) certWatcher, tlsOpt, err := func() (*certwatcher.CertWatcher, TLSOption, error) { if webhookCertPath == "" { return nil, nil, nil } certWatcher, err := certwatcher.New( filepath.Join(webhookCertPath, webhookCertName), filepath.Join(webhookCertPath, webhookCertKey), ) if err != nil { return nil, nil, err } tlsOpt := func(config *tls.Config) { config.GetCertificate = certWatcher.GetCertificate } return certWatcher, tlsOpt, nil }() if err != nil { setupLog.Error(err, "Failed to initialize webhook certificate watcher") os.Exit(1) } webhookServer := webhook.NewServer(webhook.Options{TLSOpts: []TLSOption{tlsOpt}}) // Setup controller manager. mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ BindAddress: metricsAddr, }, Client: client.Options{ FieldOwner: "netbird-operator", }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, LeaderElectionNamespace: runtimeNamespace, LeaderElection: enableLeaderElection, LeaderElectionID: "operator.netbird.io", }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } nbSetupKeyController := &controller.NBSetupKeyReconciler{ Client: mgr.GetClient(), } if err = nbSetupKeyController.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NBSetupKey") os.Exit(1) } if enableWebhooks { if err = nbwebhookv1.SetupPodWebhookWithManager(mgr, managementURL, clientImage); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "Pod") os.Exit(1) } } if len(netbirdAPIKey) > 0 { nbClient := netbird.NewWithOptions( netbird.WithManagementURL(managementURL), netbird.WithBearerToken(netbirdAPIKey), netbird.WithUserAgent(fmt.Sprintf("netbird-operator/%s (%s/%s)", version.BuildVersion(), runtime.GOOS, runtime.GOARCH)), ) if err = (&controller.NBRoutingPeerReconciler{ Client: mgr.GetClient(), Netbird: nbClient, ClientImage: clientImage, ClusterName: clusterName, ManagementURL: managementURL, NamespacedNetworks: namespacedNetworks, DefaultLabels: defaultLabelsMap, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NBRoutingPeer") os.Exit(1) } if err = (&controller.ServiceReconciler{ Client: mgr.GetClient(), ClusterName: clusterName, ClusterDNS: clusterDNS, NamespacedNetworks: namespacedNetworks, ControllerNamespace: runtimeNamespace, DefaultLabels: defaultLabelsMap, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Service") os.Exit(1) } if err = (&controller.NBResourceReconciler{ Client: mgr.GetClient(), Netbird: nbClient, AllowAutomaticPolicyCreation: allowAutomaticPolicyCreation, ClusterName: clusterName, DefaultLabels: defaultLabelsMap, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NBResource") os.Exit(1) } if err = (&controller.NBGroupReconciler{ Client: mgr.GetClient(), Netbird: nbClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NBGroup") os.Exit(1) } if err = (&controller.NBPolicyReconciler{ Client: mgr.GetClient(), Netbird: nbClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NBPolicy") os.Exit(1) } if enableWebhooks { if err = nbwebhookv1.SetupNBGroupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "NBGroup") os.Exit(1) } } if err := (&controller.SetupKeyReconciler{ Client: mgr.GetClient(), Netbird: nbClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "SetupKey") os.Exit(1) } if err := (&controller.GroupReconciler{ Client: mgr.GetClient(), Netbird: nbClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "Group") os.Exit(1) } if err := (&controller.NetworkRouterReconciler{ Client: mgr.GetClient(), Netbird: nbClient, ClientImage: clientImage, ManagementURL: managementURL, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "NetworkRouter") os.Exit(1) } if err := (&controller.NetworkResourceReconciler{ Client: mgr.GetClient(), Netbird: nbClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "NetworkResource") os.Exit(1) } if gatewayAPIEnabled { if err = (&controller.GatewayClassReconciler{ Client: mgr.GetClient(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") os.Exit(1) } if err = (&controller.GatewayReconciler{ Client: mgr.GetClient(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Gateway") os.Exit(1) } if err = (&controller.HTTPRouteReconciler{ Client: mgr.GetClient(), Netbird: nbClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "HTTPRoute") os.Exit(1) } if err = (&controller.TCPRouteReconciler{ Client: mgr.GetClient(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TCPRoute") os.Exit(1) } } } else { setupLog.Info("netbird API key not provided, ingress capabilities disabled") } // +kubebuilder:scaffold:builder if certWatcher != nil { setupLog.Info("Adding webhook certificate watcher to manager") if err := mgr.Add(certWatcher); err != nil { setupLog.Error(err, "unable to add webhook certificate watcher to manager") os.Exit(1) } } if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) } readyChecker := healthz.Ping if certWatcher != nil { readyChecker = mgr.GetWebhookServer().StartedChecker() } if err := mgr.AddReadyzCheck("readyz", readyChecker); err != nil { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } } func getRuntimeNamespace(runtimeNamespace string) (string, error) { if runtimeNamespace != "" { return runtimeNamespace, nil } inClusterNamespacePath := "/var/run/secrets/kubernetes.io/serviceaccount/namespace" b, err := os.ReadFile(inClusterNamespacePath) if errors.Is(err, os.ErrNotExist) { return "", fmt.Errorf("not running in-cluster, runtime namespace needs to be set") } if err != nil { return "", fmt.Errorf("error reading namespace file: %w", err) } return string(b), nil }