// SPDX-License-Identifier: BSD-3-Clause package controller import ( "context" "errors" "fmt" "slices" "strings" "time" "github.com/go-logr/logr" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" netbird "github.com/netbirdio/netbird/shared/management/client/rest" "github.com/netbirdio/netbird/shared/management/http/api" nbv1 "github.com/netbirdio/kubernetes-operator/api/v1" "github.com/netbirdio/kubernetes-operator/internal/util" ) const ( ResourceFinalizer = "gateway.netbird.io/resource" ) var ( errDuplicateResource = fmt.Errorf("duplicate resource") ) // NBResourceReconciler reconciles a NBResource object type NBResourceReconciler struct { client.Client Netbird *netbird.Client AllowAutomaticPolicyCreation bool ClusterName string DefaultLabels map[string]string } // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *NBResourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) { logger := ctrl.Log.WithName("NBResource").WithValues("namespace", req.Namespace, "name", req.Name) logger.Info("Reconciling NBResource") nbResource := &nbv1.NBResource{} err = r.Client.Get(ctx, req.NamespacedName, nbResource) if err != nil { if !kerrors.IsNotFound(err) { logger.Error(errKubernetesAPI, "error getting NBResource", "err", err) } return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil } originalResource := nbResource.DeepCopy() defer func() { if err != nil { // double check result is nil, otherwise error is not printed // and exponential backoff doesn't work properly res = ctrl.Result{} return } if originalResource.DeletionTimestamp != nil && len(nbResource.Finalizers) == 0 { return } if !originalResource.Status.Equal(nbResource.Status) { updateErr := r.Client.Status().Update(ctx, nbResource) if updateErr != nil { err = updateErr } } if !res.Requeue && res.RequeueAfter == 0 { res.RequeueAfter = defaultRequeueAfter } }() if !nbResource.DeletionTimestamp.IsZero() { err = r.reconcileDelete(ctx, req, nbResource) if err != nil { return ctrl.Result{}, err } return ctrl.Result{}, nil } if controllerutil.AddFinalizer(nbResource, ResourceFinalizer) { err := r.Client.Update(ctx, nbResource) if err != nil { return ctrl.Result{}, err } } groupIDs, result, err := r.handleGroups(ctx, req, nbResource, logger) if result != nil { nbResource.Status.Conditions = nbv1.NBConditionFalse("internalError", fmt.Sprintf("Error occurred handling groups: %v", err)) return *result, err } resource, err := r.handleNetBirdResource(ctx, nbResource, groupIDs, logger) if err != nil && errors.Is(err, errDuplicateResource) { return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil } if err != nil { nbResource.Status.Conditions = nbv1.NBConditionFalse("internalError", fmt.Sprintf("Error occurred handling NetBird Network Resource: %v", err)) return ctrl.Result{}, err } // resource is only nil if requeue is expected if resource == nil { return ctrl.Result{Requeue: true}, nil } err = r.handleGroupUpdate(ctx, nbResource, groupIDs, resource, logger) if err != nil { nbResource.Status.Conditions = nbv1.NBConditionFalse("internalError", fmt.Sprintf("Error occurred handling groups: %v", err)) return ctrl.Result{}, err } err = r.handlePolicy(ctx, req, nbResource, groupIDs, logger) if err != nil { nbResource.Status.Conditions = nbv1.NBConditionFalse("internalError", fmt.Sprintf("Error occurred handling policy changes: %v", err)) return ctrl.Result{}, err } nbResource.Status.Conditions = nbv1.NBConditionTrue() return ctrl.Result{}, nil } func (r *NBResourceReconciler) handlePolicyCreate(ctx context.Context, nbResource *nbv1.NBResource, req ctrl.Request, policy string, nbPolicy *nbv1.NBPolicy, logger logr.Logger) error { if len(nbResource.Spec.PolicySourceGroups) == 0 { logger.Error(errInvalidValue, "Cannot auto-generate policy, missing source groups.") return fmt.Errorf("cannot auto-generate policy, missing source groups") } name := nbResource.Spec.PolicyFriendlyName[policy] if name == "" { name = fmt.Sprintf("Autogenerated policy for resource %s/%s in cluster %s", nbResource.Namespace, nbResource.Name, r.ClusterName) } generatedName := fmt.Sprintf("%s-%s-%s", policy, req.Namespace, req.Name) *nbPolicy = nbv1.NBPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: generatedName, Annotations: map[string]string{"netbird.io/generated-by": req.NamespacedName.String()}, Finalizers: []string{"netbird.io/cleanup"}, Labels: r.DefaultLabels, }, Spec: nbv1.NBPolicySpec{ Name: name, Description: "Generated by " + req.NamespacedName.String(), SourceGroups: nbResource.Spec.PolicySourceGroups, Bidirectional: true, }, } err := r.Client.Create(ctx, nbPolicy) if kerrors.IsAlreadyExists(err) { err = r.Client.Get(ctx, types.NamespacedName{Name: generatedName}, nbPolicy) if err != nil { logger.Error(errKubernetesAPI, "err", err) return err } if nbPolicy.Annotations == nil { nbPolicy.Annotations = make(map[string]string) } nbPolicy.Labels = r.DefaultLabels nbPolicy.Annotations["netbird.io/generated-by"] = req.NamespacedName.String() nbPolicy.Spec = nbv1.NBPolicySpec{ Name: name, Description: "Generated by " + req.NamespacedName.String(), SourceGroups: nbResource.Spec.PolicySourceGroups, Bidirectional: true, } err = r.Client.Update(ctx, nbPolicy) if err != nil { logger.Error(errKubernetesAPI, "err", err) return err } } else if err != nil { logger.Error(errKubernetesAPI, "err", err) return err } if nbResource.Status.PolicyNameMapping == nil { nbResource.Status.PolicyNameMapping = make(map[string]string) } nbResource.Status.PolicyNameMapping[policy] = generatedName nbResource.Status.PolicySourceGroups = nbResource.Spec.PolicySourceGroups nbResource.Status.PolicyFriendlyName = nbResource.Spec.PolicyFriendlyName nbPolicy.Status.ManagedServiceList = append(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) err = r.Client.Status().Update(ctx, nbPolicy) if err != nil { logger.Error(errKubernetesAPI, "err", err) return err } return nil } func (r *NBResourceReconciler) handlePolicyAddUpdate(ctx context.Context, req ctrl.Request, nbResource *nbv1.NBResource, policy string, groupIDs []string, logger logr.Logger) error { var nbPolicy nbv1.NBPolicy updatePolicyStatus := false kubernetesPolicyName := policy if v, ok := nbResource.Status.PolicyNameMapping[policy]; ok { kubernetesPolicyName = v } err := r.Client.Get(ctx, types.NamespacedName{Name: kubernetesPolicyName}, &nbPolicy) if kerrors.IsNotFound(err) && r.AllowAutomaticPolicyCreation { err = r.handlePolicyCreate(ctx, nbResource, req, policy, &nbPolicy, logger) if err != nil { return err } } else if kerrors.IsNotFound(err) && !r.AllowAutomaticPolicyCreation { logger.Info("automatic policy creation is not allowed") return nil } else if err != nil { logger.Error(errKubernetesAPI, "error getting NBPolicy", "err", err, "policyName", policy) return err } if !slices.Contains(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) { nbPolicy.Status.ManagedServiceList = append(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) updatePolicyStatus = true } if !util.Equivalent(nbResource.Spec.TCPPorts, nbResource.Status.TCPPorts) { nbResource.Status.TCPPorts = nbResource.Spec.TCPPorts nbPolicy.Status.LastUpdatedAt = &metav1.Time{Time: time.Now()} updatePolicyStatus = true } if !util.Equivalent(nbResource.Spec.UDPPorts, nbResource.Status.UDPPorts) { nbResource.Status.UDPPorts = nbResource.Spec.UDPPorts nbPolicy.Status.LastUpdatedAt = &metav1.Time{Time: time.Now()} updatePolicyStatus = true } if !util.Equivalent(nbResource.Status.Groups, groupIDs) { nbResource.Status.Groups = groupIDs nbPolicy.Status.LastUpdatedAt = &metav1.Time{Time: time.Now()} updatePolicyStatus = true } if _, ok := nbResource.Status.PolicyNameMapping[policy]; ok { updatePolicySpec := false if v, ok := nbPolicy.Annotations["netbird.io/generated-by"]; !ok || v != req.NamespacedName.String() { if nbPolicy.Annotations == nil { nbPolicy.Annotations = make(map[string]string) } nbPolicy.Annotations["netbird.io/generated-by"] = req.NamespacedName.String() updatePolicySpec = true } if v, ok := nbResource.Spec.PolicyFriendlyName[policy]; ok { if nbPolicy.Spec.Name != v { nbPolicy.Spec.Name = v updatePolicySpec = true } } else { if nbPolicy.Spec.Name != fmt.Sprintf("Autogenerated policy for resource %s/%s in cluster %s", nbResource.Namespace, nbResource.Name, r.ClusterName) { nbPolicy.Spec.Name = fmt.Sprintf("Autogenerated policy for resource %s/%s in cluster %s", nbResource.Namespace, nbResource.Name, r.ClusterName) updatePolicySpec = true } } if nbPolicy.Spec.Description != "Generated by "+req.NamespacedName.String() { nbPolicy.Spec.Description = "Generated by " + req.NamespacedName.String() updatePolicySpec = true } if !util.Equivalent(nbPolicy.Spec.SourceGroups, nbResource.Spec.PolicySourceGroups) { nbPolicy.Spec.SourceGroups = nbResource.Spec.PolicySourceGroups updatePolicySpec = true } if updatePolicySpec { err := r.Client.Update(ctx, &nbPolicy) if err != nil { return err } } } if updatePolicyStatus { err := r.Client.Status().Update(ctx, &nbPolicy) if err != nil { logger.Error(errKubernetesAPI, "error updating NBPolicy", "err", err, "policyName", policy) return err } } return nil } func (r *NBResourceReconciler) handlePolicyDelete(ctx context.Context, req ctrl.Request, nbResource *nbv1.NBResource, specPolicies []string, policy string, logger logr.Logger) error { var nbPolicy nbv1.NBPolicy if !slices.Contains(specPolicies, policy) { kubeName := policy if v, ok := nbResource.Status.PolicyNameMapping[policy]; ok { kubeName = v } err := r.Client.Get(ctx, types.NamespacedName{Name: kubeName}, &nbPolicy) if !kerrors.IsNotFound(err) { if err != nil { logger.Error(errKubernetesAPI, "error getting NBPolicy", "err", err, "policyName", policy) return err } if _, ok := nbResource.Status.PolicyNameMapping[policy]; ok { // Delete Policy err := r.Client.Delete(ctx, &nbPolicy) if err != nil { logger.Error(errKubernetesAPI, "error deleting NBPolicy", "err", err, "policyName", policy) return err } delete(nbResource.Status.PolicyNameMapping, policy) } else if slices.Contains(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) { nbPolicy.Status.ManagedServiceList = util.Without(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) nbPolicy.Status.LastUpdatedAt = &metav1.Time{Time: time.Now()} err := r.Client.Status().Update(ctx, &nbPolicy) if err != nil { logger.Error(errKubernetesAPI, "error updating NBPolicy", "err", err, "policyName", policy) return err } } } } return nil } // handlePolicy update NBPolicy if defined to add self reference to policy status func (r *NBResourceReconciler) handlePolicy(ctx context.Context, req ctrl.Request, nbResource *nbv1.NBResource, groupIDs []string, logger logr.Logger) error { if nbResource.Status.PolicyName == nil && nbResource.Spec.PolicyName == "" { return nil } specPolicies := util.SplitTrim(nbResource.Spec.PolicyName, ",") var statusPolicies []string if nbResource.Status.PolicyName != nil { statusPolicies = util.SplitTrim(*nbResource.Status.PolicyName, ",") } for _, policy := range specPolicies { err := r.handlePolicyAddUpdate(ctx, req, nbResource, policy, groupIDs, logger) if err != nil { return err } } for _, policy := range statusPolicies { err := r.handlePolicyDelete(ctx, req, nbResource, specPolicies, policy, logger) if err != nil { return err } } if nbResource.Status.PolicyName == nil || *nbResource.Status.PolicyName != nbResource.Spec.PolicyName { nbResource.Status.PolicyName = &nbResource.Spec.PolicyName } return nil } // handleGroupUpdate update network resource groups func (r *NBResourceReconciler) handleGroupUpdate(ctx context.Context, nbResource *nbv1.NBResource, groupIDs []string, resource *api.NetworkResource, logger logr.Logger) error { // Handle possible updated group IDs groupIDMap := make(map[string]any) for _, g := range groupIDs { groupIDMap[g] = nil } diffFound := len(groupIDs) != len(resource.Groups) for _, g := range resource.Groups { if _, ok := groupIDMap[g.Id]; !ok { diffFound = true } } if diffFound { _, err := r.Netbird.Networks.Resources(nbResource.Spec.NetworkID).Update(ctx, resource.Id, api.NetworkResourceRequest{ Name: nbResource.Spec.Name, Description: &networkDescription, Address: nbResource.Spec.Address, Enabled: true, Groups: groupIDs, }) if err != nil { logger.Error(errNetBirdAPI, "error updating resource", "err", err) return err } } return nil } // handleNetBirdResource sync NetBird Network Resource func (r *NBResourceReconciler) handleNetBirdResource(ctx context.Context, nbResource *nbv1.NBResource, groupIDs []string, logger logr.Logger) (*api.NetworkResource, error) { var resource *api.NetworkResource var err error if nbResource.Status.NetworkResourceID != nil { resource, err = r.Netbird.Networks.Resources(nbResource.Spec.NetworkID).Get(ctx, *nbResource.Status.NetworkResourceID) if err != nil && !strings.Contains(err.Error(), "not found") { logger.Error(errNetBirdAPI, "error getting network resource", "err", err) return nil, err } } // Create/Update upstream network resource if nbResource.Status.NetworkResourceID == nil && resource == nil { resource, err := r.Netbird.Networks.Resources(nbResource.Spec.NetworkID).Create(ctx, api.NetworkResourceRequest{ Address: nbResource.Spec.Address, Enabled: true, Groups: groupIDs, Description: &networkDescription, Name: nbResource.Spec.Name, }) if err != nil && strings.Contains(err.Error(), "already exists") { log.Log.Error(errNetBirdAPI, "network resource with the same name already exists", "err", err) nbResource.Status.Conditions = nbv1.NBConditionFalse("DuplicateName", "Resource name already exists") return nil, errDuplicateResource } if err != nil { logger.Error(errNetBirdAPI, "error creating resource", "err", err) return nil, err } nbResource.Status.NetworkResourceID = &resource.Id } else if resource == nil { // Status remembers networkResourceID but resource was deleted elsewhere // remove networkID from status and re-enqueue nbResource.Status.NetworkResourceID = nil } else { resourceGroups := make([]string, 0, len(resource.Groups)) for _, v := range resource.Groups { resourceGroups = append(resourceGroups, v.Id) } if resource.Address != nbResource.Spec.Address || !resource.Enabled || !util.Equivalent(resourceGroups, groupIDs) || *resource.Description != networkDescription || resource.Name != nbResource.Spec.Name { _, err = r.Netbird.Networks.Resources(nbResource.Spec.NetworkID).Update(ctx, *nbResource.Status.NetworkResourceID, api.NetworkResourceRequest{ Address: nbResource.Spec.Address, Enabled: true, Groups: groupIDs, Description: &networkDescription, Name: nbResource.Spec.Name, }) if err != nil { return resource, err } } } return resource, nil } // handleGroups create NBGroup objects for each group specified in NBResource func (r *NBResourceReconciler) handleGroups(ctx context.Context, req ctrl.Request, nbResource *nbv1.NBResource, logger logr.Logger) ([]string, *ctrl.Result, error) { nbGroupList := nbv1.NBGroupList{} err := r.Client.List(ctx, &nbGroupList, &client.ListOptions{Namespace: req.Namespace}) if err != nil { logger.Error(errKubernetesAPI, "error listing NBGroup", "err", err) return nil, nil, err } for _, g := range nbGroupList.Items { ownerIndex := -1 for idx, o := range g.OwnerReferences { if o.UID == nbResource.UID { ownerIndex = idx break } } if ownerIndex == -1 { continue } if slices.Contains(nbResource.Spec.Groups, g.Spec.Name) { continue } if len(g.OwnerReferences) > 1 { g.OwnerReferences = slices.Delete(g.OwnerReferences, ownerIndex, ownerIndex+1) err = r.Client.Update(ctx, &g) if err != nil && !kerrors.IsNotFound(err) { logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err) return nil, nil, err } } else if len(g.OwnerReferences) == 1 { g.Finalizers = util.Without(g.Finalizers, "netbird.io/resource-cleanup") err = r.Client.Update(ctx, &g) if err != nil && !kerrors.IsNotFound(err) { logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err) return nil, nil, err } } } var groupIDs []string for _, groupName := range nbResource.Spec.Groups { nbGroup := nbv1.NBGroup{} groupNameRFC := strings.ToLower(groupName) groupNameRFC = strings.ReplaceAll(groupNameRFC, " ", "-") err := r.Client.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: groupNameRFC}, &nbGroup) if err != nil && !kerrors.IsNotFound(err) { logger.Error(errKubernetesAPI, "error getting NBGroup", "err", err) return nil, &ctrl.Result{}, err } else if kerrors.IsNotFound(err) { // Create NBGroup nbGroup = nbv1.NBGroup{ ObjectMeta: metav1.ObjectMeta{ Name: groupNameRFC, Namespace: nbResource.Namespace, OwnerReferences: []metav1.OwnerReference{ { APIVersion: nbv1.GroupVersion.Identifier(), Kind: "NBResource", Name: nbResource.Name, UID: nbResource.UID, BlockOwnerDeletion: new(true), }, }, Finalizers: []string{"netbird.io/group-cleanup", "netbird.io/resource-cleanup"}, Labels: r.DefaultLabels, }, Spec: nbv1.NBGroupSpec{ Name: groupName, }, } err = r.Client.Create(ctx, &nbGroup) if err != nil { logger.Error(errKubernetesAPI, "error creating NBGroup", "err", err) return nil, &ctrl.Result{}, err } continue } else { // Add NBResource as owner to NBGroup if not already done ownerExists := false for _, o := range nbGroup.OwnerReferences { if o.UID == nbResource.UID { ownerExists = true } } if !ownerExists { nbGroup.OwnerReferences = append(nbGroup.OwnerReferences, metav1.OwnerReference{ APIVersion: nbv1.GroupVersion.Identifier(), Kind: "NBResource", Name: nbResource.Name, UID: nbResource.UID, BlockOwnerDeletion: new(true), }) err = r.Client.Update(ctx, &nbGroup) if err != nil { logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err) return nil, &ctrl.Result{}, err } } } if nbGroup.Status.GroupID != nil { groupIDs = append(groupIDs, *nbGroup.Status.GroupID) } } // if not all groups are ready, requeue if len(groupIDs) != len(nbResource.Spec.Groups) { return nil, &ctrl.Result{RequeueAfter: 5 * time.Second}, nil } return groupIDs, nil, nil } func (r *NBResourceReconciler) reconcileDelete(ctx context.Context, req ctrl.Request, nbResource *nbv1.NBResource) error { if nbResource.Status.PolicyName != nil { for _, policy := range util.SplitTrim(*nbResource.Status.PolicyName, ",") { var nbPolicy nbv1.NBPolicy err := r.Client.Get(ctx, types.NamespacedName{Name: policy}, &nbPolicy) if err != nil && !kerrors.IsNotFound(err) { return err } if !kerrors.IsNotFound(err) && slices.Contains(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) { nbPolicy.Status.ManagedServiceList = util.Without(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) nbPolicy.Status.LastUpdatedAt = &metav1.Time{Time: time.Now()} err = r.Client.Status().Update(ctx, &nbPolicy) if err != nil { return err } } } } if nbResource.Status.NetworkResourceID != nil { err := r.Netbird.Networks.Resources(nbResource.Spec.NetworkID).Delete(ctx, *nbResource.Status.NetworkResourceID) if err != nil && !netbird.IsNotFound(err) { return err } } nbGroupList := nbv1.NBGroupList{} err := r.Client.List(ctx, &nbGroupList, &client.ListOptions{Namespace: req.Namespace}) if err != nil { return err } for _, g := range nbGroupList.Items { ownerIndex := -1 for idx, o := range g.OwnerReferences { if o.UID == nbResource.UID { ownerIndex = idx break } } if ownerIndex == -1 { continue } if len(g.OwnerReferences) > 1 { g.OwnerReferences = slices.Delete(g.OwnerReferences, ownerIndex, ownerIndex+1) err = r.Client.Update(ctx, &g) if err != nil && !kerrors.IsNotFound(err) { return err } } else if len(g.OwnerReferences) == 1 { g.Finalizers = util.Without(g.Finalizers, "netbird.io/resource-cleanup") err = r.Client.Update(ctx, &g) if err != nil && !kerrors.IsNotFound(err) { return err } } } // This is needed because finalizers have been added externally in the past. controllerutil.RemoveFinalizer(nbResource, "netbird.io/cleanup") controllerutil.RemoveFinalizer(nbResource, ResourceFinalizer) err = r.Client.Update(ctx, nbResource) if err != nil { return err } return nil } // SetupWithManager sets up the controller with the Manager. func (r *NBResourceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&nbv1.NBResource{}). Watches(&nbv1.NBGroup{}, handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &nbv1.NBResource{})). Watches(&nbv1.NBPolicy{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { if v, ok := obj.GetAnnotations()["netbird.io/generated-by"]; ok { return []reconcile.Request{ { NamespacedName: types.NamespacedName{ Namespace: strings.Split(v, "/")[0], Name: strings.Split(v, "/")[1], }, }, } } return nil })). Complete(r) }