netbird-kubernetes-operator/internal/controller/networkrouter_controller.go

// SPDX-License-Identifier: BSD-3-Clause

package controller

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"maps"
	"time"

	"github.com/fluxcd/pkg/runtime/conditions"
	"github.com/fluxcd/pkg/runtime/patch"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	policyv1 "k8s.io/api/policy/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1"
	corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
	metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
	policyv1ac "k8s.io/client-go/applyconfigurations/policy/v1"
	rbacv1ac "k8s.io/client-go/applyconfigurations/rbac/v1"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

	netbird "github.com/netbirdio/netbird/shared/management/client/rest"
	"github.com/netbirdio/netbird/shared/management/http/api"

	nbv1alpha1 "github.com/netbirdio/kubernetes-operator/api/v1alpha1"
	"github.com/netbirdio/kubernetes-operator/internal/k8sutil"
	"github.com/netbirdio/kubernetes-operator/internal/netbirdutil"
	nbv1alpha1ac "github.com/netbirdio/kubernetes-operator/pkg/applyconfigurations/api/v1alpha1"
	"github.com/netbirdio/kubernetes-operator/pkg/version"
)

type NetworkRouterReconciler struct {
	client.Client

	Netbird       *netbird.Client
	ManagementURL string
	ClientImage   string
}

// +kubebuilder:rbac:groups=netbird.io,resources=networkrouters,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=netbird.io,resources=networkrouters/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=netbird.io,resources=networkrouters/finalizers,verbs=update

// nolint:gocyclo
func (r *NetworkRouterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	netRouter := &nbv1alpha1.NetworkRouter{}
	err := r.Get(ctx, req.NamespacedName, netRouter)
	if err != nil {
		return ctrl.Result{}, client.IgnoreNotFound(err)
	}
	sp := patch.NewSerialPatcher(netRouter, r.Client)

	if !netRouter.DeletionTimestamp.IsZero() {
		return r.reconcileDelete(ctx, sp, netRouter)
	}

	ownerRef, err := k8sutil.ControllerReference(netRouter, r.Scheme())
	if err != nil {
		return ctrl.Result{}, err
	}

	// Ensure the DNS Zone exists.
	_, err = netbirdutil.GetDNSZoneByName(ctx, r.Netbird, netRouter.Spec.DNSZoneRef.Name)
	if err != nil {
		return ctrl.Result{}, err
	}

	controllerutil.AddFinalizer(netRouter, k8sutil.Finalizer("networkrouter"))

	networkID, err := func() (string, error) {
		networkReq := api.NetworkRequest{
			Name: netRouter.Name,
		}
		if netRouter.Status.NetworkID != "" {
			networkResp, err := r.Netbird.Networks.Update(ctx, netRouter.Status.NetworkID, networkReq)
			if err != nil && !netbird.IsNotFound(err) {
				return "", err
			}
			if err == nil {
				return networkResp.Id, nil
			}
		}
		networkResp, err := r.Netbird.Networks.Create(ctx, networkReq)
		if err != nil {
			return "", err
		}
		return networkResp.Id, nil
	}()
	if err != nil {
		return ctrl.Result{}, err
	}
	netRouter.Status.NetworkID = networkID
	err = sp.Patch(ctx, netRouter)
	if err != nil {
		return ctrl.Result{}, err
	}

	// Calculate unique suffix used for Netbird resources.
	sum := sha256.Sum256([]byte(netRouter.UID))
	uniqueSuffix := networkID + "-" + fmt.Sprintf("%x", sum[:4])[:8]

	// Create the group used by the router to discover peers.
	groupAC := nbv1alpha1ac.Group(fmt.Sprintf("networkrouter-%s", netRouter.Name), req.Namespace).
		WithOwnerReferences(ownerRef).
		WithSpec(
			nbv1alpha1ac.GroupSpec().
				WithName(fmt.Sprintf("networkrouter-%s", uniqueSuffix)),
		)
	err = r.Client.Apply(ctx, groupAC, client.ForceOwnership)
	if err != nil {
		return ctrl.Result{}, err
	}
	group := &nbv1alpha1.Group{
		ObjectMeta: metav1.ObjectMeta{
			Name:      *groupAC.Name,
			Namespace: *groupAC.Namespace,
		},
	}
	err = r.Client.Get(ctx, client.ObjectKeyFromObject(group), group)
	if err != nil {
		return ctrl.Result{}, client.IgnoreNotFound(err)
	}
	if group.Status.GroupID == "" {
		return ctrl.Result{}, nil
	}

	// Create the setup key used by routing peers.
	setupKeyAC := nbv1alpha1ac.SetupKey(fmt.Sprintf("networkrouter-%s", netRouter.Name), req.Namespace).
		WithOwnerReferences(ownerRef).
		WithSpec(
			nbv1alpha1ac.SetupKeySpec().
				WithName(fmt.Sprintf("networkrouter-%s", uniqueSuffix)).
				WithEphemeral(true).
				WithAutoGroups(nbv1alpha1ac.GroupReference().WithID(group.Status.GroupID)),
		)
	err = r.Client.Apply(ctx, setupKeyAC, client.ForceOwnership)
	if err != nil {
		return ctrl.Result{}, err
	}
	setupKey := nbv1alpha1.SetupKey{
		ObjectMeta: metav1.ObjectMeta{
			Name:      *setupKeyAC.Name,
			Namespace: *setupKeyAC.Namespace,
		},
	}
	err = r.Get(ctx, client.ObjectKeyFromObject(&setupKey), &setupKey)
	if err != nil {
		return ctrl.Result{}, client.IgnoreNotFound(err)
	}
	if setupKey.Status.SetupKeyID == "" {
		return ctrl.Result{}, nil
	}

	// Create the routing peer in netbird.
	routingPeerID, err := func() (string, error) {
		routerReq := api.NetworkRouterRequest{
			Enabled:    true,
			Masquerade: true,
			Metric:     9999,
			PeerGroups: new([]string{group.Status.GroupID}),
		}
		if netRouter.Status.RoutingPeerID != "" {
			resp, err := r.Netbird.Networks.Routers(networkID).Update(ctx, netRouter.Status.RoutingPeerID, routerReq)
			if err != nil && !netbird.IsNotFound(err) {
				return "", err
			}
			if err == nil {
				return resp.Id, nil
			}
		}
		resp, err := r.Netbird.Networks.Routers(networkID).Create(ctx, routerReq)
		if err != nil {
			return "", err
		}
		return resp.Id, nil
	}()
	if err != nil {
		return ctrl.Result{}, err
	}
	netRouter.Status.RoutingPeerID = routingPeerID
	err = sp.Patch(ctx, netRouter, patch.WithStatusObservedGeneration{})
	if err != nil {
		return ctrl.Result{}, err
	}

	// Setup router configuration.
	selectorLabels := map[string]string{
		"app.kubernetes.io/name":     "networkrouter",
		"app.kubernetes.io/instance": req.Name,
	}

	logLevel := "info"
	if netRouter.Spec.LogLevel != "" {
		logLevel = netRouter.Spec.LogLevel
	}

	clientImage := r.ClientImage
	if netRouter.Spec.Image != "" {
		clientImage = netRouter.Spec.Image
	}

	// Create forwarder resources.
	forwarderName := fmt.Sprintf("networkrouter-%s-forwarder", req.Name)
	forwarderSvcAC := corev1ac.Service(forwarderName, req.Namespace).
		WithLabels(map[string]string{ForwarderRouterNameLabel: netRouter.Name}).
		WithOwnerReferences(ownerRef).
		WithSpec(
			corev1ac.ServiceSpec().
				WithClusterIP("None").
				WithSelector(selectorLabels),
		)
	err = r.Client.Apply(ctx, forwarderSvcAC, client.ForceOwnership)
	if err != nil {
		return ctrl.Result{}, err
	}

	saAC := corev1ac.ServiceAccount(forwarderName, req.Namespace).
		WithOwnerReferences(ownerRef)
	err = r.Client.Apply(ctx, saAC, client.ForceOwnership)
	if err != nil {
		return ctrl.Result{}, err
	}
	roleAC := rbacv1ac.Role(forwarderName, req.Namespace).
		WithOwnerReferences(ownerRef).
		WithRules(rbacv1ac.PolicyRule().WithAPIGroups("").WithResources("configmaps").WithVerbs("get", "watch"))
	err = r.Client.Apply(ctx, roleAC, client.ForceOwnership)
	if err != nil {
		return ctrl.Result{}, err
	}
	roleBindAC := rbacv1ac.RoleBinding(forwarderName, req.Namespace).
		WithOwnerReferences(ownerRef).
		WithSubjects(rbacv1ac.Subject().WithKind(*saAC.Kind).WithName(*saAC.Name)).
		WithRoleRef(rbacv1ac.RoleRef().WithKind(*roleAC.Kind).WithName(*roleAC.Name))
	err = r.Client.Apply(ctx, roleBindAC, client.ForceOwnership)
	if err != nil {
		return ctrl.Result{}, err
	}

	// Create the deployment.
	podTemplateSpecAC := corev1ac.PodTemplateSpec().
		WithLabels(selectorLabels).
		WithSpec(corev1ac.PodSpec().
			WithTopologySpreadConstraints(
				corev1ac.TopologySpreadConstraint().
					WithMaxSkew(1).
					WithTopologyKey(corev1.LabelHostname).
					WithWhenUnsatisfiable(corev1.ScheduleAnyway).
					WithLabelSelector(metav1ac.LabelSelector().
						WithMatchLabels(selectorLabels),
					),
			).
			WithInitContainers(corev1ac.Container().
				WithName("resolv-conf").
				WithImage(clientImage).
				WithCommand("sh", "-c", "cp /etc/resolv.conf /tmp/resolv.conf && cp /etc/resolv.conf /tmp/resolv.conf.original.netbird").
				WithVolumeMounts(corev1ac.VolumeMount().
					WithName("resolv-conf").
					WithMountPath("/tmp"),
				).
				WithSecurityContext(corev1ac.SecurityContext().
					WithCapabilities(corev1ac.Capabilities().WithDrop("ALL")).
					WithReadOnlyRootFilesystem(true),
				),
			).
			WithContainers(
				corev1ac.Container().
					WithName("netbird").
					WithImage(clientImage).
					WithEnv(
						corev1ac.EnvVar().
							WithName("NB_SETUP_KEY").
							WithValueFrom(corev1ac.EnvVarSource().
								WithSecretKeyRef(corev1ac.SecretKeySelector().
									WithName(setupKey.SecretName()).
									WithKey(SetupKeySecretKey),
								),
							),
						corev1ac.EnvVar().
							WithName("NB_MANAGEMENT_URL").
							WithValue(r.ManagementURL),
						corev1ac.EnvVar().
							WithName("NB_LOG_LEVEL").
							WithValue(logLevel),
						corev1ac.EnvVar().
							WithName("NB_LOG_FILE").
							WithValue("console"),
						corev1ac.EnvVar().
							WithName("NB_DISABLE_PROFILES").
							WithValue("true"),
						corev1ac.EnvVar().
							WithName("NB_DAEMON_ADDR").
							WithValue("unix:///var/run/netbird/netbird.sock"),
						corev1ac.EnvVar().
							WithName("NB_ENTRYPOINT_SERVICE_TIMEOUT").
							WithValue("0"),
					).
					WithStartupProbe(corev1ac.Probe().WithExec(corev1ac.ExecAction().WithCommand("netbird", "status", "--check", "startup"))).
					WithReadinessProbe(corev1ac.Probe().WithExec(corev1ac.ExecAction().WithCommand("netbird", "status", "--check", "ready"))).
					WithVolumeMounts(
						corev1ac.VolumeMount().WithName("netbird-run").WithMountPath("/var/run/netbird"),
						corev1ac.VolumeMount().WithName("netbird-lib").WithMountPath("/var/lib/netbird"),
						corev1ac.VolumeMount().WithName("ssh-etc").WithMountPath("/etc/ssh"),
						corev1ac.VolumeMount().WithName("resolv-conf").WithMountPath("/etc/resolv.conf").WithSubPath("resolv.conf"),
						corev1ac.VolumeMount().WithName("resolv-conf").WithMountPath("/etc/resolv.conf.original.netbird").WithSubPath("resolv.conf.original.netbird"),
					).
					WithSecurityContext(corev1ac.SecurityContext().
						WithReadOnlyRootFilesystem(true).
						WithCapabilities(corev1ac.Capabilities().
							WithAdd("NET_ADMIN").
							WithAdd("SYS_RESOURCE").
							WithAdd("SYS_ADMIN"),
						).
						WithPrivileged(true),
					).
					WithResources(corev1ac.ResourceRequirements().
						WithRequests(corev1.ResourceList{
							corev1.ResourceCPU:    resource.MustParse("100m"),
							corev1.ResourceMemory: resource.MustParse("128Mi"),
						}),
					),
				corev1ac.Container().
					WithName("kube-egress-forwarder").
					WithImage(version.KubeEgressForwarderImage).
					WithArgs("--configmap-name", forwarderName, "--configmap-namespace", req.Namespace).
					WithVolumeMounts(
						corev1ac.VolumeMount().WithName("resolv-conf").WithMountPath("/etc/resolv.conf").WithSubPath("resolv.conf"),
					).
					WithSecurityContext(corev1ac.SecurityContext().
						WithCapabilities(corev1ac.Capabilities().WithDrop("ALL")).
						WithReadOnlyRootFilesystem(true),
					).
					WithResources(corev1ac.ResourceRequirements().
						WithRequests(corev1.ResourceList{
							corev1.ResourceCPU:    resource.MustParse("100m"),
							corev1.ResourceMemory: resource.MustParse("128Mi"),
						}),
					),
			).
			WithServiceAccountName(*saAC.Name).
			WithVolumes(
				corev1ac.Volume().WithName("netbird-run").WithEmptyDir(corev1ac.EmptyDirVolumeSource()),
				corev1ac.Volume().WithName("netbird-lib").WithEmptyDir(corev1ac.EmptyDirVolumeSource()),
				corev1ac.Volume().WithName("ssh-etc").WithEmptyDir(corev1ac.EmptyDirVolumeSource()),
				corev1ac.Volume().WithName("resolv-conf").WithEmptyDir(corev1ac.EmptyDirVolumeSource()),
			),
		)

	workloadLabels := map[string]string{}
	workloadAnnotations := map[string]string{}
	replicas := int32(3)
	if netRouter.Spec.WorkloadOverride != nil {
		if netRouter.Spec.WorkloadOverride.Labels != nil {
			workloadLabels = netRouter.Spec.WorkloadOverride.Labels
		}
		if netRouter.Spec.WorkloadOverride.Annotations != nil {
			workloadAnnotations = netRouter.Spec.WorkloadOverride.Annotations
		}
		if netRouter.Spec.WorkloadOverride.Replicas != nil {
			replicas = *netRouter.Spec.WorkloadOverride.Replicas
		}
		if netRouter.Spec.WorkloadOverride.PodTemplate != nil {
			baseJSON, err := json.Marshal(&podTemplateSpecAC)
			if err != nil {
				return ctrl.Result{}, err
			}
			overrideJSON, err := json.Marshal(netRouter.Spec.WorkloadOverride.PodTemplate)
			if err != nil {
				return ctrl.Result{}, err
			}
			mergedJSON, err := strategicpatch.StrategicMergePatch(baseJSON, overrideJSON, corev1.PodTemplateSpec{})
			if err != nil {
				return ctrl.Result{}, err
			}
			err = json.Unmarshal(mergedJSON, &podTemplateSpecAC)
			if err != nil {
				return ctrl.Result{}, err
			}
		}
	}
	maps.Copy(workloadLabels, selectorLabels)

	depAC := appsv1ac.Deployment(fmt.Sprintf("networkrouter-%s", req.Name), req.Namespace).
		WithOwnerReferences(ownerRef).
		WithLabels(workloadLabels).
		WithAnnotations(workloadAnnotations).
		WithSpec(appsv1ac.DeploymentSpec().WithReplicas(replicas).WithSelector(metav1ac.LabelSelector().WithMatchLabels(selectorLabels)).WithTemplate(podTemplateSpecAC))
	err = r.Client.Apply(ctx, depAC, client.ForceOwnership)
	if err != nil {
		return ctrl.Result{}, err
	}

	if replicas > 1 {
		pdbAC := policyv1ac.PodDisruptionBudget(fmt.Sprintf("networkrouter-%s", req.Name), req.Namespace).
			WithOwnerReferences(ownerRef).
			WithLabels(workloadLabels).
			WithAnnotations(workloadAnnotations).
			WithSpec(policyv1ac.PodDisruptionBudgetSpec().
				WithMaxUnavailable(intstr.FromInt(1)).
				WithSelector(metav1ac.LabelSelector().
					WithMatchLabels(selectorLabels),
				),
			)
		err = r.Client.Apply(ctx, pdbAC, client.ForceOwnership)
		if err != nil {
			return ctrl.Result{}, err
		}
	} else {
		pdb := policyv1.PodDisruptionBudget{
			ObjectMeta: metav1.ObjectMeta{
				Name:      fmt.Sprintf("networkrouter-%s", req.Name),
				Namespace: req.Namespace,
			},
		}
		err = r.Client.Delete(ctx, &pdb)
		if err != nil && !kerrors.IsNotFound(err) {
			return ctrl.Result{}, err
		}
	}

	dep := &appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      *depAC.Name,
			Namespace: *depAC.Namespace,
		},
	}
	err = r.Client.Get(ctx, client.ObjectKeyFromObject(dep), dep)
	if err != nil {
		return ctrl.Result{}, client.IgnoreNotFound(err)
	}
	if dep.Status.ReadyReplicas != dep.Status.Replicas {
		return ctrl.Result{}, nil
	}

	conditions.MarkTrue(netRouter, nbv1alpha1.ReadyCondition, nbv1alpha1.ReconciledReason, "")
	err = sp.Patch(ctx, netRouter, patch.WithStatusObservedGeneration{})
	if err != nil {
		return ctrl.Result{}, err
	}
	return ctrl.Result{RequeueAfter: 15 * time.Minute}, nil
}

func (r *NetworkRouterReconciler) reconcileDelete(ctx context.Context, sp *patch.SerialPatcher, netRouter *nbv1alpha1.NetworkRouter) (ctrl.Result, error) {
	if netRouter.Status.RoutingPeerID != "" {
		err := r.Netbird.Networks.Routers(netRouter.Status.NetworkID).Delete(ctx, netRouter.Status.RoutingPeerID)
		if err != nil && !netbird.IsNotFound(err) {
			return ctrl.Result{}, err
		}
	}
	if netRouter.Status.NetworkID != "" {
		err := r.Netbird.Networks.Delete(ctx, netRouter.Status.NetworkID)
		if err != nil && !netbird.IsNotFound(err) {
			return ctrl.Result{}, err
		}
	}

	controllerutil.RemoveFinalizer(netRouter, k8sutil.Finalizer("networkrouter"))
	err := sp.Patch(ctx, netRouter)
	if err != nil {
		return ctrl.Result{}, err
	}
	return ctrl.Result{}, nil
}

func (r *NetworkRouterReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&nbv1alpha1.NetworkRouter{}).
		Owns(&nbv1alpha1.Group{}).
		Owns(&nbv1alpha1.SetupKey{}).
		Owns(&appsv1.Deployment{}).
		Owns(&corev1.Service{}).
		Owns(&corev1.ServiceAccount{}).
		Owns(&rbacv1.Role{}).
		Owns(&rbacv1.RoleBinding{}).
		Complete(r)
}