Add network egress resource (#357)

This change adds a new import resource which enables exposing Netbird
resources as Kubernetes services. This remove the need to add sidecars
to every pod.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a new `NetworkEgress` custom resource (`netbird.io/v1alpha1`)
with CRD, schema validation, and status/conditions.
* Extended controller functionality to create egress services and
translate egress rules into import `EndpointSlice` resources; egress
pods now include a kube-egress-forwarder sidecar.
* **Bug Fixes**
* Added missing deep-copy and declarative apply support for the new
`NetworkEgress` API types.
* **Documentation**
* Updated README/API reference and added example manifests for
`NetworkEgress` (including IP/FQDN target usage).
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Signed-off-by: Philip Laine <philip.laine@gmail.com>

internal/controller/forwarderservice_controller.go

@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: BSD-3-Clause
+
+package controller
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"net/netip"
+	"strings"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/types"
+	corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
+	discoveryv1ac "k8s.io/client-go/applyconfigurations/discovery/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"github.com/netbirdio/kube-egress-forwarder/pkg/forwarder"
+
+	nbv1alpha1 "github.com/netbirdio/kubernetes-operator/api/v1alpha1"
+	"github.com/netbirdio/kubernetes-operator/internal/k8sutil"
+)
+
+const (
+	ForwarderRouterNameLabel   = "netbird.io/forwarder-router-name"
+	EgressRouterNameLabel      = "netbird.io/egress-router-name"
+	EgressRouterNamespaceLabel = "netbird.io/egress-router-namespace"
+	LastUpdatedLabel           = "netbird.io/last-updated"
+)
+
+// ForwarderServiceReconciler reconciles a EndpointSlice object
+type ForwarderServiceReconciler struct {
+	client.Client
+}
+
+func (r *ForwarderServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	svc := &corev1.Service{}
+	err := r.Get(ctx, req.NamespacedName, svc)
+	if err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+	if !svc.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+
+	ownerRef, err := k8sutil.ControllerReference(svc, r.Scheme())
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	routerName, ok := svc.Labels[ForwarderRouterNameLabel]
+	if !ok {
+		return ctrl.Result{}, errors.New("missing forwarder router label")
+	}
+
+	// Load port manager state.
+	ruleConfigmap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      req.Name,
+			Namespace: req.Namespace,
+		},
+	}
+	err = r.Client.Get(ctx, client.ObjectKeyFromObject(ruleConfigmap), ruleConfigmap)
+	if err != nil && !kerrors.IsNotFound(err) {
+		return ctrl.Result{}, err
+	}
+	ruleMgr, err := forwarder.NewRuleManager(ruleConfigmap.Data)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Get endpoint slices for forwarder service.
+	endpointSliceList := &discoveryv1.EndpointSliceList{}
+	err = r.Client.List(ctx, endpointSliceList, &client.MatchingLabels{discoveryv1.LabelServiceName: svc.Name})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Get services for egress resources.
+	egressSvcList := &corev1.ServiceList{}
+	err = r.Client.List(ctx, egressSvcList, &client.MatchingLabels{EgressRouterNameLabel: routerName, EgressRouterNamespaceLabel: req.Namespace})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Copy router endpoint slices to egress services.
+	lastUpdated := fmt.Sprintf("%d", time.Now().Unix())
+
+	for _, egressSvc := range egressSvcList.Items {
+		netEgress := &nbv1alpha1.NetworkEgress{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      egressSvc.OwnerReferences[0].Name,
+				Namespace: egressSvc.Namespace,
+			},
+		}
+		err = r.Get(ctx, client.ObjectKeyFromObject(netEgress), netEgress)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+
+		targetPorts := []int32{}
+		for _, port := range egressSvc.Spec.Ports {
+			dest, err := func() (string, error) {
+				switch {
+				case netEgress.Spec.Target.IP != nil:
+					addr, err := netip.ParseAddr(netEgress.Spec.Target.IP.Address)
+					if err != nil {
+						return "", err
+					}
+					return netip.AddrPortFrom(addr, uint16(port.Port)).String(), nil
+				case netEgress.Spec.Target.FQDN != nil:
+					return fmt.Sprintf("%s:%d", netEgress.Spec.Target.FQDN.Hostname, port.Port), nil
+				}
+				return "", errors.New("egress target not found")
+			}()
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+			rule := ruleMgr.Allocate(port.Protocol, dest)
+			targetPorts = append(targetPorts, rule.Port)
+		}
+		portACs := toPortApplyConfigurations(egressSvc.Spec.Ports, targetPorts)
+
+		egressSvcOwnerRef, err := k8sutil.ControllerReference(&egressSvc, r.Scheme())
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		for _, endpointSlice := range endpointSliceList.Items {
+			nameSuffx := strings.TrimPrefix(endpointSlice.Name, endpointSlice.GenerateName)
+
+			labels := map[string]string{
+				discoveryv1.LabelServiceName: egressSvc.Name,
+				discoveryv1.LabelManagedBy:   "netbird-operator.netbird.io",
+				LastUpdatedLabel:             lastUpdated,
+			}
+			maps.Copy(labels, egressSvc.Labels)
+
+			endpointACs := toEndpointApplyConfigurations(endpointSlice.Endpoints)
+			endpointSliceAC := discoveryv1ac.EndpointSlice(fmt.Sprintf("%s-%s", egressSvc.Name, nameSuffx), egressSvc.Namespace).
+				WithLabels(labels).
+				WithOwnerReferences(egressSvcOwnerRef).
+				WithAddressType(endpointSlice.AddressType).
+				WithEndpoints(endpointACs...).
+				WithPorts(portACs...)
+			err = r.Client.Apply(ctx, endpointSliceAC, client.ForceOwnership)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+	}
+
+	// Write the port config to the port conversion.
+	data, err := ruleMgr.Data()
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	cmAC := corev1ac.ConfigMap(req.Name, req.Namespace).
+		WithOwnerReferences(ownerRef).
+		WithData(data)
+	err = r.Client.Apply(ctx, cmAC, client.ForceOwnership)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Cleanup old endpoint slices.
+	updateReq, err := labels.NewRequirement(LastUpdatedLabel, selection.NotEquals, []string{lastUpdated})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	egressNameReq, err := labels.NewRequirement(EgressRouterNameLabel, selection.Equals, []string{routerName})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	egressNamespaceReq, err := labels.NewRequirement(EgressRouterNamespaceLabel, selection.Equals, []string{req.Namespace})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	sourceReq, err := labels.NewRequirement(discoveryv1.LabelServiceName, selection.NotEquals, []string{svc.Name})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	deleteSelector := labels.NewSelector().Add(*updateReq).Add(*egressNameReq).Add(*egressNamespaceReq).Add(*sourceReq)
+
+	err = r.Client.List(ctx, endpointSliceList, client.MatchingLabelsSelector{Selector: deleteSelector})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	for _, item := range endpointSliceList.Items {
+		if err := r.Client.Delete(ctx, &item); err != nil && !kerrors.IsNotFound(err) {
+			return ctrl.Result{}, err
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *ForwarderServiceReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	forwarderSvcSelector := metav1.LabelSelector{
+		MatchExpressions: []metav1.LabelSelectorRequirement{
+			{
+				Key:      ForwarderRouterNameLabel,
+				Operator: metav1.LabelSelectorOpExists,
+			},
+		},
+	}
+	forwarderSvcPred, err := predicate.LabelSelectorPredicate(forwarderSvcSelector)
+	if err != nil {
+		return err
+	}
+
+	egressSvcSelector := metav1.LabelSelector{
+		MatchExpressions: []metav1.LabelSelectorRequirement{
+			{
+				Key:      EgressRouterNameLabel,
+				Operator: metav1.LabelSelectorOpExists,
+			},
+			{
+				Key:      EgressRouterNamespaceLabel,
+				Operator: metav1.LabelSelectorOpExists,
+			},
+		},
+	}
+	egressSvcPred, err := predicate.LabelSelectorPredicate(egressSvcSelector)
+	if err != nil {
+		return err
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		Named("forwarderservice").
+		For(&corev1.Service{}, builder.WithPredicates(forwarderSvcPred)).
+		Owns(&discoveryv1.EndpointSlice{}, builder.WithPredicates(forwarderSvcPred)).
+		Owns(&corev1.ConfigMap{}).
+		Watches(&nbv1alpha1.NetworkEgress{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
+			imp, ok := obj.(*nbv1alpha1.NetworkEgress)
+			if !ok {
+				return nil
+			}
+			return []reconcile.Request{
+				{
+					NamespacedName: types.NamespacedName{
+						Name:      fmt.Sprintf("networkrouter-%s-forwarder", imp.Spec.NetworkRouterRef.Name),
+						Namespace: imp.Spec.NetworkRouterRef.Namespace,
+					},
+				},
+			}
+		})).
+		Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
+			return []reconcile.Request{
+				{
+					NamespacedName: types.NamespacedName{
+						Name:      fmt.Sprintf("networkrouter-%s-forwarder", obj.GetLabels()[EgressRouterNameLabel]),
+						Namespace: obj.GetLabels()[EgressRouterNamespaceLabel],
+					},
+				},
+			}
+		}), builder.WithPredicates(egressSvcPred)).
+		Watches(&discoveryv1.EndpointSlice{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
+			return []reconcile.Request{
+				{
+					NamespacedName: types.NamespacedName{
+						Name:      fmt.Sprintf("networkrouter-%s-forwarder", obj.GetLabels()[EgressRouterNameLabel]),
+						Namespace: obj.GetLabels()[EgressRouterNamespaceLabel],
+					},
+				},
+			}
+		}), builder.WithPredicates(egressSvcPred)).
+		Complete(r)
+}
+
+func toEndpointApplyConfigurations(endpoints []discoveryv1.Endpoint) []*discoveryv1ac.EndpointApplyConfiguration {
+	endpointACs := make([]*discoveryv1ac.EndpointApplyConfiguration, 0, len(endpoints))
+	for _, endpoint := range endpoints {
+		conditionAC := discoveryv1ac.EndpointConditions()
+		if endpoint.Conditions.Ready != nil {
+			conditionAC.WithReady(*endpoint.Conditions.Ready)
+		}
+		if endpoint.Conditions.Serving != nil {
+			conditionAC.WithServing(*endpoint.Conditions.Serving)
+		}
+		if endpoint.Conditions.Terminating != nil {
+			conditionAC.WithTerminating(*endpoint.Conditions.Terminating)
+		}
+
+		endpointAC := discoveryv1ac.Endpoint().
+			WithAddresses(endpoint.Addresses...).
+			WithConditions(conditionAC)
+
+		if endpoint.NodeName != nil {
+			endpointAC = endpointAC.WithNodeName(*endpoint.NodeName)
+		}
+		if endpoint.Zone != nil {
+			endpointAC = endpointAC.WithZone(*endpoint.Zone)
+		}
+		if endpoint.Hints != nil {
+			hintAC := discoveryv1ac.EndpointHints()
+			for _, hint := range endpoint.Hints.ForNodes {
+				hintAC = hintAC.WithForNodes(discoveryv1ac.ForNode().WithName(hint.Name))
+			}
+			for _, hint := range endpoint.Hints.ForZones {
+				hintAC = hintAC.WithForZones(discoveryv1ac.ForZone().WithName(hint.Name))
+			}
+			endpointAC = endpointAC.WithHints(hintAC)
+		}
+		if endpoint.TargetRef != nil {
+			endpointAC = endpointAC.WithTargetRef(corev1ac.ObjectReference().
+				WithResourceVersion(endpoint.TargetRef.ResourceVersion).
+				WithFieldPath(endpoint.TargetRef.FieldPath).
+				WithUID(endpoint.TargetRef.UID).
+				WithAPIVersion(endpoint.TargetRef.APIVersion).
+				WithKind(endpoint.TargetRef.Kind).
+				WithName(endpoint.TargetRef.Name).
+				WithNamespace(endpoint.TargetRef.Namespace),
+			)
+		}
+
+		endpointACs = append(endpointACs, endpointAC)
+	}
+	return endpointACs
+}
+
+func toPortApplyConfigurations(ports []corev1.ServicePort, targetPorts []int32) []*discoveryv1ac.EndpointPortApplyConfiguration {
+	portACs := make([]*discoveryv1ac.EndpointPortApplyConfiguration, 0, len(ports))
+	for i, port := range ports {
+		portAC := discoveryv1ac.EndpointPort().WithName(port.Name).WithPort(targetPorts[i]).WithProtocol(port.Protocol)
+		if port.AppProtocol != nil {
+			portAC = portAC.WithAppProtocol(*port.AppProtocol)
+		}
+		portACs = append(portACs, portAC)
+	}
+	return portACs
+}

internal/controller/networkegress_controller.go

@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: BSD-3-Clause
+
+package controller
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fluxcd/pkg/runtime/conditions"
+	"github.com/fluxcd/pkg/runtime/patch"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	netbird "github.com/netbirdio/netbird/shared/management/client/rest"
+
+	nbv1alpha1 "github.com/netbirdio/kubernetes-operator/api/v1alpha1"
+	"github.com/netbirdio/kubernetes-operator/internal/k8sutil"
+)
+
+// NetworkEgressReconciler reconciles a NetworkEgress object
+type NetworkEgressReconciler struct {
+	client.Client
+
+	Netbird *netbird.Client
+}
+
+// +kubebuilder:rbac:groups=netbird.io,resources=networkegresses,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=netbird.io,resources=networkegresses/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=netbird.io,resources=networkegresses/finalizers,verbs=update
+
+func (r *NetworkEgressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	netEgress := &nbv1alpha1.NetworkEgress{}
+	err := r.Get(ctx, req.NamespacedName, netEgress)
+	if err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+	sp := patch.NewSerialPatcher(netEgress, r.Client)
+
+	if !netEgress.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+
+	ownerRef, err := k8sutil.ControllerReference(netEgress, r.Scheme())
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	netRouter := &nbv1alpha1.NetworkRouter{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      netEgress.Spec.NetworkRouterRef.Name,
+			Namespace: netEgress.Spec.NetworkRouterRef.Namespace,
+		},
+	}
+	err = r.Get(ctx, client.ObjectKeyFromObject(netRouter), netRouter)
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			conditions.MarkFalse(netEgress, nbv1alpha1.ReadyCondition, nbv1alpha1.DependencyReason, "Referenced NetworkRouter cannot be found.")
+			err = sp.Patch(ctx, netEgress)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	// Create service for egress resource to do port remapping.
+	ports := []*corev1ac.ServicePortApplyConfiguration{}
+	for _, port := range netEgress.Spec.Ports {
+		ports = append(ports, corev1ac.ServicePort().WithName(port.Name).WithPort(port.Port))
+	}
+	routerSvcAC := corev1ac.Service(netEgress.Name, netEgress.Namespace).
+		WithLabels(map[string]string{EgressRouterNameLabel: netRouter.Name, EgressRouterNamespaceLabel: netRouter.Namespace}).
+		WithOwnerReferences(ownerRef).
+		WithSpec(
+			corev1ac.ServiceSpec().
+				WithPorts(ports...),
+		)
+	err = r.Client.Apply(ctx, routerSvcAC, client.ForceOwnership)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	conditions.MarkTrue(netEgress, nbv1alpha1.ReadyCondition, nbv1alpha1.ReconciledReason, "")
+	err = sp.Patch(ctx, netEgress, patch.WithStatusObservedGeneration{})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *NetworkEgressReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	err := mgr.GetFieldIndexer().IndexField(context.Background(), &nbv1alpha1.NetworkEgress{}, ".spec.networkRouterRef", func(obj client.Object) []string {
+		netEgress := obj.(*nbv1alpha1.NetworkEgress)
+		ref := netEgress.Spec.NetworkRouterRef
+		if ref.Name == "" {
+			return nil
+		}
+		if ref.Namespace == "" {
+			ref.Namespace = netEgress.Namespace
+		}
+		return []string{fmt.Sprintf("%s/%s", ref.Name, ref.Namespace)}
+	})
+	if err != nil {
+		return err
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&nbv1alpha1.NetworkEgress{}).
+		Owns(&corev1.Service{}).
+		Watches(
+			&nbv1alpha1.NetworkRouter{},
+			handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
+				netEgressList := &nbv1alpha1.NetworkEgressList{}
+				err := r.List(ctx, netEgressList, client.MatchingFields{".spec.networkRouterRef": fmt.Sprintf("%s/%s", obj.GetName(), obj.GetNamespace())})
+				if err != nil {
+					return nil
+				}
+
+				requests := make([]reconcile.Request, len(netEgressList.Items))
+				for i, item := range netEgressList.Items {
+					requests[i] = reconcile.Request{
+						NamespacedName: types.NamespacedName{
+							Name:      item.Name,
+							Namespace: item.Namespace,
+						},
+					}
+				}
+				return requests
+			}),
+			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
+		).
+		Complete(r)
+}

internal/controller/networkegress_controller_test.go

@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: BSD-3-Clause
+
+package controller
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	nbv1alpha1 "github.com/netbirdio/kubernetes-operator/api/v1alpha1"
+)
+
+var _ = Describe("NetworkEgress Controller", func() {
+	Context("When reconciling a resource", func() {
+		ctx := context.Background()
+
+		var netEgressRec *NetworkEgressReconciler
+		var forwarderRec *ForwarderServiceReconciler
+
+		nn := client.ObjectKey{
+			Name:      "test-resource",
+			Namespace: "network-egress",
+		}
+		BeforeEach(func() {
+			netEgressRec = &NetworkEgressReconciler{
+				Client: k8sClient,
+			}
+			forwarderRec = &ForwarderServiceReconciler{
+				Client: k8sClient,
+			}
+
+			ns := &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: nn.Namespace,
+				},
+			}
+			Expect(k8sClient.Create(ctx, ns)).To(Succeed())
+		})
+
+		AfterEach(func() {
+			ns := &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: nn.Namespace,
+				},
+			}
+			err := k8sClient.Get(ctx, client.ObjectKeyFromObject(ns), ns)
+			if kerrors.IsNotFound(err) {
+				return
+			}
+			Expect(err).ToNot(HaveOccurred())
+			Expect(k8sClient.Delete(ctx, ns)).To(Succeed())
+		})
+
+		It("creates a egress service with endpoint slices", func() {
+			netRouter := &nbv1alpha1.NetworkRouter{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "egress-router",
+					Namespace: nn.Namespace,
+				},
+				Spec: nbv1alpha1.NetworkRouterSpec{
+					DNSZoneRef: nbv1alpha1.DNSZoneReference{
+						Name: "foo.bar",
+					},
+				},
+			}
+			Expect(k8sClient.Create(ctx, netRouter)).To(Succeed())
+
+			netEgress := &nbv1alpha1.NetworkEgress{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      nn.Name,
+					Namespace: nn.Namespace,
+				},
+				Spec: nbv1alpha1.NetworkEgressSpec{
+					NetworkRouterRef: nbv1alpha1.CrossNamespaceReference{
+						Name:      netRouter.Name,
+						Namespace: nn.Namespace,
+					},
+					Target: nbv1alpha1.NetworkEgressTarget{
+						FQDN: &nbv1alpha1.NetworkEgressFQDNTarget{
+							Hostname: "example.com",
+						},
+					},
+					Ports: []nbv1alpha1.NetworkEgressPort{
+						{
+							Name: "http",
+							Port: 80,
+						},
+					},
+				},
+			}
+			Expect(k8sClient.Create(ctx, netEgress)).To(Succeed())
+			_, err := netEgressRec.Reconcile(ctx, reconcile.Request{NamespacedName: nn})
+			Expect(err).NotTo(HaveOccurred())
+
+			egressSvc := &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      nn.Name,
+					Namespace: nn.Namespace,
+				},
+			}
+			err = k8sClient.Get(ctx, client.ObjectKeyFromObject(egressSvc), egressSvc)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(egressSvc.Labels[EgressRouterNameLabel]).To(Equal(netRouter.Name))
+			Expect(egressSvc.Labels[EgressRouterNamespaceLabel]).To(Equal(netRouter.Namespace))
+			Expect(egressSvc.Spec.Ports).To(HaveLen(1))
+			Expect(egressSvc.Spec.Ports[0].TargetPort.String()).To(Equal("80"))
+
+			_, err = forwarderRec.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKey{Name: fmt.Sprintf("networkrouter-%s-forwarder", netRouter.Name), Namespace: nn.Namespace}})
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+})

internal/controller/networkrouter_controller.go

@@ -15,6 +15,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,6 +25,7 @@ import (
 	corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
 	metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
 	policyv1ac "k8s.io/client-go/applyconfigurations/policy/v1"
+	rbacv1ac "k8s.io/client-go/applyconfigurations/rbac/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -35,6 +37,7 @@ import (
 	"github.com/netbirdio/kubernetes-operator/internal/k8sutil"
 	"github.com/netbirdio/kubernetes-operator/internal/netbirdutil"
 	nbv1alpha1ac "github.com/netbirdio/kubernetes-operator/pkg/applyconfigurations/api/v1alpha1"
+	"github.com/netbirdio/kubernetes-operator/pkg/version"
 )
 
 type NetworkRouterReconciler struct {
@@ -191,7 +194,7 @@ func (r *NetworkRouterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, err
 	}
 
-	// Create the deployment.
+	// Setup router configuration.
 	selectorLabels := map[string]string{
 		"app.kubernetes.io/name":     "networkrouter",
 		"app.kubernetes.io/instance": req.Name,
@@ -207,6 +210,44 @@ func (r *NetworkRouterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		clientImage = netRouter.Spec.Image
 	}
 
+	// Create forwarder resources.
+	forwarderName := fmt.Sprintf("networkrouter-%s-forwarder", req.Name)
+	forwarderSvcAC := corev1ac.Service(forwarderName, req.Namespace).
+		WithLabels(map[string]string{ForwarderRouterNameLabel: netRouter.Name}).
+		WithOwnerReferences(ownerRef).
+		WithSpec(
+			corev1ac.ServiceSpec().
+				WithClusterIP("None").
+				WithSelector(selectorLabels),
+		)
+	err = r.Client.Apply(ctx, forwarderSvcAC, client.ForceOwnership)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	saAC := corev1ac.ServiceAccount(forwarderName, req.Namespace).
+		WithOwnerReferences(ownerRef)
+	err = r.Client.Apply(ctx, saAC, client.ForceOwnership)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	roleAC := rbacv1ac.Role(forwarderName, req.Namespace).
+		WithOwnerReferences(ownerRef).
+		WithRules(rbacv1ac.PolicyRule().WithAPIGroups("").WithResources("configmaps").WithVerbs("get", "watch"))
+	err = r.Client.Apply(ctx, roleAC, client.ForceOwnership)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	roleBindAC := rbacv1ac.RoleBinding(forwarderName, req.Namespace).
+		WithOwnerReferences(ownerRef).
+		WithSubjects(rbacv1ac.Subject().WithKind(*saAC.Kind).WithName(*saAC.Name)).
+		WithRoleRef(rbacv1ac.RoleRef().WithKind(*roleAC.Kind).WithName(*roleAC.Name))
+	err = r.Client.Apply(ctx, roleBindAC, client.ForceOwnership)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Create the deployment.
 	podTemplateSpecAC := corev1ac.PodTemplateSpec().
 		WithLabels(selectorLabels).
 		WithSpec(corev1ac.PodSpec().
@@ -232,65 +273,84 @@ func (r *NetworkRouterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 					WithReadOnlyRootFilesystem(true),
 				),
 			).
-			WithContainers(corev1ac.Container().
-				WithName("netbird").
-				WithImage(clientImage).
-				WithEnv(
-					corev1ac.EnvVar().
-						WithName("NB_SETUP_KEY").
-						WithValueFrom(corev1ac.EnvVarSource().
-							WithSecretKeyRef(corev1ac.SecretKeySelector().
-								WithName(setupKey.SecretName()).
-								WithKey(SetupKeySecretKey),
+			WithContainers(
+				corev1ac.Container().
+					WithName("netbird").
+					WithImage(clientImage).
+					WithEnv(
+						corev1ac.EnvVar().
+							WithName("NB_SETUP_KEY").
+							WithValueFrom(corev1ac.EnvVarSource().
+								WithSecretKeyRef(corev1ac.SecretKeySelector().
+									WithName(setupKey.SecretName()).
+									WithKey(SetupKeySecretKey),
+								),
 							),
-						),
-					corev1ac.EnvVar().
-						WithName("NB_MANAGEMENT_URL").
-						WithValue(r.ManagementURL),
-					corev1ac.EnvVar().
-						WithName("NB_LOG_LEVEL").
-						WithValue(logLevel),
-					corev1ac.EnvVar().
-						WithName("NB_LOG_FILE").
-						WithValue("console"),
-					corev1ac.EnvVar().
-						WithName("NB_DISABLE_PROFILES").
-						WithValue("true"),
-					corev1ac.EnvVar().
-						WithName("NB_DISABLE_UPDATE_SETTINGS").
-						WithValue("true"),
-					corev1ac.EnvVar().
-						WithName("NB_DAEMON_ADDR").
-						WithValue("unix:///var/run/netbird/netbird.sock"),
-					corev1ac.EnvVar().
-						WithName("NB_ENTRYPOINT_SERVICE_TIMEOUT").
-						WithValue("0"),
-				).
-				WithStartupProbe(corev1ac.Probe().WithExec(corev1ac.ExecAction().WithCommand("netbird", "status", "--check", "startup"))).
-				WithReadinessProbe(corev1ac.Probe().WithExec(corev1ac.ExecAction().WithCommand("netbird", "status", "--check", "ready"))).
-				WithVolumeMounts(
-					corev1ac.VolumeMount().WithName("netbird-run").WithMountPath("/var/run/netbird"),
-					corev1ac.VolumeMount().WithName("netbird-lib").WithMountPath("/var/lib/netbird"),
-					corev1ac.VolumeMount().WithName("ssh-etc").WithMountPath("/etc/ssh"),
-					corev1ac.VolumeMount().WithName("resolv-conf").WithMountPath("/etc/resolv.conf").WithSubPath("resolv.conf"),
-					corev1ac.VolumeMount().WithName("resolv-conf").WithMountPath("/etc/resolv.conf.original.netbird").WithSubPath("resolv.conf.original.netbird"),
-				).
-				WithSecurityContext(corev1ac.SecurityContext().
-					WithReadOnlyRootFilesystem(true).
-					WithCapabilities(corev1ac.Capabilities().
-						WithAdd("NET_ADMIN").
-						WithAdd("SYS_RESOURCE").
-						WithAdd("SYS_ADMIN"),
+						corev1ac.EnvVar().
+							WithName("NB_MANAGEMENT_URL").
+							WithValue(r.ManagementURL),
+						corev1ac.EnvVar().
+							WithName("NB_LOG_LEVEL").
+							WithValue(logLevel),
+						corev1ac.EnvVar().
+							WithName("NB_LOG_FILE").
+							WithValue("console"),
+						corev1ac.EnvVar().
+							WithName("NB_DISABLE_PROFILES").
+							WithValue("true"),
+						corev1ac.EnvVar().
+							WithName("NB_DISABLE_UPDATE_SETTINGS").
+							WithValue("true"),
+						corev1ac.EnvVar().
+							WithName("NB_DAEMON_ADDR").
+							WithValue("unix:///var/run/netbird/netbird.sock"),
+						corev1ac.EnvVar().
+							WithName("NB_ENTRYPOINT_SERVICE_TIMEOUT").
+							WithValue("0"),
 					).
-					WithPrivileged(true),
-				).
-				WithResources(corev1ac.ResourceRequirements().
-					WithRequests(corev1.ResourceList{
-						corev1.ResourceCPU:    resource.MustParse("100m"),
-						corev1.ResourceMemory: resource.MustParse("128Mi"),
-					}),
-				),
+					WithStartupProbe(corev1ac.Probe().WithExec(corev1ac.ExecAction().WithCommand("netbird", "status", "--check", "startup"))).
+					WithReadinessProbe(corev1ac.Probe().WithExec(corev1ac.ExecAction().WithCommand("netbird", "status", "--check", "ready"))).
+					WithVolumeMounts(
+						corev1ac.VolumeMount().WithName("netbird-run").WithMountPath("/var/run/netbird"),
+						corev1ac.VolumeMount().WithName("netbird-lib").WithMountPath("/var/lib/netbird"),
+						corev1ac.VolumeMount().WithName("ssh-etc").WithMountPath("/etc/ssh"),
+						corev1ac.VolumeMount().WithName("resolv-conf").WithMountPath("/etc/resolv.conf").WithSubPath("resolv.conf"),
+						corev1ac.VolumeMount().WithName("resolv-conf").WithMountPath("/etc/resolv.conf.original.netbird").WithSubPath("resolv.conf.original.netbird"),
+					).
+					WithSecurityContext(corev1ac.SecurityContext().
+						WithReadOnlyRootFilesystem(true).
+						WithCapabilities(corev1ac.Capabilities().
+							WithAdd("NET_ADMIN").
+							WithAdd("SYS_RESOURCE").
+							WithAdd("SYS_ADMIN"),
+						).
+						WithPrivileged(true),
+					).
+					WithResources(corev1ac.ResourceRequirements().
+						WithRequests(corev1.ResourceList{
+							corev1.ResourceCPU:    resource.MustParse("100m"),
+							corev1.ResourceMemory: resource.MustParse("128Mi"),
+						}),
+					),
+				corev1ac.Container().
+					WithName("kube-egress-forwarder").
+					WithImage(version.KubeEgressForwarderImage).
+					WithArgs("--configmap-name", forwarderName, "--configmap-namespace", req.Namespace).
+					WithVolumeMounts(
+						corev1ac.VolumeMount().WithName("resolv-conf").WithMountPath("/etc/resolv.conf").WithSubPath("resolv.conf"),
+					).
+					WithSecurityContext(corev1ac.SecurityContext().
+						WithCapabilities(corev1ac.Capabilities().WithDrop("ALL")).
+						WithReadOnlyRootFilesystem(true),
+					).
+					WithResources(corev1ac.ResourceRequirements().
+						WithRequests(corev1.ResourceList{
+							corev1.ResourceCPU:    resource.MustParse("100m"),
+							corev1.ResourceMemory: resource.MustParse("128Mi"),
+						}),
+					),
 			).
+			WithServiceAccountName(*saAC.Name).
 			WithVolumes(
 				corev1ac.Volume().WithName("netbird-run").WithEmptyDir(corev1ac.EmptyDirVolumeSource()),
 				corev1ac.Volume().WithName("netbird-lib").WithEmptyDir(corev1ac.EmptyDirVolumeSource()),
@@ -421,5 +481,9 @@ func (r *NetworkRouterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&nbv1alpha1.Group{}).
 		Owns(&nbv1alpha1.SetupKey{}).
 		Owns(&appsv1.Deployment{}).
+		Owns(&corev1.Service{}).
+		Owns(&corev1.ServiceAccount{}).
+		Owns(&rbacv1.Role{}).
+		Owns(&rbacv1.RoleBinding{}).
 		Complete(r)
 }