Add support for private gateway (#154)

This change adds support for TCPRoutes when using a private gateway
class. This is similar to annotating services today. It also moves the
gateway classes to the Helm chart as it makes things a lot simpler for
the end user as they no longer have to define them.

Signed-off-by: Philip Laine <philip.laine@gmail.com>

internal/controller/gateway_controller.go

@@ -18,10 +18,8 @@ package controller
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"slices"
-	"strings"
 	"time"
 
 	netbird "github.com/netbirdio/netbird/shared/management/client/rest"
@@ -35,6 +33,7 @@ import (
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
+	"github.com/netbirdio/kubernetes-operator/internal/gatewayutil"
 )
 
 const (
@@ -74,7 +73,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	}
 
 	// Verify Gateway configuration.
-	routingPeerName, err := getRoutingPeerName(gw.Spec.Listeners)
+	routingPeerName, err := gatewayutil.GetRoutingPeerName(gw.Spec.Listeners)
 	if err != nil {
 		cond := metav1.Condition{
 			Type:    string(gatewayv1.GatewayConditionAccepted),
@@ -111,8 +110,8 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	}
 
 	// Ensure routing peer is ready.
-	nbrp := &netbirdiov1.NBRoutingPeer{}
-	err = r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: routingPeerName}, nbrp)
+	// TODO (phillebaba): Should watch routing peer instead of retrying when not found.
+	nbrp, err := gatewayutil.GetGatewayRoutingPeer(ctx, r.Client, gw)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -195,17 +194,3 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&gatewayv1.Gateway{}).
 		Complete(r)
 }
-
-func getRoutingPeerName(listeners []gatewayv1.Listener) (string, error) {
-	if len(listeners) > 1 {
-		return "", errors.New("netbird Gateway only supports a single listener")
-	}
-	group, kind, ok := strings.Cut(string(listeners[0].Protocol), "/")
-	if !ok {
-		return "", fmt.Errorf("invalid protocol %s, expected gateway.netbird.io/NBRoutingPeer", listeners[0].Protocol)
-	}
-	if group != "gateway.netbird.io" || kind != "NBRoutingPeer" {
-		return "", fmt.Errorf("invalid group %s and kind %s, expected gateway.netbird.io/NBRoutingPeer", group, kind)
-	}
-	return string(listeners[0].Name), nil
-}

internal/controller/gatewayclass_controller.go

@@ -39,12 +39,21 @@ func (r *GatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	}
 
 	// Validate configuration.
-	if gwc.Spec.ParametersRef != nil {
+	message := func() string {
+		if gwc.Name != "netbird-public" && gwc.Name != "netbird-private" {
+			return "GatewayClass name must be netbird-public or netbird-private."
+		}
+		if gwc.Spec.ParametersRef != nil {
+			return "Parameters references is not supported."
+		}
+		return ""
+	}()
+	if message != "" {
 		cond := metav1.Condition{
 			Type:    string(gatewayv1.GatewayClassConditionStatusAccepted),
 			Status:  metav1.ConditionFalse,
 			Reason:  string(gatewayv1.GatewayClassReasonInvalidParameters),
-			Message: "Parameters references is not supported.",
+			Message: message,
 		}
 		if meta.SetStatusCondition(&gwc.Status.Conditions, cond) {
 			err = r.Client.Status().Update(ctx, &gwc)

internal/controller/httproute_controller.go

@@ -11,13 +11,13 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
+	"github.com/netbirdio/kubernetes-operator/internal/gatewayutil"
 	"github.com/netbirdio/kubernetes-operator/internal/util"
 )
 
@@ -47,36 +47,18 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	for _, parent := range hr.Spec.ParentRefs {
-		// Check if controller is responsible for route.
-		parentNamespace := hr.Namespace
-		if parent.Namespace != nil {
-			parentNamespace = string(*parent.Namespace)
-		}
-		gw := &gatewayv1.Gateway{}
-		err = r.Client.Get(ctx, types.NamespacedName{Namespace: parentNamespace, Name: string(parent.Name)}, gw)
+		gw, err := gatewayutil.GetParentGateway(ctx, r.Client, parent, hr.Namespace, GatewayControllerName)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
-		gwc := &gatewayv1.GatewayClass{}
-		err := r.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwc)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
-		if gwc.Spec.ControllerName != GatewayControllerName {
+		if gw == nil {
 			continue
 		}
-
 		if !meta.IsStatusConditionTrue(gw.Status.Conditions, string(gatewayv1.GatewayConditionProgrammed)) {
 			logger.Info("gateway is not ready", "name", gw.ObjectMeta.Name)
-			return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
+			continue
 		}
-
-		routingPeerName, err := getRoutingPeerName(gw.Spec.Listeners)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
-		nbrp := &netbirdiov1.NBRoutingPeer{}
-		err = r.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: routingPeerName}, nbrp)
+		nbrp, err := gatewayutil.GetGatewayRoutingPeer(ctx, r.Client, *gw)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
@@ -92,7 +74,6 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		svcIdx := map[string]corev1.Service{}
 		for _, rule := range hr.Spec.Rules {
 			for _, ref := range rule.BackendRefs {
-				// TODO (phillebaba): Support reference grants.
 				key := client.ObjectKey{Namespace: hr.Namespace, Name: string(ref.Name)}
 				var svc corev1.Service
 				err := r.Client.Get(ctx, key, &svc)
@@ -214,21 +195,11 @@ func (r *HTTPRouteReconciler) reconcileDelete(ctx context.Context, hr gatewayv1.
 	}
 
 	for _, parent := range hr.Spec.ParentRefs {
-		parentNamespace := hr.Namespace
-		if parent.Namespace != nil {
-			parentNamespace = string(*parent.Namespace)
-		}
-		gw := &gatewayv1.Gateway{}
-		err := r.Client.Get(ctx, types.NamespacedName{Namespace: parentNamespace, Name: string(parent.Name)}, gw)
+		gw, err := gatewayutil.GetParentGateway(ctx, r.Client, parent, hr.Namespace, GatewayControllerName)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
-		gwc := &gatewayv1.GatewayClass{}
-		err = r.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwc)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
-		if gwc.Spec.ControllerName != GatewayControllerName {
+		if gw == nil {
 			continue
 		}
 
@@ -236,7 +207,6 @@ func (r *HTTPRouteReconciler) reconcileDelete(ctx context.Context, hr gatewayv1.
 		svcIdx := map[string]corev1.Service{}
 		for _, rule := range hr.Spec.Rules {
 			for _, ref := range rule.BackendRefs {
-				// TODO (phillebaba): Support reference grants.
 				key := client.ObjectKey{Namespace: hr.Namespace, Name: string(ref.Name)}
 				var svc corev1.Service
 				err := r.Client.Get(ctx, key, &svc)

internal/controller/tcproute_controller.go

@@ -0,0 +1,181 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+
+	netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
+	"github.com/netbirdio/kubernetes-operator/internal/gatewayutil"
+)
+
+const (
+	TCPRouteFinalizer = "gateway.netbird.io/tcproute"
+)
+
+type TCPRouteReconciler struct {
+	client.Client
+
+	ClusterDNS string
+}
+
+// nolint:gocyclo
+func (r *TCPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := ctrl.Log.WithName("TCPRoute").WithValues("namespace", req.Namespace, "name", req.Name)
+
+	tr := gatewayv1alpha2.TCPRoute{}
+	err := r.Get(ctx, req.NamespacedName, &tr)
+	if err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !tr.DeletionTimestamp.IsZero() {
+		return r.reconcileDelete(ctx, tr)
+	}
+
+	for _, parent := range tr.Spec.ParentRefs {
+		gw, err := gatewayutil.GetParentGateway(ctx, r.Client, parent, tr.Namespace, GatewayControllerName)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		if gw == nil {
+			continue
+		}
+		if !meta.IsStatusConditionTrue(gw.Status.Conditions, string(gatewayv1.GatewayConditionProgrammed)) {
+			logger.Info("gateway is not ready", "name", gw.ObjectMeta.Name)
+			continue
+		}
+		nbrp, err := gatewayutil.GetGatewayRoutingPeer(ctx, r.Client, *gw)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+
+		if controllerutil.AddFinalizer(&tr, TCPRouteFinalizer) {
+			err = r.Client.Update(ctx, &tr)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+
+		// Create network resources.
+		svcIdx := map[string]corev1.Service{}
+		for _, rule := range tr.Spec.Rules {
+			for _, ref := range rule.BackendRefs {
+				key := client.ObjectKey{Namespace: tr.Namespace, Name: string(ref.Name)}
+				var svc corev1.Service
+				err := r.Client.Get(ctx, key, &svc)
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+				svcIdx[svc.Name] = svc
+			}
+		}
+
+		for _, svc := range svcIdx {
+			nbResource := netbirdiov1.NBResource{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      svc.Name,
+					Namespace: svc.Namespace,
+				},
+			}
+			_, err := controllerutil.CreateOrUpdate(ctx, r.Client, &nbResource, func() error {
+				err = controllerutil.SetControllerReference(&svc, &nbResource, r.Scheme(), controllerutil.WithBlockOwnerDeletion(false))
+				if err != nil {
+					return err
+				}
+				err = controllerutil.SetOwnerReference(&tr, &nbResource, r.Scheme())
+				if err != nil {
+					return err
+				}
+				nbResource.Spec = netbirdiov1.NBResourceSpec{
+					Name:      svc.Name,
+					NetworkID: *nbrp.Status.NetworkID,
+					Address:   fmt.Sprintf("%s.%s.%s", svc.Name, svc.Namespace, r.ClusterDNS),
+					Groups:    []string{},
+				}
+				return nil
+			})
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+	}
+	return ctrl.Result{}, nil
+}
+
+func (r *TCPRouteReconciler) reconcileDelete(ctx context.Context, tr gatewayv1alpha2.TCPRoute) (ctrl.Result, error) {
+	for _, parent := range tr.Spec.ParentRefs {
+		gw, err := gatewayutil.GetParentGateway(ctx, r.Client, parent, tr.Namespace, GatewayControllerName)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		if gw == nil {
+			continue
+		}
+
+		// Remove the resource from the resource.
+		svcIdx := map[string]corev1.Service{}
+		for _, rule := range tr.Spec.Rules {
+			for _, ref := range rule.BackendRefs {
+				key := client.ObjectKey{Namespace: tr.Namespace, Name: string(ref.Name)}
+				var svc corev1.Service
+				err := r.Client.Get(ctx, key, &svc)
+				if kerrors.IsNotFound(err) {
+					continue
+				}
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+				svcIdx[svc.Name] = svc
+			}
+		}
+		for _, svc := range svcIdx {
+			var nbResource netbirdiov1.NBResource
+			err = r.Client.Get(ctx, client.ObjectKeyFromObject(&svc), &nbResource)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+			err = controllerutil.RemoveOwnerReference(&tr, &nbResource, r.Scheme())
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+
+			if len(nbResource.OwnerReferences) > 1 {
+				err = r.Client.Update(ctx, &nbResource)
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+			} else {
+				// TODO: Precondition that nothing has changed.
+				err := r.Client.Delete(ctx, &nbResource)
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+			}
+		}
+	}
+
+	if controllerutil.RemoveFinalizer(&tr, TCPRouteFinalizer) {
+		err := r.Client.Update(ctx, &tr)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *TCPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&gatewayv1alpha2.TCPRoute{}).
+		Complete(r)
+}

internal/gatewayutil/gatewayutil.go

@@ -0,0 +1,64 @@
+package gatewayutil
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
+
+	netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
+)
+
+func GetParentGateway(ctx context.Context, k8sClient client.Client, parent gwv1.ParentReference, namespace, controllerName string) (*gwv1.Gateway, error) {
+	if parent.Namespace != nil {
+		namespace = string(*parent.Namespace)
+	}
+	gw := &gwv1.Gateway{}
+	err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: string(parent.Name)}, gw)
+	if err != nil {
+		return nil, err
+	}
+	gwc := &gwv1.GatewayClass{}
+	err = k8sClient.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwc)
+	if err != nil {
+		return nil, err
+	}
+	if string(gwc.Spec.ControllerName) != controllerName {
+		return nil, nil
+	}
+
+	// TODO (phillebaba): Enforce allowed routes in gateway.
+
+	return gw, nil
+}
+
+func GetGatewayRoutingPeer(ctx context.Context, k8sClient client.Client, gw gwv1.Gateway) (*netbirdiov1.NBRoutingPeer, error) {
+	routingPeerName, err := GetRoutingPeerName(gw.Spec.Listeners)
+	if err != nil {
+		return nil, err
+	}
+	nbrp := &netbirdiov1.NBRoutingPeer{}
+	err = k8sClient.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: routingPeerName}, nbrp)
+	if err != nil {
+		return nil, err
+	}
+	return nbrp, nil
+}
+
+func GetRoutingPeerName(listeners []gwv1.Listener) (string, error) {
+	if len(listeners) > 1 {
+		return "", errors.New("netbird Gateway only supports a single listener")
+	}
+	group, kind, ok := strings.Cut(string(listeners[0].Protocol), "/")
+	if !ok {
+		return "", fmt.Errorf("invalid protocol %s, expected gateway.netbird.io/NBRoutingPeer", listeners[0].Protocol)
+	}
+	if group != "gateway.netbird.io" || kind != "NBRoutingPeer" {
+		return "", fmt.Errorf("invalid group %s and kind %s, expected gateway.netbird.io/NBRoutingPeer", group, kind)
+	}
+	return string(listeners[0].Name), nil
+}