Add support for private gateway (#154)

This change adds support for TCPRoutes when using a private gateway
class. This is similar to annotating services today. It also moves the
gateway classes to the Helm chart as it makes things a lot simpler for
the end user as they no longer have to define them.

Signed-off-by: Philip Laine <philip.laine@gmail.com>
This commit is contained in:
Philip Laine
2026-03-23 09:59:48 +01:00
committed by GitHub
parent 2aa814f031
commit 8adc8855e5
13 changed files with 340 additions and 76 deletions
+9
View File
@@ -41,6 +41,7 @@ import (
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/controller"
@@ -59,6 +60,7 @@ func init() {
utilruntime.Must(netbirdiov1.AddToScheme(scheme))
utilruntime.Must(corev1.AddToScheme(scheme))
utilruntime.Must(gatewayv1.Install(scheme))
utilruntime.Must(gatewayv1alpha2.Install(scheme))
// +kubebuilder:scaffold:scheme
}
@@ -296,6 +298,13 @@ func main() {
setupLog.Error(err, "unable to create controller", "controller", "HTTPRoute")
os.Exit(1)
}
if err = (&controller.TCPRouteReconciler{
Client: mgr.GetClient(),
ClusterDNS: clusterDNS,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "TCPRoute")
os.Exit(1)
}
}
} else {
setupLog.Info("netbird API key not provided, ingress capabilities disabled")
@@ -1,4 +1,4 @@
# Gateway API (Public)
# Gateway API
This example walks you through how to setup a Netbird Gateway API and expose Nginx through the Netbird proxy service.
@@ -11,10 +11,11 @@ kind load docker-image docker.io/netbirdio/kubernetes-operator:dev
Install the Gateway API CRDs.
```shell
kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.5.0/standard-install.yaml
kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.5.0/experimental-install.yaml
```
Create Netbird namespace and API key secret.
```shell
kubectl create namespace netbird
kubectl -n netbird create secret generic netbird-mgmt-api-key --from-literal NB_API_KEY=${NETBIRD_API_KEY}
@@ -23,17 +24,23 @@ kubectl -n netbird create secret generic netbird-mgmt-api-key --from-literal NB_
Install the Kubernetes Operator. Make sure to use the customized values to enable Gateway API support. This assumes you have already created a secret containing a Netbird API key.
```shell
helm upgrade --install --create-namespace -f ./examples/gateway-api-public/values.yaml -n netbird netbird-operator ./helm/kubernetes-operator
helm upgrade --install --create-namespace -f ./examples/gateway-api/values.yaml -n netbird netbird-operator ./helm/kubernetes-operator
```
Create the gateway along with the routing peer. This will deploy Netbird clients that route traffic into the cluster.
```shell
kubectl apply -f ./examples/gateway-api-public/gateway.yaml
kubectl apply -f ./examples/gateway-api/gateway.yaml
```
Deploy the test Nginx application along with a HTTPRoute. The HTTPRoute will expose the service through Netbirds public proxy.
```shell
kubectl apply -f ./examples/gateway-api-public/nginx.yaml
kubectl apply -f ./examples/gateway-api/nginx.yaml
```
Expose the Kubernetes API server service as a network resource in Netbird.
```shell
kubectl apply -f ./examples/gateway-api/kubernetes.yaml
```
@@ -1,9 +1,21 @@
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
apiVersion: netbird.io/v1
kind: NBRoutingPeer
metadata:
name: public
name: netbird
namespace: netbird
spec: {}
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: private
namespace: netbird
spec:
controllerName: "gateway.netbird.io/controller"
gatewayClassName: netbird-private
listeners:
- protocol: gateway.netbird.io/NBRoutingPeer
name: netbird
port: 1
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
@@ -11,15 +23,8 @@ metadata:
name: public
namespace: netbird
spec:
gatewayClassName: public
gatewayClassName: netbird-public
listeners:
- protocol: gateway.netbird.io/NBRoutingPeer
name: netbird
port: 80
---
apiVersion: netbird.io/v1
kind: NBRoutingPeer
metadata:
name: netbird
namespace: netbird
spec: {}
port: 1
+13
View File
@@ -0,0 +1,13 @@
apiVersion: gateway.networking.k8s.io/v1alpha2
kind: TCPRoute
metadata:
name: kubernetes
namespace: default
spec:
parentRefs:
- name: private
namespace: netbird
rules:
- backendRefs:
- name: kubernetes
port: 443
@@ -0,0 +1,19 @@
{{- if .Values.gatewayAPI.enabled }}
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: netbird-private
labels:
{{- include "kubernetes-operator.labels" . | nindent 4 }}
spec:
controllerName: "gateway.netbird.io/controller"
---
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: netbird-public
labels:
{{- include "kubernetes-operator.labels" . | nindent 4 }}
spec:
controllerName: "gateway.netbird.io/controller"
{{- end }}
@@ -130,6 +130,7 @@ rules:
- gatewayclasses
- gateways
- httproutes
- tcproutes
verbs:
- get
- list
@@ -141,6 +142,7 @@ rules:
- gatewayclasses/status
- gateways/status
- httproutes/status
- tcproutes/status
verbs:
- update
- patch
+4 -19
View File
@@ -18,10 +18,8 @@ package controller
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
netbird "github.com/netbirdio/netbird/shared/management/client/rest"
@@ -35,6 +33,7 @@ import (
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/gatewayutil"
)
const (
@@ -74,7 +73,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
}
// Verify Gateway configuration.
routingPeerName, err := getRoutingPeerName(gw.Spec.Listeners)
routingPeerName, err := gatewayutil.GetRoutingPeerName(gw.Spec.Listeners)
if err != nil {
cond := metav1.Condition{
Type: string(gatewayv1.GatewayConditionAccepted),
@@ -111,8 +110,8 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
}
// Ensure routing peer is ready.
nbrp := &netbirdiov1.NBRoutingPeer{}
err = r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: routingPeerName}, nbrp)
// TODO (phillebaba): Should watch routing peer instead of retrying when not found.
nbrp, err := gatewayutil.GetGatewayRoutingPeer(ctx, r.Client, gw)
if err != nil {
return ctrl.Result{}, err
}
@@ -195,17 +194,3 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
For(&gatewayv1.Gateway{}).
Complete(r)
}
func getRoutingPeerName(listeners []gatewayv1.Listener) (string, error) {
if len(listeners) > 1 {
return "", errors.New("netbird Gateway only supports a single listener")
}
group, kind, ok := strings.Cut(string(listeners[0].Protocol), "/")
if !ok {
return "", fmt.Errorf("invalid protocol %s, expected gateway.netbird.io/NBRoutingPeer", listeners[0].Protocol)
}
if group != "gateway.netbird.io" || kind != "NBRoutingPeer" {
return "", fmt.Errorf("invalid group %s and kind %s, expected gateway.netbird.io/NBRoutingPeer", group, kind)
}
return string(listeners[0].Name), nil
}
+10 -1
View File
@@ -39,12 +39,21 @@ func (r *GatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request
}
// Validate configuration.
message := func() string {
if gwc.Name != "netbird-public" && gwc.Name != "netbird-private" {
return "GatewayClass name must be netbird-public or netbird-private."
}
if gwc.Spec.ParametersRef != nil {
return "Parameters references is not supported."
}
return ""
}()
if message != "" {
cond := metav1.Condition{
Type: string(gatewayv1.GatewayClassConditionStatusAccepted),
Status: metav1.ConditionFalse,
Reason: string(gatewayv1.GatewayClassReasonInvalidParameters),
Message: "Parameters references is not supported.",
Message: message,
}
if meta.SetStatusCondition(&gwc.Status.Conditions, cond) {
err = r.Client.Status().Update(ctx, &gwc)
+7 -37
View File
@@ -11,13 +11,13 @@ import (
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/gatewayutil"
"github.com/netbirdio/kubernetes-operator/internal/util"
)
@@ -47,36 +47,18 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
for _, parent := range hr.Spec.ParentRefs {
// Check if controller is responsible for route.
parentNamespace := hr.Namespace
if parent.Namespace != nil {
parentNamespace = string(*parent.Namespace)
}
gw := &gatewayv1.Gateway{}
err = r.Client.Get(ctx, types.NamespacedName{Namespace: parentNamespace, Name: string(parent.Name)}, gw)
gw, err := gatewayutil.GetParentGateway(ctx, r.Client, parent, hr.Namespace, GatewayControllerName)
if err != nil {
return ctrl.Result{}, err
}
gwc := &gatewayv1.GatewayClass{}
err := r.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwc)
if err != nil {
return ctrl.Result{}, err
}
if gwc.Spec.ControllerName != GatewayControllerName {
if gw == nil {
continue
}
if !meta.IsStatusConditionTrue(gw.Status.Conditions, string(gatewayv1.GatewayConditionProgrammed)) {
logger.Info("gateway is not ready", "name", gw.ObjectMeta.Name)
return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
continue
}
routingPeerName, err := getRoutingPeerName(gw.Spec.Listeners)
if err != nil {
return ctrl.Result{}, err
}
nbrp := &netbirdiov1.NBRoutingPeer{}
err = r.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: routingPeerName}, nbrp)
nbrp, err := gatewayutil.GetGatewayRoutingPeer(ctx, r.Client, *gw)
if err != nil {
return ctrl.Result{}, err
}
@@ -92,7 +74,6 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
svcIdx := map[string]corev1.Service{}
for _, rule := range hr.Spec.Rules {
for _, ref := range rule.BackendRefs {
// TODO (phillebaba): Support reference grants.
key := client.ObjectKey{Namespace: hr.Namespace, Name: string(ref.Name)}
var svc corev1.Service
err := r.Client.Get(ctx, key, &svc)
@@ -214,21 +195,11 @@ func (r *HTTPRouteReconciler) reconcileDelete(ctx context.Context, hr gatewayv1.
}
for _, parent := range hr.Spec.ParentRefs {
parentNamespace := hr.Namespace
if parent.Namespace != nil {
parentNamespace = string(*parent.Namespace)
}
gw := &gatewayv1.Gateway{}
err := r.Client.Get(ctx, types.NamespacedName{Namespace: parentNamespace, Name: string(parent.Name)}, gw)
gw, err := gatewayutil.GetParentGateway(ctx, r.Client, parent, hr.Namespace, GatewayControllerName)
if err != nil {
return ctrl.Result{}, err
}
gwc := &gatewayv1.GatewayClass{}
err = r.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwc)
if err != nil {
return ctrl.Result{}, err
}
if gwc.Spec.ControllerName != GatewayControllerName {
if gw == nil {
continue
}
@@ -236,7 +207,6 @@ func (r *HTTPRouteReconciler) reconcileDelete(ctx context.Context, hr gatewayv1.
svcIdx := map[string]corev1.Service{}
for _, rule := range hr.Spec.Rules {
for _, ref := range rule.BackendRefs {
// TODO (phillebaba): Support reference grants.
key := client.ObjectKey{Namespace: hr.Namespace, Name: string(ref.Name)}
var svc corev1.Service
err := r.Client.Get(ctx, key, &svc)
+181
View File
@@ -0,0 +1,181 @@
package controller
import (
"context"
"fmt"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/gatewayutil"
)
const (
TCPRouteFinalizer = "gateway.netbird.io/tcproute"
)
type TCPRouteReconciler struct {
client.Client
ClusterDNS string
}
// nolint:gocyclo
func (r *TCPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := ctrl.Log.WithName("TCPRoute").WithValues("namespace", req.Namespace, "name", req.Name)
tr := gatewayv1alpha2.TCPRoute{}
err := r.Get(ctx, req.NamespacedName, &tr)
if err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
if !tr.DeletionTimestamp.IsZero() {
return r.reconcileDelete(ctx, tr)
}
for _, parent := range tr.Spec.ParentRefs {
gw, err := gatewayutil.GetParentGateway(ctx, r.Client, parent, tr.Namespace, GatewayControllerName)
if err != nil {
return ctrl.Result{}, err
}
if gw == nil {
continue
}
if !meta.IsStatusConditionTrue(gw.Status.Conditions, string(gatewayv1.GatewayConditionProgrammed)) {
logger.Info("gateway is not ready", "name", gw.ObjectMeta.Name)
continue
}
nbrp, err := gatewayutil.GetGatewayRoutingPeer(ctx, r.Client, *gw)
if err != nil {
return ctrl.Result{}, err
}
if controllerutil.AddFinalizer(&tr, TCPRouteFinalizer) {
err = r.Client.Update(ctx, &tr)
if err != nil {
return ctrl.Result{}, err
}
}
// Create network resources.
svcIdx := map[string]corev1.Service{}
for _, rule := range tr.Spec.Rules {
for _, ref := range rule.BackendRefs {
key := client.ObjectKey{Namespace: tr.Namespace, Name: string(ref.Name)}
var svc corev1.Service
err := r.Client.Get(ctx, key, &svc)
if err != nil {
return ctrl.Result{}, err
}
svcIdx[svc.Name] = svc
}
}
for _, svc := range svcIdx {
nbResource := netbirdiov1.NBResource{
ObjectMeta: metav1.ObjectMeta{
Name: svc.Name,
Namespace: svc.Namespace,
},
}
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, &nbResource, func() error {
err = controllerutil.SetControllerReference(&svc, &nbResource, r.Scheme(), controllerutil.WithBlockOwnerDeletion(false))
if err != nil {
return err
}
err = controllerutil.SetOwnerReference(&tr, &nbResource, r.Scheme())
if err != nil {
return err
}
nbResource.Spec = netbirdiov1.NBResourceSpec{
Name: svc.Name,
NetworkID: *nbrp.Status.NetworkID,
Address: fmt.Sprintf("%s.%s.%s", svc.Name, svc.Namespace, r.ClusterDNS),
Groups: []string{},
}
return nil
})
if err != nil {
return ctrl.Result{}, err
}
}
}
return ctrl.Result{}, nil
}
func (r *TCPRouteReconciler) reconcileDelete(ctx context.Context, tr gatewayv1alpha2.TCPRoute) (ctrl.Result, error) {
for _, parent := range tr.Spec.ParentRefs {
gw, err := gatewayutil.GetParentGateway(ctx, r.Client, parent, tr.Namespace, GatewayControllerName)
if err != nil {
return ctrl.Result{}, err
}
if gw == nil {
continue
}
// Remove the resource from the resource.
svcIdx := map[string]corev1.Service{}
for _, rule := range tr.Spec.Rules {
for _, ref := range rule.BackendRefs {
key := client.ObjectKey{Namespace: tr.Namespace, Name: string(ref.Name)}
var svc corev1.Service
err := r.Client.Get(ctx, key, &svc)
if kerrors.IsNotFound(err) {
continue
}
if err != nil {
return ctrl.Result{}, err
}
svcIdx[svc.Name] = svc
}
}
for _, svc := range svcIdx {
var nbResource netbirdiov1.NBResource
err = r.Client.Get(ctx, client.ObjectKeyFromObject(&svc), &nbResource)
if err != nil {
return ctrl.Result{}, err
}
err = controllerutil.RemoveOwnerReference(&tr, &nbResource, r.Scheme())
if err != nil {
return ctrl.Result{}, err
}
if len(nbResource.OwnerReferences) > 1 {
err = r.Client.Update(ctx, &nbResource)
if err != nil {
return ctrl.Result{}, err
}
} else {
// TODO: Precondition that nothing has changed.
err := r.Client.Delete(ctx, &nbResource)
if err != nil {
return ctrl.Result{}, err
}
}
}
}
if controllerutil.RemoveFinalizer(&tr, TCPRouteFinalizer) {
err := r.Client.Update(ctx, &tr)
if err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *TCPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&gatewayv1alpha2.TCPRoute{}).
Complete(r)
}
+64
View File
@@ -0,0 +1,64 @@
package gatewayutil
import (
"context"
"errors"
"fmt"
"strings"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
gwv1 "sigs.k8s.io/gateway-api/apis/v1"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
)
func GetParentGateway(ctx context.Context, k8sClient client.Client, parent gwv1.ParentReference, namespace, controllerName string) (*gwv1.Gateway, error) {
if parent.Namespace != nil {
namespace = string(*parent.Namespace)
}
gw := &gwv1.Gateway{}
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: string(parent.Name)}, gw)
if err != nil {
return nil, err
}
gwc := &gwv1.GatewayClass{}
err = k8sClient.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwc)
if err != nil {
return nil, err
}
if string(gwc.Spec.ControllerName) != controllerName {
return nil, nil
}
// TODO (phillebaba): Enforce allowed routes in gateway.
return gw, nil
}
func GetGatewayRoutingPeer(ctx context.Context, k8sClient client.Client, gw gwv1.Gateway) (*netbirdiov1.NBRoutingPeer, error) {
routingPeerName, err := GetRoutingPeerName(gw.Spec.Listeners)
if err != nil {
return nil, err
}
nbrp := &netbirdiov1.NBRoutingPeer{}
err = k8sClient.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: routingPeerName}, nbrp)
if err != nil {
return nil, err
}
return nbrp, nil
}
func GetRoutingPeerName(listeners []gwv1.Listener) (string, error) {
if len(listeners) > 1 {
return "", errors.New("netbird Gateway only supports a single listener")
}
group, kind, ok := strings.Cut(string(listeners[0].Protocol), "/")
if !ok {
return "", fmt.Errorf("invalid protocol %s, expected gateway.netbird.io/NBRoutingPeer", listeners[0].Protocol)
}
if group != "gateway.netbird.io" || kind != "NBRoutingPeer" {
return "", fmt.Errorf("invalid group %s and kind %s, expected gateway.netbird.io/NBRoutingPeer", group, kind)
}
return string(listeners[0].Name), nil
}