Add ingress feature to controller (#5)

Co-authored-by: Maycon Santos <mlsmaycon@gmail.com>
This commit is contained in:
M. Essam
2025-03-06 09:57:45 +01:00
committed by GitHub
co-authored by Maycon Santos
parent cea60745d2
commit 166091b8e0
54 changed files with 5992 additions and 189 deletions
+197
View File
@@ -0,0 +1,197 @@
package controller
import (
"context"
"fmt"
"strings"
"time"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/go-logr/logr"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/util"
netbird "github.com/netbirdio/netbird/management/client/rest"
"github.com/netbirdio/netbird/management/server/http/api"
)
// NBGroupReconciler reconciles a NBGroup object
type NBGroupReconciler struct {
client.Client
Scheme *runtime.Scheme
APIKey string
ManagementURL string
netbird *netbird.Client
}
const (
// defaultRequeueAfter default requeue duration
// due to controller-runtime limitations, sync periods may reach up to 10 hours if no changes are detected
// in watched resources.
// This may cause issues when NetBird-side resources are out-of-sync and need to be reconciled, this is a temporary
// fix to this issue by syncing with NetBird more frequently.
defaultRequeueAfter = 15 * time.Minute
)
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *NBGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
logger := ctrl.Log.WithName("NBGroup").WithValues("namespace", req.Namespace, "name", req.Name)
logger.Info("Reconciling NBGroup")
nbGroup := netbirdiov1.NBGroup{}
err = r.Client.Get(ctx, req.NamespacedName, &nbGroup)
if err != nil {
if !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBGroup", "err", err)
}
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil
}
originalGroup := nbGroup.DeepCopy()
defer func() {
if !originalGroup.Status.Equal(nbGroup.Status) {
updateErr := r.Client.Status().Update(ctx, &nbGroup)
if updateErr != nil {
err = updateErr
}
}
if !res.Requeue && res.RequeueAfter == 0 {
res.RequeueAfter = defaultRequeueAfter
}
}()
if nbGroup.DeletionTimestamp != nil {
if len(nbGroup.Finalizers) == 0 {
return ctrl.Result{}, nil
}
return ctrl.Result{}, r.handleDelete(ctx, nbGroup, logger)
}
return r.syncNetBirdGroup(ctx, &nbGroup, logger)
}
// syncNetBirdGroup reconciliation logic for non-deleted objects.
func (r *NBGroupReconciler) syncNetBirdGroup(ctx context.Context, nbGroup *netbirdiov1.NBGroup, logger logr.Logger) (ctrl.Result, error) {
// Get all NetBird groups to ensure no group duplication
groups, err := r.netbird.Groups.List(ctx)
if err != nil {
logger.Error(errNetBirdAPI, "error listing groups", "err", err)
return ctrl.Result{}, err
}
var group *api.Group
for _, g := range groups {
if g.Name == nbGroup.Spec.Name {
group = &g
}
}
// Create group if not exists, and update status.groupId
if nbGroup.Status.GroupID == nil && group == nil {
logger.Info("NBGroup: Creating group on NetBird", "name", nbGroup.Spec.Name)
group, err := r.netbird.Groups.Create(ctx, api.GroupRequest{
Name: nbGroup.Spec.Name,
})
if err != nil {
nbGroup.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("NetBird API Error: %v", err))
logger.Error(errNetBirdAPI, "error creating group", "err", err)
return ctrl.Result{}, err
}
logger.Info("NBGroup: Created group on NetBird", "name", nbGroup.Spec.Name, "id", group.Id)
nbGroup.Status.GroupID = &group.Id
nbGroup.Status.Conditions = netbirdiov1.NBConditionTrue()
} else if nbGroup.Status.GroupID == nil && group != nil {
logger.Info("NBGroup: Found group with same name on NetBird", "name", nbGroup.Spec.Name, "id", group.Id)
nbGroup.Status.GroupID = &group.Id
nbGroup.Status.Conditions = netbirdiov1.NBConditionTrue()
} else if group == nil {
logger.Info("NBGroup: Group was deleted", "name", nbGroup.Spec.Name, "id", *nbGroup.Status.GroupID)
nbGroup.Status.GroupID = nil
nbGroup.Status.Conditions = netbirdiov1.NBConditionFalse("GroupGone", "Group was deleted from NetBird API")
return ctrl.Result{Requeue: true}, nil
} else {
nbGroup.Status.Conditions = netbirdiov1.NBConditionTrue()
}
if nbGroup.Status.GroupID != nil && group != nil && *nbGroup.Status.GroupID != group.Id {
// There are two possibilities here, either someone deleted and created the group in NetBird, thus the changed ID
// Or there's a conflict with something else, either way, we just need to take the new ID here
nbGroup.Status.GroupID = &group.Id
nbGroup.Status.Conditions = netbirdiov1.NBConditionTrue()
}
return ctrl.Result{}, nil
}
func (r *NBGroupReconciler) handleDelete(ctx context.Context, nbGroup netbirdiov1.NBGroup, logger logr.Logger) error {
// Group doesn't exist on NetBird, no need for cleanup
if nbGroup.Status.GroupID == nil {
nbGroup.Finalizers = util.Without(nbGroup.Finalizers, "netbird.io/group-cleanup")
err := r.Client.Update(ctx, &nbGroup)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err)
return err
}
return nil
}
err := r.netbird.Groups.Delete(ctx, *nbGroup.Status.GroupID)
if err != nil && !strings.Contains(err.Error(), "not found") && !strings.Contains(err.Error(), "linked") {
logger.Error(errNetBirdAPI, "error deleting group", "err", err)
return err
}
if err != nil && strings.Contains(err.Error(), "linked") {
logger.Info("group still linked to resources on netbird", "err", err)
// Check if group is defined elsewhere in the cluster
var groups netbirdiov1.NBGroupList
listErr := r.Client.List(ctx, &groups)
if listErr != nil {
logger.Error(errKubernetesAPI, "error listing NBGroups", "err", listErr)
return listErr
}
for _, v := range groups.Items {
if v.UID == nbGroup.UID {
continue
}
if v.Status.GroupID != nil && nbGroup.Status.GroupID != nil && *v.Status.GroupID == *nbGroup.Status.GroupID {
// Same group, multiple resources
logger.Info("group exists in another namespace", "namespace", v.Namespace, "name", v.Name)
nbGroup.Finalizers = util.Without(nbGroup.Finalizers, "netbird.io/group-cleanup")
err = r.Client.Update(ctx, &nbGroup)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err)
return err
}
return nil
}
}
// No other NBGroup with same name on the cluster
// This could be a group created by user elsewhere or some resources belonging to the group are still deleting.
return err
}
nbGroup.Finalizers = util.Without(nbGroup.Finalizers, "netbird.io/group-cleanup")
err = r.Client.Update(ctx, &nbGroup)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err)
return err
}
return nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *NBGroupReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.netbird = netbird.New(r.ManagementURL, r.APIKey)
return ctrl.NewControllerManagedBy(mgr).
For(&netbirdiov1.NBGroup{}).
Named("nbgroup").
Complete(r)
}
@@ -0,0 +1,70 @@
package controller
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
)
var _ = Describe("NBGroup Controller", func() {
Context("When reconciling a resource", func() {
const resourceName = "test-resource"
ctx := context.Background()
typeNamespacedName := types.NamespacedName{
Name: resourceName,
Namespace: "default", // TODO(user):Modify as needed
}
nbgroup := &netbirdiov1.NBGroup{}
BeforeEach(func() {
Skip("Not implemented yet")
By("creating the custom resource for the Kind NBGroup")
err := k8sClient.Get(ctx, typeNamespacedName, nbgroup)
if err != nil && errors.IsNotFound(err) {
resource := &netbirdiov1.NBGroup{
ObjectMeta: metav1.ObjectMeta{
Name: resourceName,
Namespace: "default",
},
// TODO(user): Specify other spec details if needed.
}
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
}
})
AfterEach(func() {
// TODO(user): Cleanup logic after each test, like removing the resource instance.
resource := &netbirdiov1.NBGroup{}
err := k8sClient.Get(ctx, typeNamespacedName, resource)
Expect(err).NotTo(HaveOccurred())
By("Cleanup the specific resource instance NBGroup")
Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
})
It("should successfully reconcile the resource", func() {
Skip("Not implemented yet")
By("Reconciling the created resource")
controllerReconciler := &NBGroupReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
}
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
NamespacedName: typeNamespacedName,
})
Expect(err).NotTo(HaveOccurred())
// TODO(user): Add more specific assertions depending on your controller's reconciliation logic.
// Example: If you expect a certain status condition after reconciliation, verify it here.
})
})
})
+378
View File
@@ -0,0 +1,378 @@
package controller
import (
"context"
"fmt"
"strconv"
"strings"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/go-logr/logr"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/util"
netbird "github.com/netbirdio/netbird/management/client/rest"
"github.com/netbirdio/netbird/management/server/http/api"
)
// NBPolicyReconciler reconciles a NBPolicy object
type NBPolicyReconciler struct {
client.Client
Scheme *runtime.Scheme
ClusterName string
APIKey string
ManagementURL string
netbird *netbird.Client
}
var (
errUnknownProtocol = fmt.Errorf("Unknown protocol")
errKubernetesAPI = fmt.Errorf("kubernetes API error")
errNetBirdAPI = fmt.Errorf("netbird API error")
)
// getResources get all NBResource objects in policy.status.managedServiceList
func (r *NBPolicyReconciler) getResources(ctx context.Context, nbPolicy *netbirdiov1.NBPolicy, logger logr.Logger) ([]netbirdiov1.NBResource, error) {
var resourceList []netbirdiov1.NBResource
var updatedManagedServiceList []string
for _, rss := range nbPolicy.Status.ManagedServiceList {
var resource netbirdiov1.NBResource
namespacedName := types.NamespacedName{Namespace: strings.Split(rss, "/")[0], Name: strings.Split(rss, "/")[1]}
err := r.Client.Get(ctx, namespacedName, &resource)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "Error getting NBResource", "namespace", namespacedName.Namespace, "name", namespacedName.Name)
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("Error getting NBResource: %v", err))
return nil, err
}
if err == nil && resource.DeletionTimestamp == nil {
updatedManagedServiceList = append(updatedManagedServiceList, rss)
resourceList = append(resourceList, resource)
}
}
nbPolicy.Status.ManagedServiceList = updatedManagedServiceList
return resourceList, nil
}
// mapResources map each NBResource ports and protocols into one object to generate the policy
// returns map[protocol] => ports, destination group IDs
func (r *NBPolicyReconciler) mapResources(ctx context.Context, nbPolicy *netbirdiov1.NBPolicy, resources []netbirdiov1.NBResource, logger logr.Logger) (map[string][]int32, []string, error) {
portMapping := map[string]map[int32]interface{}{
"tcp": make(map[int32]interface{}),
"udp": make(map[int32]interface{}),
}
groups, err := r.groupNamesToIDs(ctx, nbPolicy.Spec.DestinationGroups, logger)
if err != nil {
return nil, nil, err
}
for _, resource := range resources {
if resource.Status.PolicyName != nil && *resource.Status.PolicyName == nbPolicy.Name {
// Groups
groups = append(groups, resource.Status.Groups...)
for _, p := range resource.Spec.TCPPorts {
portMapping["tcp"][p] = nil
}
for _, p := range resource.Spec.UDPPorts {
portMapping["udp"][p] = nil
}
}
}
ports := make(map[string][]int32)
for k, vs := range portMapping {
for v := range vs {
ports[k] = append(ports[k], v)
}
}
return ports, groups, nil
}
// createPolicy helper for creating policy with settings
func (r *NBPolicyReconciler) createPolicy(ctx context.Context, nbPolicy *netbirdiov1.NBPolicy, protocol string, sourceGroupIDs, destinationGroupIDs, ports []string, logger logr.Logger) (*string, error) {
policyName := fmt.Sprintf("%s %s", nbPolicy.Spec.Name, strings.ToUpper(protocol))
logger.Info("Creating NetBird Policy", "name", policyName, "description", nbPolicy.Spec.Description, "protocol", protocol, "sources", sourceGroupIDs, "destinations", destinationGroupIDs, "ports", ports, "bidirectional", nbPolicy.Spec.Bidirectional)
policy, err := r.netbird.Policies.Create(ctx, api.PostApiPoliciesJSONRequestBody{
Enabled: true,
Name: policyName,
Description: &nbPolicy.Spec.Description,
Rules: []api.PolicyRuleUpdate{
{
Enabled: true,
Name: policyName,
Description: &nbPolicy.Spec.Description,
Action: api.PolicyRuleUpdateActionAccept,
Protocol: api.PolicyRuleUpdateProtocol(protocol),
Bidirectional: nbPolicy.Spec.Bidirectional,
Sources: &sourceGroupIDs,
Destinations: &destinationGroupIDs,
Ports: &ports,
},
},
})
if err != nil {
logger.Error(errNetBirdAPI, "Error creating Policy", "err", err)
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("Error creating policy: %v", err))
return nil, err
}
return policy.Id, nil
}
// updatePolicy helper for updating policy with settings
func (r *NBPolicyReconciler) updatePolicy(ctx context.Context, policyID *string, nbPolicy *netbirdiov1.NBPolicy, protocol string, sourceGroupIDs, destinationGroupIDs, ports []string, logger logr.Logger) (*string, bool, error) {
policyName := fmt.Sprintf("%s %s", nbPolicy.Spec.Name, strings.ToUpper(protocol))
logger.Info("Updating NetBird Policy", "name", policyName, "description", nbPolicy.Spec.Description, "protocol", protocol, "sources", sourceGroupIDs, "destinations", destinationGroupIDs, "ports", ports, "bidirectional", nbPolicy.Spec.Bidirectional)
policy, err := r.netbird.Policies.Update(ctx, *policyID, api.PutApiPoliciesPolicyIdJSONRequestBody{
Enabled: true,
Name: policyName,
Description: &nbPolicy.Spec.Description,
Rules: []api.PolicyRuleUpdate{
{
Enabled: true,
Name: policyName,
Description: &nbPolicy.Spec.Description,
Action: api.PolicyRuleUpdateActionAccept,
Protocol: api.PolicyRuleUpdateProtocol(protocol),
Bidirectional: nbPolicy.Spec.Bidirectional,
Sources: &sourceGroupIDs,
Destinations: &destinationGroupIDs,
Ports: &ports,
},
},
})
if err != nil && !strings.Contains(err.Error(), "not found") {
logger.Error(errNetBirdAPI, "Error updating Policy", "err", err)
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("Error updating policy: %v", err))
return policyID, false, err
}
requeue := false
if err != nil && strings.Contains(err.Error(), "not found") {
logger.Info("Policy deleted from NetBird API, recreating", "protocol", protocol)
policyID = nil
requeue = true
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("Gone", "Policy deleted from NetBird API")
}
if err == nil && (policyID == nil || *policy.Id != *policyID) {
policyID = policy.Id
}
return policyID, requeue, nil
}
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *NBPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
logger := ctrl.Log.WithName("NBPolicy").WithValues("namespace", req.Namespace, "name", req.Name)
logger.Info("Reconciling NBPolicy")
var nbPolicy netbirdiov1.NBPolicy
err = r.Client.Get(ctx, req.NamespacedName, &nbPolicy)
if err != nil {
if errors.IsNotFound(err) {
err = nil
}
if err != nil {
logger.Error(errKubernetesAPI, "error getting NBPolicy", "err", err)
}
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, err
}
originalPolicy := nbPolicy.DeepCopy()
defer func() {
if !originalPolicy.Status.Equal(nbPolicy.Status) {
updateErr := r.Client.Status().Update(ctx, &nbPolicy)
if updateErr != nil {
err = updateErr
}
}
if !res.Requeue && res.RequeueAfter == 0 {
res.RequeueAfter = defaultRequeueAfter
}
}()
if nbPolicy.DeletionTimestamp != nil {
if len(nbPolicy.Finalizers) == 0 {
return ctrl.Result{}, nil
}
return ctrl.Result{}, r.handleDelete(ctx, nbPolicy, logger)
}
resourceList, err := r.getResources(ctx, &nbPolicy, logger)
if err != nil {
return ctrl.Result{}, err
}
portMapping, destGroups, err := r.mapResources(ctx, &nbPolicy, resourceList, logger)
if err != nil {
return ctrl.Result{}, err
}
sourceGroupIDs, err := r.groupNamesToIDs(ctx, nbPolicy.Spec.SourceGroups, logger)
if err != nil {
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("Error getting group IDs: %v", err))
return ctrl.Result{}, err
}
requeue, err := r.syncPolicy(ctx, &nbPolicy, sourceGroupIDs, destGroups, portMapping, logger)
if requeue || err != nil {
return ctrl.Result{Requeue: requeue}, err
}
nbPolicy.Status.Conditions = netbirdiov1.NBConditionTrue()
return ctrl.Result{}, nil
}
// syncPolicy ensure upstream policy is up-to-date
func (r *NBPolicyReconciler) syncPolicy(ctx context.Context, nbPolicy *netbirdiov1.NBPolicy, sourceGroups, destGroups []string, portMapping map[string][]int32, logger logr.Logger) (bool, error) {
requeue := false
for protocol, ports := range portMapping {
var policyID *string
switch protocol {
case "tcp":
policyID = nbPolicy.Status.TCPPolicyID
case "udp":
policyID = nbPolicy.Status.UDPPolicyID
default:
logger.Error(errKubernetesAPI, "Unknown protocol", "protocol", protocol)
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("ConfigError", fmt.Sprintf("Unknown protocol: %s", protocol))
return requeue, errUnknownProtocol
}
if len(nbPolicy.Spec.Protocols) > 0 && !util.Contains(nbPolicy.Spec.Protocols, protocol) {
if policyID != nil {
logger.Info("Deleting protocol policy as NBPolicy has restricted protocols", "protocol", protocol)
err := r.netbird.Policies.Delete(ctx, *policyID)
if err != nil && !strings.Contains(err.Error(), "not found") {
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("Error deleting policy: %v", err))
return requeue, err
}
policyID = nil
} else {
logger.Info("Ignoring protocol as NBPolicy has restricted protocols", "protocol", protocol)
}
} else if len(ports) == 0 && policyID == nil {
logger.Info("0 ports found for protocol in policy", "protocol", protocol)
continue
} else if len(destGroups) == 0 && policyID == nil {
logger.Info("no destinations found for protocol in policy", "protocol", protocol)
continue
} else if len(sourceGroups) == 0 && policyID == nil {
logger.Info("no sources found for protocol in policy", "protocol", protocol)
continue
} else if len(ports) == 0 || len(destGroups) == 0 || len(sourceGroups) == 0 {
// Delete policy
logger.Info("Deleting policy", "protocol", protocol)
err := r.netbird.Policies.Delete(ctx, *policyID)
if err != nil && !strings.Contains(err.Error(), "not found") {
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("Error deleting policy: %v", err))
return requeue, err
}
policyID = nil
} else {
var stringPorts []string
for _, v := range ports {
stringPorts = append(stringPorts, strconv.FormatInt(int64(v), 10))
}
for _, v := range nbPolicy.Spec.Ports {
stringPorts = append(stringPorts, strconv.FormatInt(int64(v), 10))
}
var err error
if policyID == nil {
policyID, err = r.createPolicy(ctx, nbPolicy, protocol, sourceGroups, destGroups, stringPorts, logger)
} else {
policyID, requeue, err = r.updatePolicy(ctx, policyID, nbPolicy, protocol, sourceGroups, destGroups, stringPorts, logger)
}
if err != nil {
return requeue, err
}
}
switch protocol {
case "tcp":
nbPolicy.Status.TCPPolicyID = policyID
case "udp":
nbPolicy.Status.UDPPolicyID = policyID
default:
logger.Error(errKubernetesAPI, "Unknown protocol", "protocol", protocol)
nbPolicy.Status.Conditions = netbirdiov1.NBConditionFalse("ConfigError", fmt.Sprintf("Unknown protocol: %s", protocol))
return requeue, errUnknownProtocol
}
}
return requeue, nil
}
func (r *NBPolicyReconciler) handleDelete(ctx context.Context, nbPolicy netbirdiov1.NBPolicy, logger logr.Logger) error {
if nbPolicy.Status.TCPPolicyID != nil {
err := r.netbird.Policies.Delete(ctx, *nbPolicy.Status.TCPPolicyID)
if err != nil {
return err
}
nbPolicy.Status.TCPPolicyID = nil
}
if nbPolicy.Status.UDPPolicyID != nil {
err := r.netbird.Policies.Delete(ctx, *nbPolicy.Status.UDPPolicyID)
if err != nil && !strings.Contains("not found", err.Error()) {
return err
}
nbPolicy.Status.UDPPolicyID = nil
}
if util.Contains(nbPolicy.Finalizers, "netbird.io/cleanup") {
nbPolicy.Finalizers = util.Without(nbPolicy.Finalizers, "netbird.io/cleanup")
err := r.Client.Update(ctx, &nbPolicy)
if err != nil {
logger.Error(errKubernetesAPI, "Error updating NBPolicy", "err", err)
return err
}
}
return nil
}
// groupNamesToIDs map list of NetBird group names to group IDs
func (r *NBPolicyReconciler) groupNamesToIDs(ctx context.Context, groupNames []string, logger logr.Logger) ([]string, error) {
groups, err := r.netbird.Groups.List(ctx)
if err != nil {
logger.Error(errNetBirdAPI, "Error listing Groups", "err", err)
return nil, err
}
groupNameIDMapping := make(map[string]string)
for _, g := range groups {
groupNameIDMapping[g.Name] = g.Id
}
ret := make([]string, 0, len(groupNames))
for _, g := range groupNames {
ret = append(ret, groupNameIDMapping[g])
}
return ret, nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *NBPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.netbird = netbird.New(r.ManagementURL, r.APIKey)
return ctrl.NewControllerManagedBy(mgr).
For(&netbirdiov1.NBPolicy{}).
Named("nbpolicy").
Complete(r)
}
@@ -0,0 +1,70 @@
package controller
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
)
var _ = Describe("NBPolicy Controller", func() {
Context("When reconciling a resource", func() {
const resourceName = "test-resource"
ctx := context.Background()
typeNamespacedName := types.NamespacedName{
Name: resourceName,
Namespace: "default", // TODO(user):Modify as needed
}
nbpolicy := &netbirdiov1.NBPolicy{}
BeforeEach(func() {
Skip("Not implemented yet")
By("creating the custom resource for the Kind NBPolicy")
err := k8sClient.Get(ctx, typeNamespacedName, nbpolicy)
if err != nil && errors.IsNotFound(err) {
resource := &netbirdiov1.NBPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: resourceName,
Namespace: "default",
},
// TODO(user): Specify other spec details if needed.
}
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
}
})
AfterEach(func() {
// TODO(user): Cleanup logic after each test, like removing the resource instance.
resource := &netbirdiov1.NBPolicy{}
err := k8sClient.Get(ctx, typeNamespacedName, resource)
Expect(err).NotTo(HaveOccurred())
By("Cleanup the specific resource instance NBPolicy")
Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
})
It("should successfully reconcile the resource", func() {
Skip("Not implemented yet")
By("Reconciling the created resource")
controllerReconciler := &NBPolicyReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
}
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
NamespacedName: typeNamespacedName,
})
Expect(err).NotTo(HaveOccurred())
// TODO(user): Add more specific assertions depending on your controller's reconciliation logic.
// Example: If you expect a certain status condition after reconciliation, verify it here.
})
})
})
@@ -0,0 +1,413 @@
package controller
import (
"context"
"fmt"
"strings"
"time"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"github.com/go-logr/logr"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/util"
netbird "github.com/netbirdio/netbird/management/client/rest"
"github.com/netbirdio/netbird/management/server/http/api"
)
// NBResourceReconciler reconciles a NBResource object
type NBResourceReconciler struct {
client.Client
Scheme *runtime.Scheme
APIKey string
ManagementURL string
netbird *netbird.Client
}
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *NBResourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
logger := ctrl.Log.WithName("NBResource").WithValues("namespace", req.Namespace, "name", req.Name)
logger.Info("Reconciling NBResource")
nbResource := &netbirdiov1.NBResource{}
err = r.Client.Get(ctx, req.NamespacedName, nbResource)
if err != nil {
if !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBResource", "err", err)
}
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil
}
originalResource := nbResource.DeepCopy()
defer func() {
if !originalResource.Status.Equal(nbResource.Status) {
updateErr := r.Client.Status().Update(ctx, nbResource)
if updateErr != nil {
err = updateErr
}
}
if !res.Requeue && res.RequeueAfter == 0 {
res.RequeueAfter = defaultRequeueAfter
}
}()
if nbResource.DeletionTimestamp != nil {
if len(nbResource.Finalizers) == 0 {
return ctrl.Result{}, nil
}
return ctrl.Result{}, r.handleDelete(ctx, req, nbResource, logger)
}
groupIDs, result, err := r.handleGroups(ctx, req, nbResource, logger)
if result != nil {
nbResource.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("Error occurred handling groups: %v", err))
return *result, err
}
resource, err := r.handleNetBirdResource(ctx, nbResource, groupIDs, logger)
if err != nil {
nbResource.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("Error occurred handling NetBird Network Resource: %v", err))
return ctrl.Result{}, err
}
// resource is only nil if requeue is expected
if resource == nil {
return ctrl.Result{Requeue: true}, nil
}
err = r.handleGroupUpdate(ctx, nbResource, groupIDs, resource, logger)
if err != nil {
nbResource.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("Error occurred handling groups: %v", err))
return ctrl.Result{}, err
}
err = r.handlePolicy(ctx, req, nbResource, groupIDs, logger)
if err != nil {
nbResource.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("Error occurred handling policy changes: %v", err))
}
nbResource.Status.Conditions = netbirdiov1.NBConditionTrue()
return ctrl.Result{}, nil
}
// handlePolicy update NBPolicy if defined to add self reference to policy status
func (r *NBResourceReconciler) handlePolicy(ctx context.Context, req ctrl.Request, nbResource *netbirdiov1.NBResource, groupIDs []string, logger logr.Logger) error {
if nbResource.Status.PolicyName == nil && nbResource.Spec.PolicyName == "" {
return nil
}
updatePolicyStatus := false
var nbPolicy netbirdiov1.NBPolicy
if nbResource.Spec.PolicyName == "" && nbResource.Status.PolicyName != nil {
// Remove self reference from policy status
nbResource.Status.PolicyName = nil
err := r.Client.Get(ctx, types.NamespacedName{Name: *nbResource.Status.PolicyName}, &nbPolicy)
if err != nil {
logger.Error(errKubernetesAPI, "error getting NBPolicy", "err", err, "policyName", nbResource.Spec.PolicyName)
return err
}
if util.Contains(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) {
nbPolicy.Status.ManagedServiceList = util.Without(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String())
nbPolicy.Status.LastUpdatedAt = &v1.Time{Time: time.Now()}
updatePolicyStatus = true
}
} else {
// Update policy settings if any difference is found
// TODO: Handle updated policy name by removing reference from old policy name in status.policyName
err := r.Client.Get(ctx, types.NamespacedName{Name: nbResource.Spec.PolicyName}, &nbPolicy)
if err != nil {
logger.Error(errKubernetesAPI, "error getting NBPolicy", "err", err, "policyName", nbResource.Spec.PolicyName)
return err
}
if nbResource.Status.PolicyName == nil || *nbResource.Status.PolicyName != nbPolicy.Name {
nbResource.Status.PolicyName = &nbPolicy.Name
}
if !util.Contains(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) {
nbPolicy.Status.ManagedServiceList = append(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String())
nbPolicy.Status.LastUpdatedAt = &v1.Time{Time: time.Now()}
updatePolicyStatus = true
}
if !util.Equivalent(nbResource.Spec.TCPPorts, nbResource.Status.TCPPorts) {
nbResource.Status.TCPPorts = nbResource.Spec.TCPPorts
nbPolicy.Status.LastUpdatedAt = &v1.Time{Time: time.Now()}
updatePolicyStatus = true
}
if !util.Equivalent(nbResource.Spec.UDPPorts, nbResource.Status.UDPPorts) {
nbResource.Status.UDPPorts = nbResource.Spec.UDPPorts
nbPolicy.Status.LastUpdatedAt = &v1.Time{Time: time.Now()}
updatePolicyStatus = true
}
if !util.Equivalent(nbResource.Status.Groups, groupIDs) {
nbResource.Status.Groups = groupIDs
nbPolicy.Status.LastUpdatedAt = &v1.Time{Time: time.Now()}
updatePolicyStatus = true
}
}
if updatePolicyStatus {
err := r.Client.Status().Update(ctx, &nbPolicy)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBPolicy", "err", err, "policyName", nbResource.Spec.PolicyName)
return err
}
}
return nil
}
// handleGroupUpdate update network resource groups
func (r *NBResourceReconciler) handleGroupUpdate(ctx context.Context, nbResource *netbirdiov1.NBResource, groupIDs []string, resource *api.NetworkResource, logger logr.Logger) error {
// Handle possible updated group IDs
groupIDMap := make(map[string]interface{})
for _, g := range groupIDs {
groupIDMap[g] = nil
}
diffFound := len(groupIDs) != len(resource.Groups)
for _, g := range resource.Groups {
if _, ok := groupIDMap[g.Id]; !ok {
diffFound = true
}
}
if diffFound {
_, err := r.netbird.Networks.Resources(nbResource.Spec.NetworkID).Update(ctx, resource.Id, api.NetworkResourceRequest{
Name: nbResource.Spec.Name,
Description: &networkDescription,
Address: nbResource.Spec.Address,
Enabled: true,
Groups: groupIDs,
})
if err != nil {
logger.Error(errNetBirdAPI, "error updating resource", "err", err)
return err
}
}
return nil
}
// handleNetBirdResource sync NetBird Network Resource
func (r *NBResourceReconciler) handleNetBirdResource(ctx context.Context, nbResource *netbirdiov1.NBResource, groupIDs []string, logger logr.Logger) (*api.NetworkResource, error) {
var resource *api.NetworkResource
var err error
if nbResource.Status.NetworkResourceID != nil {
resource, err = r.netbird.Networks.Resources(nbResource.Spec.NetworkID).Get(ctx, *nbResource.Status.NetworkResourceID)
if err != nil && !strings.Contains(err.Error(), "not found") {
logger.Error(errNetBirdAPI, "error getting network resource", "err", err)
return nil, err
}
}
// Create/Update upstream network resource
if nbResource.Status.NetworkResourceID == nil && resource == nil {
resource, err := r.netbird.Networks.Resources(nbResource.Spec.NetworkID).Create(ctx, api.NetworkResourceRequest{
Address: nbResource.Spec.Address,
Enabled: true,
Groups: groupIDs,
Description: &networkDescription,
Name: nbResource.Spec.Name,
})
if err != nil {
logger.Error(errNetBirdAPI, "error creating resource", "err", err)
return nil, err
}
nbResource.Status.NetworkResourceID = &resource.Id
} else if nbResource.Status.NetworkResourceID == nil && resource != nil {
nbResource.Status.NetworkResourceID = &resource.Id
} else if resource == nil {
// Status remembers networkResourceID but resource was deleted elsewhere
// remove networkID from status and re-enqueue
nbResource.Status.NetworkResourceID = nil
} else {
resourceGroups := make([]string, 0, len(resource.Groups))
for _, v := range resource.Groups {
resourceGroups = append(resourceGroups, v.Id)
}
if resource.Address != nbResource.Spec.Address ||
!resource.Enabled ||
!util.Equivalent(resourceGroups, groupIDs) ||
*resource.Description != networkDescription ||
resource.Name != nbResource.Spec.Name {
_, err = r.netbird.Networks.Resources(nbResource.Spec.NetworkID).Update(ctx, *nbResource.Status.NetworkResourceID, api.NetworkResourceRequest{
Address: nbResource.Spec.Address,
Enabled: true,
Groups: groupIDs,
Description: &networkDescription,
Name: nbResource.Spec.Name,
})
if err != nil {
return resource, err
}
}
}
return resource, nil
}
// handleGroups create NBGroup objects for each group specified in NBResource
func (r *NBResourceReconciler) handleGroups(ctx context.Context, req ctrl.Request, nbResource *netbirdiov1.NBResource, logger logr.Logger) ([]string, *ctrl.Result, error) {
var groupIDs []string
for _, groupName := range nbResource.Spec.Groups {
nbGroup := netbirdiov1.NBGroup{}
groupNameRFC := strings.ToLower(groupName)
groupNameRFC = strings.ReplaceAll(groupNameRFC, " ", "-")
err := r.Client.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: groupNameRFC}, &nbGroup)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBGroup", "err", err)
return nil, &ctrl.Result{}, err
} else if errors.IsNotFound(err) {
// Create NBGroup
nbGroup = netbirdiov1.NBGroup{
ObjectMeta: v1.ObjectMeta{
Name: groupNameRFC,
Namespace: nbResource.Namespace,
OwnerReferences: []v1.OwnerReference{
{
APIVersion: nbResource.APIVersion,
Kind: nbResource.Kind,
Name: nbResource.Name,
UID: nbResource.UID,
BlockOwnerDeletion: util.Ptr(true),
},
},
Finalizers: []string{"netbird.io/group-cleanup", "netbird.io/resource-cleanup"},
},
Spec: netbirdiov1.NBGroupSpec{
Name: groupName,
},
}
err = r.Client.Create(ctx, &nbGroup)
if err != nil {
logger.Error(errKubernetesAPI, "error creating NBGroup", "err", err)
return nil, &ctrl.Result{}, err
}
continue
} else {
// Add NBResource as owner to NBGroup if not already done
ownerExists := false
for _, o := range nbGroup.OwnerReferences {
if o.UID == nbResource.UID {
ownerExists = true
}
}
if !ownerExists {
nbGroup.OwnerReferences = append(nbGroup.OwnerReferences, v1.OwnerReference{
APIVersion: nbResource.APIVersion,
Kind: nbResource.Kind,
Name: nbResource.Name,
UID: nbResource.UID,
BlockOwnerDeletion: util.Ptr(true),
})
err = r.Client.Update(ctx, &nbGroup)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err)
return nil, &ctrl.Result{}, err
}
}
}
if nbGroup.Status.GroupID != nil {
groupIDs = append(groupIDs, *nbGroup.Status.GroupID)
}
}
// if not all groups are ready, requeue
if len(groupIDs) != len(nbResource.Spec.Groups) {
return nil, &ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
return groupIDs, nil, nil
}
func (r *NBResourceReconciler) handleDelete(ctx context.Context, req ctrl.Request, nbResource *netbirdiov1.NBResource, logger logr.Logger) error {
if nbResource.Status.PolicyName != nil {
var nbPolicy netbirdiov1.NBPolicy
err := r.Client.Get(ctx, types.NamespacedName{Name: *nbResource.Status.PolicyName}, &nbPolicy)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBPolicy", "err", err, "policyName", nbResource.Spec.PolicyName)
return err
}
if !errors.IsNotFound(err) && util.Contains(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String()) {
nbPolicy.Status.ManagedServiceList = util.Without(nbPolicy.Status.ManagedServiceList, req.NamespacedName.String())
nbPolicy.Status.LastUpdatedAt = &v1.Time{Time: time.Now()}
err = r.Client.Status().Update(ctx, &nbPolicy)
if err != nil {
return err
}
}
}
if nbResource.Status.NetworkResourceID != nil {
err := r.netbird.Networks.Resources(nbResource.Spec.NetworkID).Delete(ctx, *nbResource.Status.NetworkResourceID)
if err != nil && !strings.Contains(err.Error(), "not found") {
logger.Error(errNetBirdAPI, "error deleting resource", "err", err)
return err
}
nbResource.Status.NetworkResourceID = nil
}
nbGroupList := netbirdiov1.NBGroupList{}
err := r.Client.List(ctx, &nbGroupList, &client.ListOptions{Namespace: req.Namespace})
if err != nil {
logger.Error(errKubernetesAPI, "error listing NBGroup", "err", err)
return err
}
for _, g := range nbGroupList.Items {
// TODO: Handle multiple owners
if len(g.OwnerReferences) > 0 && g.OwnerReferences[0].UID == nbResource.UID {
g.Finalizers = util.Without(g.Finalizers, "netbird.io/resource-cleanup")
err = r.Client.Update(ctx, &g)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err)
return err
}
}
}
nbResource.Finalizers = nil
err = r.Client.Update(ctx, nbResource)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBGroup", "err", err)
return err
}
return nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *NBResourceReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.netbird = netbird.New(r.ManagementURL, r.APIKey)
return ctrl.NewControllerManagedBy(mgr).
For(&netbirdiov1.NBResource{}).
Named("nbresource").
Watches(&netbirdiov1.NBGroup{}, handler.EnqueueRequestForOwner(r.Scheme, mgr.GetRESTMapper(), &netbirdiov1.NBResource{})).
Complete(r)
}
@@ -0,0 +1,69 @@
package controller
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
)
var _ = Describe("NBResource Controller", func() {
Context("When reconciling a resource", func() {
const resourceName = "test-resource"
ctx := context.Background()
typeNamespacedName := types.NamespacedName{
Name: resourceName,
Namespace: "default", // TODO(user):Modify as needed
}
nbresource := &netbirdiov1.NBResource{}
BeforeEach(func() {
Skip("Not implemented yet")
By("creating the custom resource for the Kind NBResource")
err := k8sClient.Get(ctx, typeNamespacedName, nbresource)
if err != nil && errors.IsNotFound(err) {
resource := &netbirdiov1.NBResource{
ObjectMeta: metav1.ObjectMeta{
Name: resourceName,
Namespace: "default",
},
// TODO(user): Specify other spec details if needed.
}
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
}
})
AfterEach(func() {
// TODO(user): Cleanup logic after each test, like removing the resource instance.
resource := &netbirdiov1.NBResource{}
err := k8sClient.Get(ctx, typeNamespacedName, resource)
Expect(err).NotTo(HaveOccurred())
By("Cleanup the specific resource instance NBResource")
Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
})
It("should successfully reconcile the resource", func() {
By("Reconciling the created resource")
controllerReconciler := &NBResourceReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
}
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
NamespacedName: typeNamespacedName,
})
Expect(err).NotTo(HaveOccurred())
// TODO(user): Add more specific assertions depending on your controller's reconciliation logic.
// Example: If you expect a certain status condition after reconciliation, verify it here.
})
})
})
@@ -0,0 +1,641 @@
package controller
import (
"context"
"fmt"
"strings"
"time"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"github.com/go-logr/logr"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/util"
netbird "github.com/netbirdio/netbird/management/client/rest"
"github.com/netbirdio/netbird/management/server/http/api"
)
// NBRoutingPeerReconciler reconciles a NBRoutingPeer object
type NBRoutingPeerReconciler struct {
client.Client
Scheme *runtime.Scheme
ClientImage string
ClusterName string
APIKey string
ManagementURL string
NamespacedNetworks bool
netbird *netbird.Client
}
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *NBRoutingPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
logger := ctrl.Log.WithName("NBRoutingPeer").WithValues("namespace", req.Namespace, "name", req.Name)
logger.Info("Reconciling NBRoutingPeer")
nbrp := &netbirdiov1.NBRoutingPeer{}
err = r.Get(ctx, req.NamespacedName, nbrp)
if err != nil {
if !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBRoutingPeer", "err", err)
}
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil
}
originalNBRP := nbrp.DeepCopy()
defer func() {
if originalNBRP.Status.NetworkID != nbrp.Status.NetworkID ||
originalNBRP.Status.RouterID != nbrp.Status.RouterID ||
originalNBRP.Status.SetupKeyID != nbrp.Status.SetupKeyID ||
!util.Equivalent(originalNBRP.Status.Conditions, nbrp.Status.Conditions) {
err = r.Client.Status().Update(ctx, nbrp)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBRoutingPeer Status", "err", err)
}
}
if !res.Requeue && res.RequeueAfter == 0 {
res.RequeueAfter = defaultRequeueAfter
}
}()
if nbrp.DeletionTimestamp != nil {
if len(nbrp.Finalizers) == 0 {
return ctrl.Result{}, nil
}
return r.handleDelete(ctx, req, nbrp, logger)
}
logger.Info("NBRoutingPeer: Checking network")
err = r.handleNetwork(ctx, req, nbrp, logger)
if err != nil {
return ctrl.Result{}, err
}
logger.Info("NBRoutingPeer: Checking groups")
nbGroup, result, err := r.handleGroup(ctx, req, nbrp, logger)
if nbGroup == nil {
return *result, err
}
logger.Info("NBRoutingPeer: Checking setup keys")
result, err = r.handleSetupKey(ctx, req, nbrp, *nbGroup, logger)
if result != nil {
return *result, err
}
logger.Info("NBRoutingPeer: Checking network router")
err = r.handleRouter(ctx, nbrp, *nbGroup, logger)
if err != nil {
return ctrl.Result{}, err
}
logger.Info("NBRoutingPeer: Checking deployment")
err = r.handleDeployment(ctx, req, nbrp, logger)
if err != nil {
return ctrl.Result{}, err
}
nbrp.Status.Conditions = netbirdiov1.NBConditionTrue()
return ctrl.Result{}, nil
}
// handleDeployment reconcile routing peer Deployment
func (r *NBRoutingPeerReconciler) handleDeployment(ctx context.Context, req ctrl.Request, nbrp *netbirdiov1.NBRoutingPeer, logger logr.Logger) error {
routingPeerDeployment := appsv1.Deployment{}
err := r.Client.Get(ctx, req.NamespacedName, &routingPeerDeployment)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting Deployment", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("error getting Deployment: %v", err))
return err
}
// Create deployment
if errors.IsNotFound(err) {
var replicas int32 = 3
if nbrp.Spec.Replicas != nil {
replicas = *nbrp.Spec.Replicas
}
routingPeerDeployment = appsv1.Deployment{
ObjectMeta: v1.ObjectMeta{
Name: nbrp.Name,
Namespace: nbrp.Namespace,
OwnerReferences: []v1.OwnerReference{
{
APIVersion: nbrp.APIVersion,
Kind: nbrp.Kind,
Name: nbrp.Name,
UID: nbrp.UID,
BlockOwnerDeletion: util.Ptr(true),
},
},
Labels: nbrp.Spec.Labels,
Annotations: nbrp.Spec.Annotations,
},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Selector: &v1.LabelSelector{
MatchLabels: map[string]string{
"app.kubernetes.io/name": "netbird-router",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
Labels: map[string]string{
"app.kubernetes.io/name": "netbird-router",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "netbird",
Image: r.ClientImage,
Env: []corev1.EnvVar{
{
Name: "NB_SETUP_KEY",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: nbrp.Name,
},
Key: "setupKey",
},
},
},
{
Name: "NB_MANAGEMENT_URL",
Value: r.ManagementURL,
},
},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
},
},
},
},
},
},
}
err = r.Client.Create(ctx, &routingPeerDeployment)
if err != nil {
logger.Error(errKubernetesAPI, "error creating Deployment", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("error creating Deployment: %v", err))
return err
}
} else {
updatedDeployment := routingPeerDeployment.DeepCopy()
updatedDeployment.ObjectMeta.Name = nbrp.Name
updatedDeployment.ObjectMeta.Namespace = nbrp.Namespace
updatedDeployment.ObjectMeta.OwnerReferences = []v1.OwnerReference{
{
APIVersion: nbrp.APIVersion,
Kind: nbrp.Kind,
Name: nbrp.Name,
UID: nbrp.UID,
BlockOwnerDeletion: util.Ptr(true),
},
}
updatedDeployment.ObjectMeta.Labels = nbrp.Spec.Labels
for k, v := range nbrp.Spec.Annotations {
updatedDeployment.ObjectMeta.Annotations[k] = nbrp.Spec.Annotations[v]
}
var replicas int32 = 3
if nbrp.Spec.Replicas != nil {
replicas = *nbrp.Spec.Replicas
}
updatedDeployment.Spec.Replicas = &replicas
updatedDeployment.Spec.Selector = &v1.LabelSelector{
MatchLabels: map[string]string{
"app.kubernetes.io/name": "netbird-router",
},
}
updatedDeployment.Spec.Template.ObjectMeta.Labels = map[string]string{
"app.kubernetes.io/name": "netbird-router",
}
if len(updatedDeployment.Spec.Template.Spec.Containers) != 1 {
updatedDeployment.Spec.Template.Spec.Containers = []corev1.Container{}
}
updatedDeployment.Spec.Template.Spec.Containers[0].Name = "netbird"
updatedDeployment.Spec.Template.Spec.Containers[0].Image = r.ClientImage
updatedDeployment.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{
{
Name: "NB_SETUP_KEY",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: nbrp.Name,
},
Key: "setupKey",
},
},
},
{
Name: "NB_MANAGEMENT_URL",
Value: r.ManagementURL,
},
}
updatedDeployment.Spec.Template.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
}
patch := client.StrategicMergeFrom(&routingPeerDeployment)
bs, _ := patch.Data(updatedDeployment)
// To ensure no useless patching is done to the deployment being watched
// Minimum patch size is 2 for "{}"
if len(bs) <= 2 {
return nil
}
err = r.Client.Patch(ctx, updatedDeployment, patch)
if err != nil {
logger.Error(errKubernetesAPI, "error updating Deployment", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("error updating Deployment: %v", err))
return err
}
}
return nil
}
// handleRouter reconcile network routing peer in NetBird management API
func (r *NBRoutingPeerReconciler) handleRouter(ctx context.Context, nbrp *netbirdiov1.NBRoutingPeer, nbGroup netbirdiov1.NBGroup, logger logr.Logger) error {
// Check NetworkRouter exists
routers, err := r.netbird.Networks.Routers(*nbrp.Status.NetworkID).List(ctx)
if err != nil {
logger.Error(errNetBirdAPI, "error listing network routers", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("error listing network routers: %v", err))
return err
}
if nbrp.Status.RouterID == nil || len(routers) == 0 {
if len(routers) > 0 {
// Router exists but isn't saved to status
nbrp.Status.RouterID = &routers[0].Id
} else {
// Create network router
router, err := r.netbird.Networks.Routers(*nbrp.Status.NetworkID).Create(ctx, api.NetworkRouterRequest{
Enabled: true,
Masquerade: true,
Metric: 9999,
PeerGroups: &[]string{*nbGroup.Status.GroupID},
})
if err != nil {
logger.Error(errNetBirdAPI, "error creating network router", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("error creating network router: %v", err))
return err
}
nbrp.Status.RouterID = &router.Id
}
} else {
// Ensure network router settings are correct
if !routers[0].Enabled || !routers[0].Masquerade || routers[0].Metric != 9999 || len(*routers[0].PeerGroups) != 1 || (*routers[0].PeerGroups)[0] != *nbGroup.Status.GroupID {
_, err = r.netbird.Networks.Routers(*nbrp.Status.NetworkID).Update(ctx, routers[0].Id, api.NetworkRouterRequest{
Enabled: true,
Masquerade: true,
Metric: 9999,
PeerGroups: &[]string{*nbGroup.Status.GroupID},
})
if err != nil {
logger.Error(errNetBirdAPI, "error updating network router", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("error updating network router: %v", err))
return err
}
}
}
return nil
}
// handleSetupKey reconcile setup key and regenerate if invalid
func (r *NBRoutingPeerReconciler) handleSetupKey(ctx context.Context, req ctrl.Request, nbrp *netbirdiov1.NBRoutingPeer, nbGroup netbirdiov1.NBGroup, logger logr.Logger) (*ctrl.Result, error) {
networkName := r.ClusterName
if r.NamespacedNetworks {
networkName += "-" + req.Namespace
}
// Check if setup key exists
if nbrp.Status.SetupKeyID == nil {
// Create new setup key with group Status.GroupID
setupKey, err := r.netbird.SetupKeys.Create(ctx, api.CreateSetupKeyRequest{
AutoGroups: []string{*nbGroup.Status.GroupID},
Ephemeral: util.Ptr(true),
Name: networkName,
Type: "reusable",
})
if err != nil {
logger.Error(errNetBirdAPI, "error creating setup key", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("error creating setup key: %v", err))
return &ctrl.Result{}, err
}
nbrp.Status.SetupKeyID = &setupKey.Id
skSecret := corev1.Secret{
ObjectMeta: v1.ObjectMeta{
Name: nbrp.Name,
Namespace: nbrp.Namespace,
OwnerReferences: []v1.OwnerReference{
{
APIVersion: nbrp.APIVersion,
Kind: nbrp.Kind,
Name: nbrp.Name,
UID: nbrp.UID,
BlockOwnerDeletion: util.Ptr(true),
},
},
},
StringData: map[string]string{
"setupKey": setupKey.Key,
},
}
err = r.Client.Create(ctx, &skSecret)
if errors.IsAlreadyExists(err) {
err = r.Client.Update(ctx, &skSecret)
}
if err != nil {
logger.Error(errKubernetesAPI, "error creating Secret", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("error creating secret: %v", err))
return &ctrl.Result{}, err
}
} else {
// Check SetupKey is not revoked
setupKey, err := r.netbird.SetupKeys.Get(ctx, *nbrp.Status.SetupKeyID)
if err != nil && !strings.Contains(err.Error(), "not found") {
logger.Error(errNetBirdAPI, "error getting setup key", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("error getting setup key: %v", err))
return &ctrl.Result{}, err
}
if (err != nil && strings.Contains(err.Error(), "not found")) || setupKey.Revoked {
nbrp.Status.SetupKeyID = nil
// Requeue to avoid repeating code
return &ctrl.Result{Requeue: true}, nil
}
// Check if secret is valid
skSecret := corev1.Secret{}
err = r.Client.Get(ctx, req.NamespacedName, &skSecret)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting Secret", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("error getting secret: %v", err))
return &ctrl.Result{}, err
}
if _, ok := skSecret.Data["setupKey"]; errors.IsNotFound(err) || !ok {
// Someone deleted setup key secret
// Revoke SK from NetBird and re-generate
err = r.netbird.SetupKeys.Delete(ctx, *nbrp.Status.SetupKeyID)
if err != nil {
logger.Error(errNetBirdAPI, "error deleting setup key", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("error deleting setup key: %v", err))
return &ctrl.Result{}, err
}
nbrp.Status.SetupKeyID = nil
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("Gone", "generated secret was deleted")
// Requeue to avoid repeating code
return &ctrl.Result{Requeue: true}, nil
}
}
return nil, nil
}
// handleGroup creates/updates NBGroup for routing peer
func (r *NBRoutingPeerReconciler) handleGroup(ctx context.Context, req ctrl.Request, nbrp *netbirdiov1.NBRoutingPeer, logger logr.Logger) (*netbirdiov1.NBGroup, *ctrl.Result, error) {
networkName := r.ClusterName
if r.NamespacedNetworks {
networkName += "-" + req.Namespace
}
// Check if NetBird Group exists
nbGroup := netbirdiov1.NBGroup{}
err := r.Client.Get(ctx, req.NamespacedName, &nbGroup)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBGroup", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("error getting NBGroup: %v", err))
return nil, &ctrl.Result{}, err
}
if errors.IsNotFound(err) {
nbGroup = netbirdiov1.NBGroup{
ObjectMeta: v1.ObjectMeta{
Name: nbrp.Name,
Namespace: nbrp.Namespace,
OwnerReferences: []v1.OwnerReference{
{
APIVersion: nbrp.APIVersion,
Kind: nbrp.Kind,
Name: nbrp.Name,
UID: nbrp.UID,
BlockOwnerDeletion: util.Ptr(true),
},
},
Finalizers: []string{"netbird.io/group-cleanup", "netbird.io/routing-peer-cleanup"},
},
Spec: netbirdiov1.NBGroupSpec{
Name: networkName,
},
}
err = r.Client.Create(ctx, &nbGroup)
if err != nil {
logger.Error(errKubernetesAPI, "error creating NBGroup", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("internalError", fmt.Sprintf("error creating NBGroup: %v", err))
return nil, &ctrl.Result{}, err
}
// Requeue after 5 seconds to ensure group creation is successful by NBGroup controller.
return nil, &ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
if nbGroup.Status.GroupID == nil {
// Group is not yet created successfully, requeue
return nil, &ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
return &nbGroup, nil, nil
}
// handleNetwork Create/Update NetBird Network
func (r *NBRoutingPeerReconciler) handleNetwork(ctx context.Context, req ctrl.Request, nbrp *netbirdiov1.NBRoutingPeer, logger logr.Logger) error {
networkName := r.ClusterName
if r.NamespacedNetworks {
networkName += "-" + req.Namespace
}
if nbrp.Status.NetworkID == nil {
// Check if network exists
networks, err := r.netbird.Networks.List(ctx)
if err != nil {
logger.Error(errNetBirdAPI, "error listing networks", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("error listing networks: %v", err))
return err
}
var network *api.Network
for _, n := range networks {
if n.Name == networkName {
logger.Info("network already exists", "network-id", n.Id)
network = &n
}
}
if network != nil {
nbrp.Status.NetworkID = &network.Id
} else {
logger.Info("creating network", "name", networkName)
network, err := r.netbird.Networks.Create(ctx, api.NetworkRequest{
Name: networkName,
Description: &networkDescription,
})
if err != nil {
logger.Error(errNetBirdAPI, "error creating network", "err", err)
nbrp.Status.Conditions = netbirdiov1.NBConditionFalse("APIError", fmt.Sprintf("error creating network: %v", err))
return err
}
nbrp.Status.NetworkID = &network.Id
}
}
return nil
}
func (r *NBRoutingPeerReconciler) handleDelete(ctx context.Context, req ctrl.Request, nbrp *netbirdiov1.NBRoutingPeer, logger logr.Logger) (ctrl.Result, error) {
nbDeployment := appsv1.Deployment{}
err := r.Client.Get(ctx, req.NamespacedName, &nbDeployment)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting Deployment", "err", err)
return ctrl.Result{}, err
}
if err == nil {
err = r.Client.Delete(ctx, &nbDeployment)
if err != nil {
logger.Error(errKubernetesAPI, "error deleting Deployment", "err", err)
return ctrl.Result{}, err
}
}
if nbrp.Status.SetupKeyID != nil {
logger.Info("Deleting setup key", "id", *nbrp.Status.SetupKeyID)
err = r.netbird.SetupKeys.Delete(ctx, *nbrp.Status.SetupKeyID)
if err != nil && !strings.Contains(err.Error(), "not found") {
logger.Error(errNetBirdAPI, "error deleting setupKey", "err", err)
return ctrl.Result{}, err
}
setupKeyID := *nbrp.Status.SetupKeyID
nbrp.Status.SetupKeyID = nil
logger.Info("Setup key deleted", "id", setupKeyID)
}
if nbrp.Status.RouterID != nil {
err = r.netbird.Networks.Routers(*nbrp.Status.NetworkID).Delete(ctx, *nbrp.Status.RouterID)
if err != nil && !strings.Contains(err.Error(), "not found") {
logger.Error(errNetBirdAPI, "error deleting Network Router", "err", err)
return ctrl.Result{}, err
}
nbrp.Status.RouterID = nil
}
nbGroup := netbirdiov1.NBGroup{}
err = r.Client.Get(ctx, req.NamespacedName, &nbGroup)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBGroup", "err", err)
return ctrl.Result{}, err
}
if nbrp.Status.NetworkID != nil {
nbResourceList := netbirdiov1.NBResourceList{}
err = r.Client.List(ctx, &nbResourceList)
if err != nil {
logger.Error(errKubernetesAPI, "error listing NBResource", "err", err)
return ctrl.Result{}, err
}
for _, nbrs := range nbResourceList.Items {
if nbrs.Spec.NetworkID == *nbrp.Status.NetworkID {
logger.Info("Deleting NBResource", "namespace", nbrs.Namespace, "name", nbrs.Name)
err = r.Client.Delete(ctx, &nbrs)
if err != nil {
logger.Error(errKubernetesAPI, "error deleting NBResource", "err", err)
return ctrl.Result{}, err
}
}
}
if len(nbResourceList.Items) == 0 {
logger.Info("Deleting NetBird Network", "id", *nbrp.Status.NetworkID)
err = r.netbird.Networks.Delete(ctx, *nbrp.Status.NetworkID)
if err != nil && !strings.Contains(err.Error(), "not found") {
logger.Error(errNetBirdAPI, "error deleting Network", "err", err)
return ctrl.Result{}, err
}
nbrp.Status.NetworkID = nil
}
}
if nbGroup.Spec.Name != "" && util.Contains(nbGroup.Finalizers, "netbird.io/routing-peer-cleanup") {
nbGroup.Finalizers = util.Without(nbGroup.Finalizers, "netbird.io/routing-peer-cleanup")
logger.Info("Removing netbird.io/routing-peer-cleanup finalizer NBGroup", "namespace", nbGroup.Namespace, "name", nbGroup.Name)
err = r.Client.Update(ctx, &nbGroup)
if err != nil {
logger.Error(errKubernetesAPI, "error deleting NBGroup", "err", err)
return ctrl.Result{}, err
}
}
if nbrp.Status.NetworkID != nil {
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
if len(nbrp.Finalizers) > 0 {
logger.Info("Removing finalizers", "namespace", nbrp.Namespace, "name", nbrp.Name)
nbrp.Finalizers = nil
err = r.Client.Update(ctx, nbrp)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBRoutingPeer finalizers", "err", err)
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *NBRoutingPeerReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.netbird = netbird.New(r.ManagementURL, r.APIKey)
return ctrl.NewControllerManagedBy(mgr).
For(&netbirdiov1.NBRoutingPeer{}).
Named("nbroutingpeer").
Watches(&appsv1.Deployment{}, handler.EnqueueRequestForOwner(r.Scheme, mgr.GetRESTMapper(), &netbirdiov1.NBRoutingPeer{})).
Watches(&corev1.Secret{}, handler.EnqueueRequestForOwner(r.Scheme, mgr.GetRESTMapper(), &netbirdiov1.NBRoutingPeer{})).
Watches(&netbirdiov1.NBGroup{}, handler.EnqueueRequestForOwner(r.Scheme, mgr.GetRESTMapper(), &netbirdiov1.NBRoutingPeer{})).
Complete(r)
}
@@ -0,0 +1,70 @@
package controller
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
)
var _ = Describe("NBRoutingPeer Controller", func() {
Context("When reconciling a resource", func() {
const resourceName = "test-resource"
ctx := context.Background()
typeNamespacedName := types.NamespacedName{
Name: resourceName,
Namespace: "default", // TODO(user):Modify as needed
}
nbroutingpeer := &netbirdiov1.NBRoutingPeer{}
BeforeEach(func() {
Skip("Not implemented yet")
By("creating the custom resource for the Kind NBRoutingPeer")
err := k8sClient.Get(ctx, typeNamespacedName, nbroutingpeer)
if err != nil && errors.IsNotFound(err) {
resource := &netbirdiov1.NBRoutingPeer{
ObjectMeta: metav1.ObjectMeta{
Name: resourceName,
Namespace: "default",
},
// TODO(user): Specify other spec details if needed.
}
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
}
})
AfterEach(func() {
// TODO(user): Cleanup logic after each test, like removing the resource instance.
resource := &netbirdiov1.NBRoutingPeer{}
err := k8sClient.Get(ctx, typeNamespacedName, resource)
Expect(err).NotTo(HaveOccurred())
By("Cleanup the specific resource instance NBRoutingPeer")
Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
})
It("should successfully reconcile the resource", func() {
Skip("Not implemented yet")
By("Reconciling the created resource")
controllerReconciler := &NBRoutingPeerReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
}
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
NamespacedName: typeNamespacedName,
})
Expect(err).NotTo(HaveOccurred())
// TODO(user): Add more specific assertions depending on your controller's reconciliation logic.
// Example: If you expect a certain status condition after reconciliation, verify it here.
})
})
})
+19 -19
View File
@@ -29,7 +29,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
@@ -45,21 +44,22 @@ type NBSetupKeyReconciler struct {
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *NBSetupKeyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = log.FromContext(ctx)
logger := ctrl.Log.WithName("NBSetupKey").WithValues("namespace", req.Namespace, "name", req.Name)
logger.Info("Reconciling NBSetupKey")
nbSetupKey := netbirdiov1.NBSetupKey{}
err := r.Get(ctx, req.NamespacedName, &nbSetupKey)
if err != nil {
ctrl.Log.Error(fmt.Errorf("internalError"), "error getting NBSetupKey", "err", err, "namespace", req.Namespace, "name", req.Name)
logger.Error(fmt.Errorf("internalError"), "error getting NBSetupKey", "err", err)
return ctrl.Result{}, nil
}
if nbSetupKey.Spec.SecretKeyRef.Name == "" || nbSetupKey.Spec.SecretKeyRef.Key == "" {
ctrl.Log.Error(fmt.Errorf("invalid NBSetupKey"), "secretKeyRef must contain both secret name and secret key", "namespace", req.Namespace, "name", req.Name)
logger.Error(fmt.Errorf("invalid NBSetupKey"), "secretKeyRef must contain both secret name and secret key")
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{
Conditions: []netbirdiov1.NBSetupKeyCondition{
Conditions: []netbirdiov1.NBCondition{
{
Type: netbirdiov1.Ready,
Type: netbirdiov1.NBSetupKeyReady,
Status: corev1.ConditionFalse,
LastProbeTime: v1.Now(),
Reason: "InvalidConfig",
@@ -82,12 +82,12 @@ func (r *NBSetupKeyReconciler) Reconcile(ctx context.Context, req ctrl.Request)
err = r.Get(ctx, types.NamespacedName{Namespace: nbSetupKey.Namespace, Name: nbSetupKey.Spec.SecretKeyRef.Name}, &secret)
if err != nil {
if !errors.IsNotFound(err) {
ctrl.Log.Error(fmt.Errorf("internalError"), "error getting secret", "err", err, "namespace", req.Namespace, "name", req.Name)
logger.Error(fmt.Errorf("internalError"), "error getting secret", "err", err)
return ctrl.Result{}, err
}
ctrl.Log.Error(fmt.Errorf("invalid NBSetupKey"), "secret referenced not found", "err", err, "namespace", req.Namespace, "name", req.Name)
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{Conditions: []netbirdiov1.NBSetupKeyCondition{{
Type: netbirdiov1.Ready,
logger.Error(fmt.Errorf("invalid NBSetupKey"), "secret referenced not found", "err", err)
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{Conditions: []netbirdiov1.NBCondition{{
Type: netbirdiov1.NBSetupKeyReady,
Status: corev1.ConditionFalse,
LastProbeTime: v1.Now(),
Reason: "SecretNotExists",
@@ -97,9 +97,9 @@ func (r *NBSetupKeyReconciler) Reconcile(ctx context.Context, req ctrl.Request)
uuidBytes, ok := secret.Data[nbSetupKey.Spec.SecretKeyRef.Key]
if !ok {
ctrl.Log.Error(fmt.Errorf("invalid NBSetupKey"), "secret key referenced not found", "namespace", req.Namespace, "name", req.Name)
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{Conditions: []netbirdiov1.NBSetupKeyCondition{{
Type: netbirdiov1.Ready,
logger.Error(fmt.Errorf("invalid NBSetupKey"), "secret key referenced not found")
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{Conditions: []netbirdiov1.NBCondition{{
Type: netbirdiov1.NBSetupKeyReady,
Status: corev1.ConditionFalse,
LastProbeTime: v1.Now(),
Reason: "SecretKeyNotExists",
@@ -109,17 +109,17 @@ func (r *NBSetupKeyReconciler) Reconcile(ctx context.Context, req ctrl.Request)
_, err = uuid.Parse(string(uuidBytes))
if err != nil {
ctrl.Log.Error(fmt.Errorf("invalid NBSetupKey"), "setupKey is not a valid UUID", "err", err, "namespace", req.Namespace, "name", req.Name)
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{Conditions: []netbirdiov1.NBSetupKeyCondition{{
Type: netbirdiov1.Ready,
logger.Error(fmt.Errorf("invalid NBSetupKey"), "setupKey is not a valid UUID", "err", err)
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{Conditions: []netbirdiov1.NBCondition{{
Type: netbirdiov1.NBSetupKeyReady,
Status: corev1.ConditionFalse,
LastProbeTime: v1.Now(),
Reason: "InvalidSetupKey",
Message: "Referenced secret is not a valid SetupKey",
}}})
}
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{Conditions: []netbirdiov1.NBSetupKeyCondition{{
Type: netbirdiov1.Ready,
return ctrl.Result{}, r.setStatus(ctx, &nbSetupKey, netbirdiov1.NBSetupKeyStatus{Conditions: []netbirdiov1.NBCondition{{
Type: netbirdiov1.NBSetupKeyReady,
Status: corev1.ConditionTrue,
LastProbeTime: v1.Now(),
}}})
@@ -151,6 +151,6 @@ func (r *NBSetupKeyReconciler) SetupWithManager(mgr ctrl.Manager) error {
return nil
}),
). // Trigger reconciliation when the labeled Busybox resource changes
). // Trigger reconciliation when a referenced secret changes
Complete(r)
}
+259
View File
@@ -0,0 +1,259 @@
package controller
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/go-logr/logr"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
"github.com/netbirdio/kubernetes-operator/internal/util"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// ServiceReconciler reconciles a Service object
type ServiceReconciler struct {
client.Client
Scheme *runtime.Scheme
ClusterName string
ClusterDNS string
NamespacedNetworks bool
ControllerNamespace string
}
const (
// ServiceExposeAnnotation Service annotation for exposing
ServiceExposeAnnotation = "netbird.io/expose"
serviceGroupsAnnotation = "netbird.io/groups"
serviceResourceAnnotation = "netbird.io/resource-name"
servicePolicyAnnotation = "netbird.io/policy"
servicePortsAnnotation = "netbird.io/policy-ports"
serviceProtocolAnnotation = "netbird.io/policy-protocol"
)
var (
networkDescription = "Created by kubernetes-operator"
)
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := ctrl.Log.WithName("Service").WithValues("namespace", req.Namespace, "name", req.Name)
logger.Info("Reconciling Service")
svc := corev1.Service{}
err := r.Get(ctx, req.NamespacedName, &svc)
if err != nil {
if !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting Service", "err", err)
}
return ctrl.Result{}, nil
}
_, shouldExpose := svc.Annotations[ServiceExposeAnnotation]
// If Service is being deleted, un-expose
shouldExpose = shouldExpose && svc.DeletionTimestamp == nil
if shouldExpose {
return r.exposeService(ctx, req, svc, logger)
}
return r.hideService(ctx, req, svc, logger)
}
// hideService deletes NBResource for Service
func (r *ServiceReconciler) hideService(ctx context.Context, req ctrl.Request, svc corev1.Service, logger logr.Logger) (ctrl.Result, error) {
var nbResource netbirdiov1.NBResource
err := r.Client.Get(ctx, req.NamespacedName, &nbResource)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBResource", "err", err)
return ctrl.Result{}, err
}
if !errors.IsNotFound(err) {
err = r.Client.Delete(ctx, &nbResource)
if err != nil {
logger.Error(errKubernetesAPI, "error deleting NBResource", "err", err)
return ctrl.Result{}, err
}
}
if util.Contains(svc.Finalizers, "netbird.io/cleanup") {
svc.Finalizers = util.Without(svc.Finalizers, "netbird.io/cleanup")
err := r.Client.Update(ctx, &svc)
if err != nil {
logger.Error(errKubernetesAPI, "error updating Service", "err", err)
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
// exposeService creates/updates NBResource for Service
func (r *ServiceReconciler) exposeService(ctx context.Context, req ctrl.Request, svc corev1.Service, logger logr.Logger) (ctrl.Result, error) {
routerNamespace := r.ControllerNamespace
if r.NamespacedNetworks {
routerNamespace = req.Namespace
}
if !util.Contains(svc.Finalizers, "netbird.io/cleanup") {
svc.Finalizers = append(svc.Finalizers, "netbird.io/cleanup")
err := r.Client.Update(ctx, &svc)
if err != nil {
logger.Error(errKubernetesAPI, "error updating Service", "err", err)
return ctrl.Result{}, err
}
}
var routingPeer netbirdiov1.NBRoutingPeer
// Check if NBRoutingPeer exists
err := r.Client.Get(ctx, types.NamespacedName{Namespace: routerNamespace, Name: "router"}, &routingPeer)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBRoutingPeer", "err", err)
return ctrl.Result{}, err
}
// Create NBRoutingPeer with default values if not exists
if errors.IsNotFound(err) {
routingPeer = netbirdiov1.NBRoutingPeer{
ObjectMeta: v1.ObjectMeta{
Name: "router",
Namespace: routerNamespace,
Finalizers: []string{"netbird.io/cleanup"},
},
Spec: netbirdiov1.NBRoutingPeerSpec{},
}
err = r.Client.Create(ctx, &routingPeer)
if err != nil {
logger.Error(errKubernetesAPI, "error creating NBRoutingPeer", "err", err)
return ctrl.Result{}, err
}
logger.Info("Network not available")
// Requeue to make sure network is created
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
if routingPeer.Status.NetworkID == nil {
logger.Info("Network not available")
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
var nbResource netbirdiov1.NBResource
err = r.Client.Get(ctx, req.NamespacedName, &nbResource)
if err != nil && !errors.IsNotFound(err) {
logger.Error(errKubernetesAPI, "error getting NBResource", "err", err)
return ctrl.Result{}, err
}
nbrsErr := r.reconcileNBResource(&nbResource, req, svc, routingPeer)
if nbrsErr != nil {
return ctrl.Result{}, nbrsErr
}
if errors.IsNotFound(err) {
err = r.Client.Create(ctx, &nbResource)
if err != nil {
logger.Error(errKubernetesAPI, "error creating NBResource", "err", err)
return ctrl.Result{}, err
}
} else {
err = r.Client.Update(ctx, &nbResource)
if err != nil {
logger.Error(errKubernetesAPI, "error updating NBResource", "err", err)
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
// reconcileNBResource ensures NBResource settings are in-line with Service definition and annotations
func (r *ServiceReconciler) reconcileNBResource(nbResource *netbirdiov1.NBResource, req ctrl.Request, svc corev1.Service, routingPeer netbirdiov1.NBRoutingPeer) error {
groups := []string{fmt.Sprintf("%s-%s-%s", r.ClusterName, req.Namespace, req.Name)}
if v, ok := svc.Annotations[serviceGroupsAnnotation]; ok {
groups = nil
for _, g := range strings.Split(v, ",") {
groups = append(groups, strings.TrimSpace(g))
}
}
resourceName := fmt.Sprintf("%s-%s", req.Namespace, req.Name)
if v, ok := svc.Annotations[serviceResourceAnnotation]; ok {
resourceName = v
}
nbResource.ObjectMeta.Name = req.Name
nbResource.ObjectMeta.Namespace = req.Namespace
nbResource.Finalizers = []string{"netbird.io/cleanup"}
nbResource.Spec.Name = resourceName
nbResource.Spec.NetworkID = *routingPeer.Status.NetworkID
nbResource.Spec.Address = fmt.Sprintf("%s.%s.%s", svc.Name, svc.Namespace, r.ClusterDNS)
nbResource.Spec.Groups = groups
if v, ok := svc.Annotations[servicePolicyAnnotation]; ok {
nbResource.Spec.PolicyName = v
var filterProtocols []string
if v, ok := svc.Annotations[serviceProtocolAnnotation]; ok {
filterProtocols = []string{v}
}
var filterPorts []int32
if v, ok := svc.Annotations[servicePortsAnnotation]; ok {
for _, v := range strings.Split(v, ",") {
port, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return err
}
filterPorts = append(filterPorts, int32(port))
}
}
for _, p := range svc.Spec.Ports {
if len(filterProtocols) > 0 && !util.Contains(filterProtocols, string(p.Protocol)) {
continue
}
if len(filterPorts) > 0 && !util.Contains(filterPorts, p.Port) {
continue
}
switch p.Protocol {
case corev1.ProtocolSCTP:
if !util.Contains(nbResource.Spec.TCPPorts, p.Port) {
nbResource.Spec.TCPPorts = append(nbResource.Spec.TCPPorts, p.Port)
}
case corev1.ProtocolTCP:
if !util.Contains(nbResource.Spec.TCPPorts, p.Port) {
nbResource.Spec.TCPPorts = append(nbResource.Spec.TCPPorts, p.Port)
}
case corev1.ProtocolUDP:
if !util.Contains(nbResource.Spec.UDPPorts, p.Port) {
nbResource.Spec.UDPPorts = append(nbResource.Spec.UDPPorts, p.Port)
}
default:
return errUnknownProtocol
}
}
}
// TODO: Handle removed policy name
return nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *ServiceReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&corev1.Service{}).
Named("service").
Complete(r)
}
@@ -0,0 +1,17 @@
package controller
import (
. "github.com/onsi/ginkgo/v2"
)
var _ = Describe("Service Controller", func() {
Context("When reconciling a resource", func() {
It("should successfully reconcile the resource", func() {
Skip("Not implemented yet")
// TODO(user): Add more specific assertions depending on your controller's reconciliation logic.
// Example: If you expect a certain status condition after reconciliation, verify it here.
})
})
})
+5
View File
@@ -32,6 +32,8 @@ import (
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
foobarv1 "k8s.io/api/core/v1"
netbirdiov1 "github.com/netbirdio/kubernetes-operator/api/v1"
// +kubebuilder:scaffold:imports
)
@@ -62,6 +64,9 @@ var _ = BeforeSuite(func() {
err = netbirdiov1.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())
err = foobarv1.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())
// +kubebuilder:scaffold:scheme
By("bootstrapping test environment")