Refactor uninstallation process (#74)

Docs PR: https://github.com/netbirdio/docs/pull/457

Changes:
* Split kubernetes-operator Chart to operator only chart
(kubernetes-operator) and configuration chart (netbird-operator-config)
* Remove delete validation webhooks for non-needed resources
* Allow abandoning Groups when still linked to a resource for over a
minute after deletion
* Fix duplciate Network Resource retrying heavily
* Fix SetupKey creation duplication
* Skip deleting routing peer since deleting network automatically
deletes it anyway

helm/kubernetes-operator/templates/kubernetes-nbresource.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       initContainers:
       - name: wait-network-ready
-        image: "bitnami/kubectl:latest"
+        image: "netbirdio/kubectl:latest"
         command:
         - bash
         - -c
@@ -35,7 +35,7 @@ spec:
         - kubectl wait --for 'jsonpath={.status.networkID}' -n {{ $routerNS }} nbroutingpeer router; 
       containers:
       - name: apply-nbresource
-        image: "bitnami/kubectl:latest"
+        image: "netbirdio/kubectl:latest"
         env:
         - name: NBRESOURCE_VALUE
           value: |

helm/kubernetes-operator/templates/nbpolicies.yaml

@@ -8,6 +8,8 @@ metadata:
   labels:
     app.kubernetes.io/component: operator
     {{- include "kubernetes-operator.labels" $ | nindent 4 }}
+  annotations:
+    helm.sh/resource-policy: keep
   name: {{ $k }}
 spec:
   name: {{ $v.name }}

helm/kubernetes-operator/templates/nbroutingpeers.yaml

@@ -10,6 +10,8 @@ metadata:
   labels:
     app.kubernetes.io/component: operator
     {{- include "kubernetes-operator.labels" $ | nindent 4 }}
+  annotations:
+    helm.sh/resource-policy: keep
   name: router
   namespace: {{ $k }}
 {{ $spec := merge $defaults $v }}
@@ -51,6 +53,8 @@ metadata:
   labels:
     app.kubernetes.io/component: operator
     {{- include "kubernetes-operator.labels" $ | nindent 4 }}
+  annotations:
+    helm.sh/resource-policy: keep
   name: router
 {{- if or (or (or .replicas .resources) (or .labels .annotations)) (or .nodeSelector .tolerations) }}
 spec:

helm/kubernetes-operator/templates/pre-delete.yaml

@@ -1,69 +1,41 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: {{ include "kubernetes-operator.fullname" . }}-delete-routers
-  labels:
-    app.kubernetes.io/component: operator
-    {{- include "kubernetes-operator.labels" . | nindent 4 }}
-  annotations:
-    helm.sh/hook: pre-delete
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-spec:
-  backoffLimit: 3
-  template:
-    metadata:
-      name: {{ include "kubernetes-operator.fullname" . }}
-      labels:
-        app.kubernetes.io/component: operator
-        {{- include "kubernetes-operator.labels" . | nindent 8 }}
-        {{- with .Values.operator.podLabels }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-    spec:
-      containers:
-      - name: pre-delete
-        image: "bitnami/kubectl:latest"
-        args:
-        - delete
-        - --all
-        - -A
-        - --cascade=foreground
-        - --ignore-not-found
-        - NBRoutingPeer
-      serviceAccountName: {{ include "kubernetes-operator.serviceAccountName" . }}
-      restartPolicy: Never
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: {{ include "kubernetes-operator.fullname" . }}-delete-policies
-  labels:
-    app.kubernetes.io/component: operator
-    {{- include "kubernetes-operator.labels" . | nindent 4 }}
-  annotations:
-    helm.sh/hook: pre-delete
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-spec:
-  backoffLimit: 3
-  template:
-    metadata:
-      name: {{ include "kubernetes-operator.fullname" . }}
-      labels:
-        app.kubernetes.io/component: operator
-        {{- include "kubernetes-operator.labels" . | nindent 8 }}
-        {{- with .Values.operator.podLabels }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-    spec:
-      containers:
-      - name: pre-delete
-        image: "bitnami/kubectl:latest"
-        args:
-        - delete
-        - --all
-        - --cascade=foreground
-        - --ignore-not-found
-        - NBPolicy
-      serviceAccountName: {{ include "kubernetes-operator.serviceAccountName" . }}
-      restartPolicy: Never
----
+{{/*apiVersion: batch/v1*/}}
+{{/*kind: Job*/}}
+{{/*metadata:*/}}
+{{/*  name: {{ include "kubernetes-operator.fullname" . }}-delete-router-deployments*/}}
+{{/*  labels:*/}}
+{{/*    app.kubernetes.io/component: operator*/}}
+{{/*    {{- include "kubernetes-operator.labels" . | nindent 4 }}*/}}
+{{/*  annotations:*/}}
+{{/*    helm.sh/hook: pre-delete*/}}
+{{/*    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded*/}}
+{{/*spec:*/}}
+{{/*  backoffLimit: 3*/}}
+{{/*  template:*/}}
+{{/*    metadata:*/}}
+{{/*      name: {{ include "kubernetes-operator.fullname" . }}*/}}
+{{/*      labels:*/}}
+{{/*        app.kubernetes.io/component: operator*/}}
+{{/*        {{- include "kubernetes-operator.labels" . | nindent 8 }}*/}}
+{{/*        {{- with .Values.operator.podLabels }}*/}}
+{{/*        {{- toYaml . | nindent 8 }}*/}}
+{{/*        {{- end }}*/}}
+{{/*    spec:*/}}
+{{/*      containers:*/}}
+{{/*      - name: pre-delete*/}}
+{{/*        image: "netbirdio/kubectl:latest"*/}}
+{{/*        imagePullPolicy: {{ .Values.operator.image.pullPolicy }}*/}}
+{{/*        command:*/}}
+{{/*          - sh*/}}
+{{/*          - -c*/}}
+{{/*        args:*/}}
+{{/*          - kubectl get NBRoutingPeer -A --no-headers -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | while read "L"; do kubectl patch --type=json -p '[{"op":"replace","path":"/spec/disableDeployment","value":true}]' NBRoutingPeer -n $(echo "$L" | awk '{print $1}') $(echo "$L" | awk '{print $2}'); done*/}}
+{{/*      - name: delete-wait*/}}
+{{/*        image: "netbirdio/kubectl:latest"*/}}
+{{/*        imagePullPolicy: {{ .Values.operator.image.pullPolicy }}*/}}
+{{/*        command:*/}}
+{{/*          - sh*/}}
+{{/*          - -c*/}}
+{{/*        args:*/}}
+{{/*          - kubectl get NBRoutingPeer -A --no-headers -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | while read "L"; do kubectl wait --for=delete deployment -n $(echo "$L" | awk '{print $1}') $(echo "$L" | awk '{print $2}'); done*/}}
+{{/*      serviceAccountName: {{ include "kubernetes-operator.serviceAccountName" . }}*/}}
+{{/*      restartPolicy: Never*/}}

helm/kubernetes-operator/templates/webhook.yaml

@@ -96,84 +96,6 @@ webhooks:
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-{{- if $.Values.webhook.enableCertManager }}
-  annotations:
-    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ template "kubernetes-operator.fullname" . }}-serving-cert
-{{- end }}
-  name: {{ include "kubernetes-operator.fullname" . }}-vnbresource-webhook
-  labels:
-    {{- include "kubernetes-operator.labels" . | nindent 4 }}
-webhooks:
-- clientConfig:
-    {{- if not $.Values.webhook.enableCertManager }}
-    caBundle: {{ $tls.caCert }}
-    {{ end }}
-    service:
-      name: {{ template "kubernetes-operator.webhookService" . }}
-      namespace: {{ $.Release.Namespace }}
-      path: /validate-netbird-io-v1-nbresource
-  failurePolicy: {{ .Values.webhook.failurePolicy }}
-  name: vnbresource-v1.netbird.io
-  admissionReviewVersions:
-  - v1
-  {{- if .Values.webhook.namespaceSelectors }}
-  namespaceSelector:
-    matchExpressions:
-    {{ toYaml .Values.webhook.namespaceSelectors | nindent 4 }}
-  {{ end }}
-  rules:
-  - apiGroups:
-    - netbird.io
-    apiVersions:
-    - v1
-    operations:
-    - DELETE
-    resources:
-    - "nbresources"
-  sideEffects: None
----
-apiVersion: admissionregistration.k8s.io/v1
-kind: ValidatingWebhookConfiguration
-metadata:
-{{- if $.Values.webhook.enableCertManager }}
-  annotations:
-    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ template "kubernetes-operator.fullname" . }}-serving-cert
-{{- end }}
-  name: {{ include "kubernetes-operator.fullname" . }}-vnbroutingpeer-webhook
-  labels:
-    {{- include "kubernetes-operator.labels" . | nindent 4 }}
-webhooks:
-- clientConfig:
-    {{- if not $.Values.webhook.enableCertManager }}
-    caBundle: {{ $tls.caCert }}
-    {{ end }}
-    service:
-      name: {{ template "kubernetes-operator.webhookService" . }}
-      namespace: {{ $.Release.Namespace }}
-      path: /validate-netbird-io-v1-nbroutingpeer
-  failurePolicy: {{ .Values.webhook.failurePolicy }}
-  name: vnbroutingpeer-v1.netbird.io
-  admissionReviewVersions:
-  - v1
-  {{- if .Values.webhook.namespaceSelectors }}
-  namespaceSelector:
-    matchExpressions:
-    {{ toYaml .Values.webhook.namespaceSelectors | nindent 4 }}
-  {{ end }}
-  rules:
-  - apiGroups:
-    - netbird.io
-    apiVersions:
-    - v1
-    operations:
-    - DELETE
-    resources:
-    - "nbroutingpeers"
-  sideEffects: None
----
-apiVersion: admissionregistration.k8s.io/v1
-kind: ValidatingWebhookConfiguration
-metadata:
 {{- if $.Values.webhook.enableCertManager }}
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ template "kubernetes-operator.fullname" . }}-serving-cert