From 91c07eb76a71f86084e8afb44ff28371b4e6fa7d Mon Sep 17 00:00:00 2001 From: Yuzu Date: Thu, 13 Aug 2026 02:18:51 +0700 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20migrate=20to?= =?UTF-8?q?=20Go=20Kubernetes=20operator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devcontainer/docker-compose.yml | 1 - .devcontainer/post-create.sh | 4 +- .env.example | 5 - apps/backend/src/application/di-container.ts | 8 +- .../services/reverse-proxy.service.ts | 16 +- .../application/services/server.service.ts | 19 +- apps/backend/src/index.ts | 3 + .../infrastructure/event-handlers/index.ts | 1 + .../reverse-proxy-event.handler.ts | 29 + .../event-handlers/server-event.handler.ts | 5 +- .../src/services/operator-resource-sync.ts | 320 +++++++++ bun.lock | 37 +- k8s/rbac/operator-rbac.yaml | 55 -- operator/Makefile | 12 +- .../api/v1alpha1/minecraftserver_types.go | 2 +- .../api/v1alpha1/reverseproxyserver_types.go | 2 +- ...nikura.kirameki.cafe_minecraftservers.yaml | 2 +- ...ura.kirameki.cafe_reverseproxyservers.yaml | 2 +- operator/config/manager/deployment.yaml | 2 - operator/config/rbac/backend.yaml | 29 + .../reverseproxyserver_controller.go | 28 + operator/internal/resources/common.go | 1 + .../internal/resources/reverseproxyserver.go | 4 +- .../resources/reverseproxyserver_test.go | 43 ++ package.json | 9 +- packages/k8s-operator/Dockerfile | 32 - packages/k8s-operator/package.json | 33 - packages/k8s-operator/src/config/constants.ts | 40 -- .../src/config/resource-defaults.ts | 15 - .../src/controllers/base-controller.ts | 53 -- .../controllers/reverse-proxy-controller.ts | 124 ---- .../src/controllers/server-controller.ts | 107 --- packages/k8s-operator/src/crds/rbac.ts | 154 ----- .../k8s-operator/src/crds/reverseProxy.ts | 156 ----- packages/k8s-operator/src/crds/server.ts | 132 ---- packages/k8s-operator/src/index.ts | 81 --- .../src/resources/reverseProxyServer.ts | 230 ------- packages/k8s-operator/src/resources/server.ts | 426 ------------ .../k8s-operator/src/scripts/apply-crds.ts | 37 - .../src/services/notification.service.ts | 80 --- packages/k8s-operator/src/types/index.ts | 87 --- packages/k8s-operator/src/types/k8s-types.ts | 25 - .../k8s-operator/src/utils/crd-registrar.ts | 633 ------------------ packages/k8s-operator/src/utils/k8s-client.ts | 71 -- packages/k8s-operator/src/utils/logger.ts | 5 - packages/k8s-operator/src/utils/memory.ts | 32 - .../k8s-operator/src/utils/rbac-registrar.ts | 203 ------ .../k8s-operator/src/utils/service-type.ts | 19 - packages/k8s-operator/tsconfig.json | 26 - scripts/install.sh | 22 +- 50 files changed, 532 insertions(+), 2930 deletions(-) create mode 100644 apps/backend/src/infrastructure/event-handlers/reverse-proxy-event.handler.ts create mode 100644 apps/backend/src/services/operator-resource-sync.ts delete mode 100644 k8s/rbac/operator-rbac.yaml create mode 100644 operator/config/rbac/backend.yaml create mode 100644 operator/internal/resources/reverseproxyserver_test.go delete mode 100644 packages/k8s-operator/Dockerfile delete mode 100644 packages/k8s-operator/package.json delete mode 100644 packages/k8s-operator/src/config/constants.ts delete mode 100644 packages/k8s-operator/src/config/resource-defaults.ts delete mode 100644 packages/k8s-operator/src/controllers/base-controller.ts delete mode 100644 packages/k8s-operator/src/controllers/reverse-proxy-controller.ts delete mode 100644 packages/k8s-operator/src/controllers/server-controller.ts delete mode 100644 packages/k8s-operator/src/crds/rbac.ts delete mode 100644 packages/k8s-operator/src/crds/reverseProxy.ts delete mode 100644 packages/k8s-operator/src/crds/server.ts delete mode 100644 packages/k8s-operator/src/index.ts delete mode 100644 packages/k8s-operator/src/resources/reverseProxyServer.ts delete mode 100644 packages/k8s-operator/src/resources/server.ts delete mode 100644 packages/k8s-operator/src/scripts/apply-crds.ts delete mode 100644 packages/k8s-operator/src/services/notification.service.ts delete mode 100644 packages/k8s-operator/src/types/index.ts delete mode 100644 packages/k8s-operator/src/types/k8s-types.ts delete mode 100644 packages/k8s-operator/src/utils/crd-registrar.ts delete mode 100644 packages/k8s-operator/src/utils/k8s-client.ts delete mode 100644 packages/k8s-operator/src/utils/logger.ts delete mode 100644 packages/k8s-operator/src/utils/memory.ts delete mode 100644 packages/k8s-operator/src/utils/rbac-registrar.ts delete mode 100644 packages/k8s-operator/src/utils/service-type.ts delete mode 100644 packages/k8s-operator/tsconfig.json diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 4665644..25c40e5 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -35,7 +35,6 @@ services: - WEB_URL=http://localhost:3001 - API_URL=http://localhost:3000 - KUBERNETES_NAMESPACE=minikura - - ENABLE_CRD_REFLECTION=true db: image: postgres:17 diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index ace2afd..28203cb 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -92,8 +92,8 @@ echo "==> Creating minikura namespace..." kubectl create namespace minikura --dry-run=client -o yaml | kubectl apply -f - 2>/dev/null || true echo "==> Installing CRDs..." -kubectl apply -f /workspace/operator/config/crd 2>/dev/null \ - || echo "[WARN] CRD install failed; run 'make install-crds' in operator/" +make -C /workspace/operator install-crds 2>/dev/null \ + || echo "[WARN] CRD install failed; run 'bun run operator:crds'" # Install dependencies echo "==> Installing dependencies..." diff --git a/.env.example b/.env.example index 698a216..480f03f 100644 --- a/.env.example +++ b/.env.example @@ -11,8 +11,3 @@ API_URL="http://localhost:3000" # Kubernetes Configuration KUBERNETES_NAMESPACE="minikura" - -# Kubernetes Operator Configuration -# Enable CRD reflection to automatically sync database state to Kubernetes Custom Resources -# Set to "false" to disable automatic CRD creation from database entries -ENABLE_CRD_REFLECTION="true" diff --git a/apps/backend/src/application/di-container.ts b/apps/backend/src/application/di-container.ts index afe537d..ca4a696 100644 --- a/apps/backend/src/application/di-container.ts +++ b/apps/backend/src/application/di-container.ts @@ -2,6 +2,7 @@ import { PrismaReverseProxyRepository } from "../infrastructure/repositories/pri import { PrismaServerRepository } from "../infrastructure/repositories/prisma/server.repository.impl"; import { PrismaUserRepository } from "../infrastructure/repositories/prisma/user.repository.impl"; import { K8sService } from "../services/k8s"; +import { OperatorResourceSync } from "../services/operator-resource-sync"; import { WebSocketService } from "../services/websocket"; import { ReverseProxyService } from "./services/reverse-proxy.service"; import { ServerService } from "./services/server.service"; @@ -12,9 +13,10 @@ const serverRepo = new PrismaServerRepository(); const reverseProxyRepo = new PrismaReverseProxyRepository(); const webSocketService = new WebSocketService(); const k8sService = new K8sService(); +const operatorResourceSync = new OperatorResourceSync(); export const userService = new UserService(userRepo); -export const serverService = new ServerService(serverRepo, k8sService); -export const reverseProxyService = new ReverseProxyService(reverseProxyRepo); +export const serverService = new ServerService(serverRepo, k8sService, operatorResourceSync); +export const reverseProxyService = new ReverseProxyService(reverseProxyRepo, operatorResourceSync); export const wsService = webSocketService; -export { k8sService }; +export { k8sService, operatorResourceSync }; diff --git a/apps/backend/src/application/services/reverse-proxy.service.ts b/apps/backend/src/application/services/reverse-proxy.service.ts index ee719e7..8661c81 100644 --- a/apps/backend/src/application/services/reverse-proxy.service.ts +++ b/apps/backend/src/application/services/reverse-proxy.service.ts @@ -9,6 +9,7 @@ import type { ReverseProxyRepository, ReverseProxyUpdateInput, } from "../../domain/repositories/reverse-proxy.repository"; +import type { OperatorResourceSync } from "../../services/operator-resource-sync"; import type { IReverseProxyService } from "../interfaces/reverse-proxy.service.interface"; import { BaseCrudService } from "./base-crud.service"; @@ -26,7 +27,10 @@ export class ReverseProxyService > implements IReverseProxyService { - constructor(reverseProxyRepo: ReverseProxyRepository) { + constructor( + reverseProxyRepo: ReverseProxyRepository, + private operatorResourceSync: OperatorResourceSync + ) { super( reverseProxyRepo, { @@ -65,4 +69,14 @@ export class ReverseProxyService deleteReverseProxy(id: string) { return this.delete(id); } + + override async setEnvVariable(proxyId: string, key: string, value: string): Promise { + await super.setEnvVariable(proxyId, key, value); + await this.operatorResourceSync.syncReverseProxyById(proxyId); + } + + override async deleteEnvVariable(proxyId: string, key: string): Promise { + await super.deleteEnvVariable(proxyId, key); + await this.operatorResourceSync.syncReverseProxyById(proxyId); + } } diff --git a/apps/backend/src/application/services/server.service.ts b/apps/backend/src/application/services/server.service.ts index 26930a9..2e74e5f 100644 --- a/apps/backend/src/application/services/server.service.ts +++ b/apps/backend/src/application/services/server.service.ts @@ -10,6 +10,10 @@ import type { ServerUpdateInput, } from "../../domain/repositories/server.repository"; import type { K8sService } from "../../services/k8s"; +import { + type OperatorResourceSync, + operatorResourceName, +} from "../../services/operator-resource-sync"; import type { IServerService } from "../interfaces/server.service.interface"; import { BaseCrudService } from "./base-crud.service"; @@ -29,7 +33,8 @@ export class ServerService { constructor( serverRepo: ServerRepository, - private k8sService: K8sService + private k8sService: K8sService, + private operatorResourceSync: OperatorResourceSync ) { super( serverRepo, @@ -70,9 +75,19 @@ export class ServerService return this.delete(id); } + override async setEnvVariable(serverId: string, key: string, value: string): Promise { + await super.setEnvVariable(serverId, key, value); + await this.operatorResourceSync.syncServerById(serverId); + } + + override async deleteEnvVariable(serverId: string, key: string): Promise { + await super.deleteEnvVariable(serverId, key); + await this.operatorResourceSync.syncServerById(serverId); + } + async getConnectionInfo(serverId: string) { await this.getServerById(serverId); - const serviceName = `minecraft-${serverId}`; + const serviceName = `minecraft-${operatorResourceName(serverId)}`; return this.k8sService.getServerConnectionInfo(serviceName); } } diff --git a/apps/backend/src/index.ts b/apps/backend/src/index.ts index 2982040..1120804 100644 --- a/apps/backend/src/index.ts +++ b/apps/backend/src/index.ts @@ -16,6 +16,9 @@ import { terminalRoutes } from "./routes/terminal"; import { userRoutes } from "./routes/users"; import "./infrastructure/event-handlers"; +import { operatorResourceSync } from "./application/di-container"; + +operatorResourceSync.start(); const app = new Elysia({ adapter: node() }) .use(errorHandler) diff --git a/apps/backend/src/infrastructure/event-handlers/index.ts b/apps/backend/src/infrastructure/event-handlers/index.ts index 8ac022a..5f80ef7 100644 --- a/apps/backend/src/infrastructure/event-handlers/index.ts +++ b/apps/backend/src/infrastructure/event-handlers/index.ts @@ -1,3 +1,4 @@ +import "./reverse-proxy-event.handler"; import "./server-event.handler"; import "./user-event.handler"; diff --git a/apps/backend/src/infrastructure/event-handlers/reverse-proxy-event.handler.ts b/apps/backend/src/infrastructure/event-handlers/reverse-proxy-event.handler.ts new file mode 100644 index 0000000..b5edfba --- /dev/null +++ b/apps/backend/src/infrastructure/event-handlers/reverse-proxy-event.handler.ts @@ -0,0 +1,29 @@ +import { operatorResourceSync, wsService } from "../../application/di-container"; +import { + ReverseProxyCreatedEvent, + ReverseProxyDeletedEvent, + ReverseProxyUpdatedEvent, +} from "../../domain/events/reverse-proxy-lifecycle.events"; +import { eventBus } from "../event-bus"; +import { logger } from "../logger"; + +eventBus.subscribe(ReverseProxyCreatedEvent, async (event) => { + logger.info( + { proxyId: event.proxyId, proxyType: event.proxyType }, + "Reverse proxy created event" + ); + wsService.broadcast("create", event.proxyType, event.proxyId); + await operatorResourceSync.syncReverseProxyById(event.proxyId); +}); + +eventBus.subscribe(ReverseProxyUpdatedEvent, async (event) => { + logger.info({ proxyId: event.proxyId }, "Reverse proxy updated event"); + wsService.broadcast("update", "reverse-proxy", event.proxyId); + await operatorResourceSync.syncReverseProxyById(event.proxyId); +}); + +eventBus.subscribe(ReverseProxyDeletedEvent, async (event) => { + logger.info({ proxyId: event.proxyId }, "Reverse proxy deleted event"); + wsService.broadcast("delete", "reverse-proxy", event.proxyId); + await operatorResourceSync.deleteReverseProxy(event.proxyId); +}); diff --git a/apps/backend/src/infrastructure/event-handlers/server-event.handler.ts b/apps/backend/src/infrastructure/event-handlers/server-event.handler.ts index c444ed7..ff17f04 100644 --- a/apps/backend/src/infrastructure/event-handlers/server-event.handler.ts +++ b/apps/backend/src/infrastructure/event-handlers/server-event.handler.ts @@ -1,4 +1,4 @@ -import { wsService } from "../../application/di-container"; +import { operatorResourceSync, wsService } from "../../application/di-container"; import { ServerCreatedEvent, ServerDeletedEvent, @@ -10,14 +10,17 @@ import { logger } from "../logger"; eventBus.subscribe(ServerCreatedEvent, async (event) => { logger.info({ serverId: event.serverId, serverType: event.serverType }, "Server created event"); wsService.broadcast("create", event.serverType, event.serverId); + await operatorResourceSync.syncServerById(event.serverId); }); eventBus.subscribe(ServerUpdatedEvent, async (event) => { logger.info({ serverId: event.serverId }, "Server updated event"); wsService.broadcast("update", "server", event.serverId); + await operatorResourceSync.syncServerById(event.serverId); }); eventBus.subscribe(ServerDeletedEvent, async (event) => { logger.info({ serverId: event.serverId }, "Server deleted event"); wsService.broadcast("delete", "server", event.serverId); + await operatorResourceSync.deleteServer(event.serverId); }); diff --git a/apps/backend/src/services/operator-resource-sync.ts b/apps/backend/src/services/operator-resource-sync.ts new file mode 100644 index 0000000..4b1e583 --- /dev/null +++ b/apps/backend/src/services/operator-resource-sync.ts @@ -0,0 +1,320 @@ +import * as k8s from "@kubernetes/client-node"; +import { prisma, type ReverseProxyWithEnvVars, type ServerWithEnvVars } from "@minikura/db"; +import { buildKubeConfig } from "@minikura/shared/kube-auth"; +import { logger } from "../infrastructure/logger"; + +const API_GROUP = "minikura.kirameki.cafe"; +const API_VERSION = "v1alpha1"; +const FIELD_MANAGER = "minikura-backend"; +const SYNC_INTERVAL_MS = 30_000; + +type CustomResource = { + apiVersion: string; + kind: string; + metadata: { + name: string; + namespace: string; + labels: Record; + resourceVersion?: string; + }; + spec: Record; +}; + +export function operatorResourceName(id: string): string { + const normalized = id + .toLowerCase() + .replace(/[^a-z0-9.-]+/g, "-") + .replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, "") + .slice(0, 63); + if (!normalized) throw new Error(`Cannot derive a Kubernetes resource name from ${id}`); + return normalized; +} + +function serviceType(type: string): "ClusterIP" | "NodePort" | "LoadBalancer" { + if (type === "NODE_PORT") return "NodePort"; + if (type === "LOAD_BALANCER") return "LoadBalancer"; + return "ClusterIP"; +} + +function labels(id: string): Record { + return { + "app.kubernetes.io/managed-by": FIELD_MANAGER, + "minikura.kirameki.cafe/database-id": id.slice(0, 63), + }; +} + +export class OperatorResourceSync { + private readonly namespace = process.env.KUBERNETES_NAMESPACE || "minikura"; + private coreApi?: k8s.CoreV1Api; + private customObjectsApi?: k8s.CustomObjectsApi; + private syncing = false; + + constructor() { + try { + const kubeConfig = buildKubeConfig(); + this.coreApi = kubeConfig.makeApiClient(k8s.CoreV1Api); + this.customObjectsApi = kubeConfig.makeApiClient(k8s.CustomObjectsApi); + } catch (error) { + logger.warn({ err: error }, "Operator resource synchronization is unavailable"); + } + } + + start(): void { + void this.syncAll(); + const timer = setInterval(() => void this.syncAll(), SYNC_INTERVAL_MS); + timer.unref(); + } + + async syncAll(): Promise { + if (this.syncing || !this.coreApi || !this.customObjectsApi) return; + this.syncing = true; + try { + const [servers, proxies] = await Promise.all([ + prisma.server.findMany({ include: { env_variables: true } }), + prisma.reverseProxyServer.findMany({ include: { env_variables: true } }), + ]); + await Promise.all([ + ...servers.map((server) => this.syncServer(server)), + ...proxies.map((proxy) => this.syncReverseProxy(proxy)), + ]); + await Promise.all([ + this.deleteStaleResources( + "minecraftservers", + servers.map((server) => operatorResourceName(server.id)) + ), + this.deleteStaleResources( + "reverseproxyservers", + proxies.map((proxy) => operatorResourceName(proxy.id)) + ), + ]); + } catch (error) { + logger.error({ err: error }, "Failed to synchronize operator resources"); + } finally { + this.syncing = false; + } + } + + async syncServerById(id: string): Promise { + if (!this.coreApi || !this.customObjectsApi) return; + const server = await prisma.server.findUnique({ + where: { id }, + include: { env_variables: true }, + }); + if (server) await this.syncServer(server); + } + + async syncReverseProxyById(id: string): Promise { + if (!this.coreApi || !this.customObjectsApi) return; + const proxy = await prisma.reverseProxyServer.findUnique({ + where: { id }, + include: { env_variables: true }, + }); + if (proxy) await this.syncReverseProxy(proxy); + } + + async deleteServer(id: string): Promise { + if (!this.customObjectsApi) return; + await this.deleteResource("minecraftservers", operatorResourceName(id)); + } + + async deleteReverseProxy(id: string): Promise { + if (!this.customObjectsApi) return; + await this.deleteResource("reverseproxyservers", operatorResourceName(id)); + } + + private async syncServer(server: ServerWithEnvVars): Promise { + const name = operatorResourceName(server.id); + const secretName = `${name}-api-key`; + await this.upsertSecret(secretName, server.api_key, labels(server.id)); + await this.upsertResource("minecraftservers", { + apiVersion: `${API_GROUP}/${API_VERSION}`, + kind: "MinecraftServer", + metadata: { name, namespace: this.namespace, labels: labels(server.id) }, + spec: { + type: server.type, + description: server.description ?? undefined, + listenPort: server.listen_port, + serviceType: serviceType(server.service_type), + nodePort: server.node_port ?? undefined, + jarType: server.jar_type, + minecraftVersion: server.minecraft_version, + resources: { + memoryLimitMB: server.memory, + memoryRequestMB: server.memory_request, + cpuRequest: server.cpu_request ?? undefined, + cpuLimit: server.cpu_limit ?? undefined, + }, + jvm: { + opts: server.jvm_opts ?? undefined, + useAikarFlags: server.use_aikar_flags, + useMeowIceFlags: server.use_meowice_flags, + heapPercent: 80, + }, + properties: { + difficulty: server.difficulty, + gameMode: server.game_mode, + maxPlayers: server.max_players, + pvp: server.pvp, + onlineMode: server.online_mode, + motd: server.motd ?? undefined, + levelSeed: server.level_seed ?? undefined, + levelType: server.level_type ?? undefined, + }, + env: server.env_variables.map((entry) => ({ name: entry.key, value: entry.value })), + apiKeySecretRef: secretName, + }, + }); + } + + private async syncReverseProxy(proxy: ReverseProxyWithEnvVars): Promise { + const name = operatorResourceName(proxy.id); + const secretName = `${name}-api-key`; + await this.upsertSecret(secretName, proxy.api_key, labels(proxy.id)); + await this.upsertResource("reverseproxyservers", { + apiVersion: `${API_GROUP}/${API_VERSION}`, + kind: "ReverseProxyServer", + metadata: { name, namespace: this.namespace, labels: labels(proxy.id) }, + spec: { + type: proxy.type, + description: proxy.description ?? undefined, + externalAddress: proxy.external_address, + externalPort: proxy.external_port, + listenPort: proxy.listen_port, + serviceType: serviceType(proxy.service_type), + nodePort: proxy.node_port ?? undefined, + resources: { + memoryLimitMB: proxy.memory, + memoryRequestMB: proxy.memory, + cpuRequest: proxy.cpu_request ?? undefined, + cpuLimit: proxy.cpu_limit ?? undefined, + }, + jvm: { heapPercent: 80 }, + env: proxy.env_variables.map((entry) => ({ name: entry.key, value: entry.value })), + apiKeySecretRef: secretName, + }, + }); + } + + private async upsertResource(plural: string, resource: CustomResource): Promise { + if (!this.customObjectsApi) return; + try { + const existing = (await this.customObjectsApi.getNamespacedCustomObject({ + group: API_GROUP, + version: API_VERSION, + namespace: this.namespace, + plural, + name: resource.metadata.name, + })) as { metadata?: { resourceVersion?: string } }; + resource.metadata.resourceVersion = existing.metadata?.resourceVersion; + await this.customObjectsApi.replaceNamespacedCustomObject({ + group: API_GROUP, + version: API_VERSION, + namespace: this.namespace, + plural, + name: resource.metadata.name, + body: resource, + fieldManager: FIELD_MANAGER, + }); + } catch (error) { + if (!this.isNotFound(error)) throw error; + await this.customObjectsApi.createNamespacedCustomObject({ + group: API_GROUP, + version: API_VERSION, + namespace: this.namespace, + plural, + body: resource, + fieldManager: FIELD_MANAGER, + }); + } + } + + private async upsertSecret( + name: string, + apiKey: string, + resourceLabels: Record + ): Promise { + if (!this.coreApi) return; + const secret: k8s.V1Secret = { + metadata: { name, namespace: this.namespace, labels: resourceLabels }, + stringData: { "api-key": apiKey }, + type: "Opaque", + }; + try { + const existing = await this.coreApi.readNamespacedSecret({ name, namespace: this.namespace }); + secret.metadata = { + ...secret.metadata, + resourceVersion: existing.metadata?.resourceVersion, + }; + await this.coreApi.replaceNamespacedSecret({ + name, + namespace: this.namespace, + body: secret, + fieldManager: FIELD_MANAGER, + }); + } catch (error) { + if (!this.isNotFound(error)) throw error; + await this.coreApi.createNamespacedSecret({ + namespace: this.namespace, + body: secret, + fieldManager: FIELD_MANAGER, + }); + } + } + + private async deleteStaleResources(plural: string, expectedNames: string[]): Promise { + if (!this.customObjectsApi) return; + const response = (await this.customObjectsApi.listNamespacedCustomObject({ + group: API_GROUP, + version: API_VERSION, + namespace: this.namespace, + plural, + labelSelector: `app.kubernetes.io/managed-by=${FIELD_MANAGER}`, + })) as { items?: Array<{ metadata?: { name?: string } }> }; + const expected = new Set(expectedNames); + await Promise.all( + (response.items ?? []) + .map((item) => item.metadata?.name) + .filter((name): name is string => !!name && !expected.has(name)) + .map((name) => this.deleteResource(plural, name)) + ); + } + + private async deleteResource(plural: string, name: string): Promise { + if (!this.customObjectsApi) return; + try { + await this.customObjectsApi.deleteNamespacedCustomObject({ + group: API_GROUP, + version: API_VERSION, + namespace: this.namespace, + plural, + name, + propagationPolicy: "Foreground", + }); + } catch (error) { + if (!this.isNotFound(error)) throw error; + } + await this.deleteSecret(`${name}-api-key`); + } + + private async deleteSecret(name: string): Promise { + if (!this.coreApi) return; + try { + await this.coreApi.deleteNamespacedSecret({ name, namespace: this.namespace }); + } catch (error) { + if (!this.isNotFound(error)) throw error; + } + } + + private isNotFound(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + (("code" in error && error.code === 404) || + ("response" in error && + typeof error.response === "object" && + error.response !== null && + "statusCode" in error.response && + error.response.statusCode === 404)) + ); + } +} diff --git a/bun.lock b/bun.lock index e2b460c..ccd0483 100644 --- a/bun.lock +++ b/bun.lock @@ -126,27 +126,6 @@ "typescript": "^7.0.2", }, }, - "packages/k8s-operator": { - "name": "@minikura/k8s-operator", - "version": "1.0.0", - "dependencies": { - "@kubernetes/client-node": "^1.4.0", - "@minikura/api": "workspace:*", - "@minikura/db": "workspace:*", - "dotenv-mono": "^1.5.1", - "node-fetch": "^3.3.2", - "pg": "^8.23.0", - "pino": "^10.3.1", - "pino-pretty": "^13.1.3", - "undici": "^8.10.0", - "yaml": "^2.9.0", - }, - "devDependencies": { - "@types/node": "^26.2.0", - "tsx": "^4.23.12", - "typescript": "^7.0.2", - }, - }, "packages/shared": { "name": "@minikura/shared", "version": "1.0.0", @@ -354,8 +333,6 @@ "@minikura/db": ["@minikura/db@workspace:packages/db"], - "@minikura/k8s-operator": ["@minikura/k8s-operator@workspace:packages/k8s-operator"], - "@minikura/shared": ["@minikura/shared@workspace:packages/shared"], "@minikura/web": ["@minikura/web@workspace:apps/web"], @@ -880,8 +857,6 @@ "d3-zoom": ["d3-zoom@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-drag": "2 - 3", "d3-interpolate": "1 - 3", "d3-selection": "2 - 3", "d3-transition": "2 - 3" } }, "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw=="], - "data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="], - "dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -972,8 +947,6 @@ "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], - "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="], - "fflate": ["fflate@0.8.2", "", {}, "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A=="], "file-type": ["file-type@21.3.0", "", { "dependencies": { "@tokenizer/inflate": "^0.4.1", "strtok3": "^10.3.4", "token-types": "^6.1.1", "uint8array-extras": "^1.4.0" } }, "sha512-8kPJMIGz1Yt/aPEwOsrR97ZyZaD1Iqm8PClb1nYFclUCkBi0Ma5IsYNQzvSFS9ib51lWyIw5mIT9rWzI/xjpzA=="], @@ -990,8 +963,6 @@ "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], - "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="], - "fraction.js": ["fraction.js@5.3.4", "", {}, "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ=="], "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], @@ -1126,9 +1097,7 @@ "next": ["next@16.3.0", "", { "dependencies": { "@next/env": "16.3.0", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.5.23", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.3.0", "@next/swc-darwin-x64": "16.3.0", "@next/swc-linux-arm64-gnu": "16.3.0", "@next/swc-linux-arm64-musl": "16.3.0", "@next/swc-linux-x64-gnu": "16.3.0", "@next/swc-linux-x64-musl": "16.3.0", "@next/swc-win32-arm64-msvc": "16.3.0", "@next/swc-win32-x64-msvc": "16.3.0", "sharp": "^0.35.3" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-NEdGOzH+08eTXMUp9UYkA99Nhi5N6Thrhc1jgFOQgfgnGK/dA2hRwBpXep+exdFQrnwlRf/3Wixyp8lLBUpE2A=="], - "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], - - "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], "node-releases": ["node-releases@2.0.53", "", {}, "sha512-D9UOmYG3UH1V+ENW56t5QXBwJw1YEY18ruVeus89Rw+SyIgjPkCO84bRzO3uNIYosJbNwiabWVn48o3uJLjxFQ=="], @@ -1378,8 +1347,6 @@ "vitest": ["vitest@4.0.17", "", { "dependencies": { "@vitest/expect": "4.0.17", "@vitest/mocker": "4.0.17", "@vitest/pretty-format": "4.0.17", "@vitest/runner": "4.0.17", "@vitest/snapshot": "4.0.17", "@vitest/spy": "4.0.17", "@vitest/utils": "4.0.17", "es-module-lexer": "^1.7.0", "expect-type": "^1.2.2", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^3.10.0", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.0.3", "vite": "^6.0.0 || ^7.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.0.17", "@vitest/browser-preview": "4.0.17", "@vitest/browser-webdriverio": "4.0.17", "@vitest/ui": "4.0.17", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-FQMeF0DJdWY0iOnbv466n/0BudNdKj1l5jYgl5JVTwjSsZSlqyXFt/9+1sEyhR6CLowbZpV7O1sCHrzBhucKKg=="], - "web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="], - "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], @@ -1420,8 +1387,6 @@ "@kubernetes/client-node/@types/node": ["@types/node@24.10.9", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-ne4A0IpG3+2ETuREInjPNhUGis1SFjv1d5asp8MzEAGtOZeTeHVDOYqOgqfhvseqg/iXty2hjBf1zAOb7RNiNw=="], - "@kubernetes/client-node/node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], - "@kubernetes/client-node/ws": ["ws@8.18.2", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ=="], "@prisma/config/effect": ["effect@3.20.0", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" } }, "sha512-qMLfDJscrNG8p/aw+IkT9W7fgj50Z4wG5bLBy0Txsxz8iUHjDIkOgO3SV0WZfnQbNG2VJYb0b+rDLMrhM4+Krw=="], diff --git a/k8s/rbac/operator-rbac.yaml b/k8s/rbac/operator-rbac.yaml deleted file mode 100644 index 1ba163b..0000000 --- a/k8s/rbac/operator-rbac.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: minikura-operator - namespace: minikura ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: minikura-operator-role -rules: - - apiGroups: [""] - resources: ["services", "pods", "persistentvolumeclaims", "configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: [""] - resources: ["pods/log"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list"] - - apiGroups: ["apps"] - resources: ["deployments", "statefulsets"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: ["networking.k8s.io"] - resources: ["ingresses"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: ["minikura.kirameki.cafe"] - resources: ["minecraftservers", "reverseproxyservers"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: minikura-operator-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: minikura-operator-role -subjects: - - kind: ServiceAccount - name: minikura-operator - namespace: minikura ---- -# Secret to hold the service account token -apiVersion: v1 -kind: Secret -metadata: - name: minikura-operator-token - namespace: minikura - annotations: - kubernetes.io/service-account.name: minikura-operator -type: kubernetes.io/service-account-token diff --git a/operator/Makefile b/operator/Makefile index a105dd0..cbb870f 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -1,5 +1,6 @@ IMG ?= minikura-operator:latest NAMESPACE ?= minikura +CONTROLLER_GEN_VERSION ?= v0.20.0 CONTROLLER_GEN ?= $(shell go env GOPATH)/bin/controller-gen .PHONY: all @@ -7,7 +8,7 @@ all: generate manifests fmt vet test build .PHONY: controller-gen controller-gen: - @test -x $(CONTROLLER_GEN) || go install sigs.k8s.io/controller-tools/cmd/controller-gen@latest + @test -x $(CONTROLLER_GEN) || go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION) .PHONY: generate generate: controller-gen @@ -52,6 +53,11 @@ uninstall-crds: .PHONY: deploy deploy: manifests + kubectl create namespace $(NAMESPACE) --dry-run=client -o yaml | kubectl apply -f - kubectl apply -f config/crd - kubectl apply -n $(NAMESPACE) -f config/rbac - kubectl apply -n $(NAMESPACE) -f config/manager + kubectl apply -f config/rbac/role.yaml + kubectl apply -n $(NAMESPACE) -f config/rbac/service_account.yaml + kubectl apply -n $(NAMESPACE) -f config/rbac/backend.yaml + kubectl patch clusterrolebinding minikura-operator-rolebinding --type=json -p='[{"op":"replace","path":"/subjects/0/namespace","value":"$(NAMESPACE)"}]' + kubectl patch clusterrolebinding minikura-backend-operator-resources --type=json -p='[{"op":"replace","path":"/subjects/0/namespace","value":"$(NAMESPACE)"}]' + sed 's|minikura-operator:latest|$(IMG)|' config/manager/deployment.yaml | kubectl apply -n $(NAMESPACE) -f - diff --git a/operator/api/v1alpha1/minecraftserver_types.go b/operator/api/v1alpha1/minecraftserver_types.go index 556b9b7..7cca351 100644 --- a/operator/api/v1alpha1/minecraftserver_types.go +++ b/operator/api/v1alpha1/minecraftserver_types.go @@ -141,7 +141,7 @@ type MinecraftServerStatus struct { // +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.spec.type` // +kubebuilder:printcolumn:name="Version",type=string,JSONPath=`.spec.minecraftVersion` // +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` -// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.readyReplicas` // +kubebuilder:printcolumn:name="Endpoint",type=string,JSONPath=`.status.endpoint` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type MinecraftServer struct { diff --git a/operator/api/v1alpha1/reverseproxyserver_types.go b/operator/api/v1alpha1/reverseproxyserver_types.go index 44f99c1..bf67c0d 100644 --- a/operator/api/v1alpha1/reverseproxyserver_types.go +++ b/operator/api/v1alpha1/reverseproxyserver_types.go @@ -89,7 +89,7 @@ type ReverseProxyServerStatus struct { // +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.spec.type` // +kubebuilder:printcolumn:name="Address",type=string,JSONPath=`.spec.externalAddress` // +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` -// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.readyReplicas` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type ReverseProxyServer struct { metav1.TypeMeta `json:",inline"` diff --git a/operator/config/crd/minikura.kirameki.cafe_minecraftservers.yaml b/operator/config/crd/minikura.kirameki.cafe_minecraftservers.yaml index 9e450be..2bc5dce 100644 --- a/operator/config/crd/minikura.kirameki.cafe_minecraftservers.yaml +++ b/operator/config/crd/minikura.kirameki.cafe_minecraftservers.yaml @@ -28,7 +28,7 @@ spec: type: string - jsonPath: .status.readyReplicas name: Ready - type: string + type: integer - jsonPath: .status.endpoint name: Endpoint type: string diff --git a/operator/config/crd/minikura.kirameki.cafe_reverseproxyservers.yaml b/operator/config/crd/minikura.kirameki.cafe_reverseproxyservers.yaml index 67f02d9..711d3af 100644 --- a/operator/config/crd/minikura.kirameki.cafe_reverseproxyservers.yaml +++ b/operator/config/crd/minikura.kirameki.cafe_reverseproxyservers.yaml @@ -28,7 +28,7 @@ spec: type: string - jsonPath: .status.readyReplicas name: Ready - type: string + type: integer - jsonPath: .metadata.creationTimestamp name: Age type: date diff --git a/operator/config/manager/deployment.yaml b/operator/config/manager/deployment.yaml index 4dc554c..407871c 100644 --- a/operator/config/manager/deployment.yaml +++ b/operator/config/manager/deployment.yaml @@ -29,8 +29,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: NODE_ENV - value: production ports: - name: metrics containerPort: 8080 diff --git a/operator/config/rbac/backend.yaml b/operator/config/rbac/backend.yaml new file mode 100644 index 0000000..8d3b0e2 --- /dev/null +++ b/operator/config/rbac/backend.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: minikura-backend +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: minikura-backend-operator-resources +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "create", "update", "delete"] + - apiGroups: ["minikura.kirameki.cafe"] + resources: ["minecraftservers", "reverseproxyservers"] + verbs: ["get", "list", "create", "update", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: minikura-backend-operator-resources +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: minikura-backend-operator-resources +subjects: + - kind: ServiceAccount + name: minikura-backend + namespace: minikura diff --git a/operator/internal/controller/reverseproxyserver_controller.go b/operator/internal/controller/reverseproxyserver_controller.go index 2377e94..4f61747 100644 --- a/operator/internal/controller/reverseproxyserver_controller.go +++ b/operator/internal/controller/reverseproxyserver_controller.go @@ -50,10 +50,38 @@ func (r *ReverseProxyServerReconciler) Reconcile(ctx context.Context, req ctrl.R if err := apply(ctx, r.Client, &rp, resources.ProxyDeployment(&rp), r.Scheme); err != nil { return r.fail(ctx, &rp, "DeploymentFailed", err) } + if err := r.pruneStaleResources(ctx, &rp); err != nil { + return r.fail(ctx, &rp, "PruneFailed", err) + } return ctrl.Result{}, r.updateStatus(ctx, &rp) } +func (r *ReverseProxyServerReconciler) pruneStaleResources(ctx context.Context, rp *v1alpha1.ReverseProxyServer) error { + current := resources.ProxyName(rp.Spec.Type, rp.Name) + for _, kind := range []v1alpha1.ProxyKind{v1alpha1.ProxyVelocity, v1alpha1.ProxyBungeeCord} { + name := resources.ProxyName(kind, rp.Name) + if name == current { + continue + } + for _, obj := range []client.Object{&appsv1.Deployment{}, &corev1.Service{}, &corev1.ConfigMap{}} { + key := client.ObjectKey{Name: name, Namespace: rp.Namespace} + if err := r.Get(ctx, key, obj); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err + } + if metav1.IsControlledBy(obj, rp) { + if err := r.Delete(ctx, obj); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + } + } + return nil +} + func (r *ReverseProxyServerReconciler) backends(ctx context.Context, rp *v1alpha1.ReverseProxyServer) ([]string, error) { selector := labels.Everything() if rp.Spec.BackendSelector != nil { diff --git a/operator/internal/resources/common.go b/operator/internal/resources/common.go index 477f958..32a5b4a 100644 --- a/operator/internal/resources/common.go +++ b/operator/internal/resources/common.go @@ -13,6 +13,7 @@ import ( const ( MinecraftImage = "itzg/minecraft-server" + ProxyImage = "itzg/mc-proxy:latest" ContainerPort = 25565 DefaultHeapPercent = 80 ) diff --git a/operator/internal/resources/reverseproxyserver.go b/operator/internal/resources/reverseproxyserver.go index cfafc22..3c843d3 100644 --- a/operator/internal/resources/reverseproxyserver.go +++ b/operator/internal/resources/reverseproxyserver.go @@ -46,8 +46,8 @@ func ProxyService(rp *v1alpha1.ReverseProxyServer) *corev1.Service { func proxyEnv(rp *v1alpha1.ReverseProxyServer) []corev1.EnvVar { env := []corev1.EnvVar{ - {Name: "EULA", Value: "TRUE"}, {Name: "TYPE", Value: string(rp.Spec.Type)}, + {Name: "NETWORKADDRESS_CACHE_TTL", Value: "30"}, {Name: "MINIKURA_EXTERNAL_ADDRESS", Value: rp.Spec.ExternalAddress}, } @@ -83,7 +83,7 @@ func ProxyDeployment(rp *v1alpha1.ReverseProxyServer) *appsv1.Deployment { Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: "proxy", - Image: MinecraftImage, + Image: ProxyImage, Ports: []corev1.ContainerPort{{ Name: "minecraft", ContainerPort: rp.Spec.ListenPort, diff --git a/operator/internal/resources/reverseproxyserver_test.go b/operator/internal/resources/reverseproxyserver_test.go new file mode 100644 index 0000000..379f3c8 --- /dev/null +++ b/operator/internal/resources/reverseproxyserver_test.go @@ -0,0 +1,43 @@ +package resources + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1alpha1 "github.com/YuzuZensai/Minikura/operator/api/v1alpha1" +) + +func testProxy() *v1alpha1.ReverseProxyServer { + return &v1alpha1.ReverseProxyServer{ + ObjectMeta: metav1.ObjectMeta{Name: "edge", Namespace: "minikura"}, + Spec: v1alpha1.ReverseProxyServerSpec{ + Type: v1alpha1.ProxyVelocity, + ExternalAddress: "play.example.com", + ExternalPort: 25565, + ListenPort: 25577, + Resources: v1alpha1.Resources{MemoryLimitMB: 512}, + }, + } +} + +func TestProxyDeploymentUsesProxyImage(t *testing.T) { + container := ProxyDeployment(testProxy()).Spec.Template.Spec.Containers[0] + if container.Image != ProxyImage { + t.Errorf("image = %q, want %q", container.Image, ProxyImage) + } +} + +func TestProxyEnvCarriesRuntimeConfig(t *testing.T) { + env := proxyEnv(testProxy()) + for _, want := range []struct{ key, value string }{ + {"TYPE", "VELOCITY"}, + {"NETWORKADDRESS_CACHE_TTL", "30"}, + {"MINIKURA_EXTERNAL_ADDRESS", "play.example.com"}, + } { + got, ok := envValue(env, want.key) + if !ok || got != want.value { + t.Errorf("%s = %q, %v; want %q", want.key, got, ok, want.value) + } + } +} diff --git a/package.json b/package.json index a90df23..8a3778c 100644 --- a/package.json +++ b/package.json @@ -20,10 +20,11 @@ "db:studio": "bun --filter @minikura/db studio", "db:push": "bun --filter @minikura/db push", "db:reset": "bun --filter @minikura/db reset", - "k8s:dev": "bun --filter @minikura/k8s-operator dev", - "k8s:build": "bun --filter @minikura/k8s-operator build", - "k8s:start": "bun --filter @minikura/k8s-operator start", - "k8s:crd": "bun --filter @minikura/k8s-operator apply-crds", + "operator:dev": "make -C operator run", + "operator:build": "make -C operator build", + "operator:test": "make -C operator test", + "operator:crds": "make -C operator install-crds", + "operator:deploy": "make -C operator deploy", "setup": "bash scripts/install.sh" }, "devDependencies": { diff --git a/packages/k8s-operator/Dockerfile b/packages/k8s-operator/Dockerfile deleted file mode 100644 index 9800b92..0000000 --- a/packages/k8s-operator/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM node:18-alpine AS build -WORKDIR /app - -# Copy package files -COPY package.json ./ -COPY tsconfig.json ./ - -# Copy source files -COPY src/ ./src/ - -# Install dependencies -RUN npm install - -# Build -RUN npm run build - -# Create production image -FROM node:18-alpine -WORKDIR /app - -# Copy package.json and built files -COPY --from=build /app/package.json ./ -COPY --from=build /app/dist ./dist - -# Install production dependencies -RUN npm install --production - -# Set environment variables -ENV NODE_ENV=production - -# Run -CMD ["node", "dist/index.js"] \ No newline at end of file diff --git a/packages/k8s-operator/package.json b/packages/k8s-operator/package.json deleted file mode 100644 index c1973ee..0000000 --- a/packages/k8s-operator/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@minikura/k8s-operator", - "version": "1.0.0", - "description": "Kubernetes operator for Minikura that syncs database to Kubernetes resources", - "main": "dist/index.js", - "type": "module", - "scripts": { - "build": "tsc", - "start": "node dist/index.js", - "dev": "tsx watch src/index.ts", - "dev:bun": "bun --watch src/index.ts", - "watch": "tsx watch src/index.ts", - "apply-crds": "tsx src/scripts/apply-crds.ts", - "typecheck": "tsc --noEmit" - }, - "dependencies": { - "@kubernetes/client-node": "^1.4.0", - "@minikura/api": "workspace:*", - "@minikura/db": "workspace:*", - "dotenv-mono": "^1.5.1", - "node-fetch": "^3.3.2", - "pg": "^8.23.0", - "pino": "^10.3.1", - "pino-pretty": "^13.1.3", - "undici": "^8.10.0", - "yaml": "^2.9.0" - }, - "devDependencies": { - "@types/node": "^26.2.0", - "tsx": "^4.23.12", - "typescript": "^7.0.2" - } -} diff --git a/packages/k8s-operator/src/config/constants.ts b/packages/k8s-operator/src/config/constants.ts deleted file mode 100644 index 33517a4..0000000 --- a/packages/k8s-operator/src/config/constants.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { dotenvLoad } from "dotenv-mono"; - -const _dotenv = dotenvLoad(); - -export { API_GROUP, LABEL_PREFIX } from "@minikura/api"; -export const API_VERSION = "v1alpha1"; - -export const KUBERNETES_NAMESPACE_ENV = process.env.KUBERNETES_NAMESPACE; -export const NAMESPACE = process.env.KUBERNETES_NAMESPACE || "minikura"; - -export const ENABLE_CRD_REFLECTION = process.env.ENABLE_CRD_REFLECTION === "true"; - -export const RESOURCE_TYPES = { - MINECRAFT_SERVER: { - kind: "MinecraftServer", - plural: "minecraftservers", - singular: "minecraftserver", - shortNames: ["mcs"], - }, - REVERSE_PROXY_SERVER: { - kind: "ReverseProxyServer", - plural: "reverseproxyservers", - singular: "reverseproxyserver", - shortNames: ["rps"], - }, -}; - -export const SYNC_INTERVAL = 30 * 1000; - -export const IMAGES = { - MINECRAFT: "itzg/minecraft-server", - REVERSE_PROXY: "itzg/minecraft-server", -}; - -export const DEFAULTS = { - MEMORY: "1G", - CPU_REQUEST: "250m", - CPU_LIMIT: "1000m", - STORAGE_SIZE: "1Gi", -}; diff --git a/packages/k8s-operator/src/config/resource-defaults.ts b/packages/k8s-operator/src/config/resource-defaults.ts deleted file mode 100644 index 6925e75..0000000 --- a/packages/k8s-operator/src/config/resource-defaults.ts +++ /dev/null @@ -1,15 +0,0 @@ -export const RESOURCE_DEFAULTS = { - server: { - memory: "1G", - javaMemoryFactor: 0.8, - }, - proxy: { - memory: "512M", - javaMemoryFactor: 0.8, - }, -} as const; - -export const JAVA_MEMORY_FACTOR = 0.8; - -export const DEFAULT_SERVER_MEMORY = "1G"; -export const DEFAULT_PROXY_MEMORY = "512M"; diff --git a/packages/k8s-operator/src/controllers/base-controller.ts b/packages/k8s-operator/src/controllers/base-controller.ts deleted file mode 100644 index 92f7a8b..0000000 --- a/packages/k8s-operator/src/controllers/base-controller.ts +++ /dev/null @@ -1,53 +0,0 @@ -import type { PrismaClient } from "@minikura/db"; -import type { Logger } from "pino"; -import { SYNC_INTERVAL } from "../config/constants"; -import { KubernetesClient } from "../utils/k8s-client"; -import { createLogger } from "../utils/logger"; - -export abstract class BaseController { - protected prisma: PrismaClient; - protected k8sClient: KubernetesClient; - protected namespace: string; - protected logger: Logger; - private intervalId: ReturnType | null = null; - - constructor(prisma: PrismaClient, namespace: string) { - this.prisma = prisma; - this.k8sClient = KubernetesClient.getInstance(); - this.namespace = namespace; - this.logger = createLogger({ controller: this.getControllerName() }); - } - - public startWatching(): void { - this.logger.info( - { namespace: this.namespace, syncInterval: SYNC_INTERVAL }, - "Starting controller watch loop" - ); - - this.syncResources().catch((err) => { - this.logger.error({ err }, "Error during initial resource synchronization"); - }); - - this.intervalId = setInterval(() => { - this.syncResources().catch((err) => { - this.logger.error({ err }, "Error during periodic resource synchronization"); - }); - }, SYNC_INTERVAL); - - this.logger.debug( - { intervalMs: SYNC_INTERVAL }, - "Polling interval established for resource synchronization" - ); - } - - public stopWatching(): void { - if (this.intervalId) { - clearInterval(this.intervalId); - this.intervalId = null; - this.logger.info("Controller watch loop stopped"); - } - } - - protected abstract getControllerName(): string; - protected abstract syncResources(): Promise; -} diff --git a/packages/k8s-operator/src/controllers/reverse-proxy-controller.ts b/packages/k8s-operator/src/controllers/reverse-proxy-controller.ts deleted file mode 100644 index 5734ec0..0000000 --- a/packages/k8s-operator/src/controllers/reverse-proxy-controller.ts +++ /dev/null @@ -1,124 +0,0 @@ -import type { CustomEnvironmentVariable, ReverseProxyServer } from "@minikura/db"; -import { - createReverseProxyServer, - deleteReverseProxyServer, -} from "../resources/reverseProxyServer"; -import type { ReverseProxyConfig } from "../types"; -import { BaseController } from "./base-controller"; - -type ReverseProxyWithEnvVars = ReverseProxyServer & { - env_variables: CustomEnvironmentVariable[]; -}; - -export class ReverseProxyController extends BaseController { - private deployedProxies = new Map(); - - protected getControllerName(): string { - return "ReverseProxyController"; - } - - protected async syncResources(): Promise { - try { - const appsApi = this.k8sClient.getAppsApi(); - const coreApi = this.k8sClient.getCoreApi(); - const networkingApi = this.k8sClient.getNetworkingApi(); - - const proxies = (await this.prisma.reverseProxyServer.findMany({ - include: { - env_variables: true, - }, - })) as ReverseProxyWithEnvVars[]; - - const currentProxyIds = new Set(proxies.map((proxy) => proxy.id)); - - for (const [proxyId, proxy] of this.deployedProxies.entries()) { - if (!currentProxyIds.has(proxyId)) { - this.logger.info( - { proxyId, proxyType: proxy.type }, - "Reverse proxy removed from database, deleting K8s resources" - ); - await deleteReverseProxyServer(proxy.id, proxy.type, appsApi, coreApi, this.namespace); - this.deployedProxies.delete(proxyId); - } - } - - for (const proxy of proxies) { - const deployedProxy = this.deployedProxies.get(proxy.id); - - if (!deployedProxy || this.hasProxyChanged(deployedProxy, proxy)) { - const action = !deployedProxy ? "Creating" : "Updating"; - this.logger.info( - { - proxyId: proxy.id, - proxyType: proxy.type, - action: action.toLowerCase(), - externalAddress: proxy.external_address, - externalPort: proxy.external_port, - listenPort: proxy.listen_port, - }, - `${action} reverse proxy server in Kubernetes` - ); - - const proxyConfig: ReverseProxyConfig = { - id: proxy.id, - external_address: proxy.external_address, - external_port: proxy.external_port, - listen_port: proxy.listen_port, - description: proxy.description, - apiKey: proxy.api_key, - type: proxy.type, - memory: proxy.memory, - service_type: proxy.service_type, - env_variables: proxy.env_variables?.map((ev) => ({ - key: ev.key, - value: ev.value, - })), - }; - - await createReverseProxyServer( - proxyConfig, - appsApi, - coreApi, - networkingApi, - this.namespace - ); - - this.deployedProxies.set(proxy.id, { ...proxy }); - } - } - } catch (error) { - this.logger.error({ err: error }, "Failed to sync reverse proxy servers to Kubernetes"); - throw error; - } - } - - private hasProxyChanged( - oldProxy: ReverseProxyWithEnvVars, - newProxy: ReverseProxyWithEnvVars - ): boolean { - const basicPropsChanged = - oldProxy.external_address !== newProxy.external_address || - oldProxy.external_port !== newProxy.external_port || - oldProxy.listen_port !== newProxy.listen_port || - oldProxy.description !== newProxy.description || - oldProxy.service_type !== newProxy.service_type || - oldProxy.memory !== newProxy.memory || - oldProxy.type !== newProxy.type; - - if (basicPropsChanged) return true; - - const oldEnvVars = oldProxy.env_variables || []; - const newEnvVars = newProxy.env_variables || []; - - if (oldEnvVars.length !== newEnvVars.length) return true; - - for (const newEnv of newEnvVars) { - const oldEnv = oldEnvVars.find((e) => e.key === newEnv.key); - if (!oldEnv || oldEnv.value !== newEnv.value) { - return true; - } - } - - return false; - } -} diff --git a/packages/k8s-operator/src/controllers/server-controller.ts b/packages/k8s-operator/src/controllers/server-controller.ts deleted file mode 100644 index 6cbb3dd..0000000 --- a/packages/k8s-operator/src/controllers/server-controller.ts +++ /dev/null @@ -1,107 +0,0 @@ -import type { CustomEnvironmentVariable, Server } from "@minikura/db"; -import { createServer, deleteServer } from "../resources/server"; -import type { ServerConfig } from "../types"; -import { BaseController } from "./base-controller"; - -type ServerWithEnvVars = Server & { - env_variables: CustomEnvironmentVariable[]; -}; - -export class ServerController extends BaseController { - private deployedServers = new Map(); - - protected getControllerName(): string { - return "ServerController"; - } - - protected async syncResources(): Promise { - try { - const appsApi = this.k8sClient.getAppsApi(); - const coreApi = this.k8sClient.getCoreApi(); - const networkingApi = this.k8sClient.getNetworkingApi(); - - const servers = (await this.prisma.server.findMany({ - include: { - env_variables: true, - }, - })) as ServerWithEnvVars[]; - - const currentServerIds = new Set(servers.map((server) => server.id)); - - for (const [serverId, server] of this.deployedServers.entries()) { - if (!currentServerIds.has(serverId)) { - this.logger.info( - { serverId, serverName: server.id }, - "Server removed from database, deleting K8s resources" - ); - await deleteServer(serverId, appsApi, coreApi, this.namespace); - this.deployedServers.delete(serverId); - } - } - - for (const server of servers) { - const deployedServer = this.deployedServers.get(server.id); - - if (!deployedServer || this.hasServerChanged(deployedServer, server)) { - const action = !deployedServer ? "Creating" : "Updating"; - this.logger.info( - { - serverId: server.id, - serverType: server.type, - action: action.toLowerCase(), - memory: server.memory, - port: server.listen_port, - }, - `${action} Minecraft server in Kubernetes` - ); - - const serverConfig: ServerConfig = { - id: server.id, - type: server.type, - apiKey: server.api_key, - description: server.description, - listen_port: server.listen_port, - memory: server.memory, - service_type: server.service_type, - env_variables: server.env_variables?.map((ev) => ({ - key: ev.key, - value: ev.value, - })), - }; - - await createServer(serverConfig, appsApi, coreApi, networkingApi, this.namespace); - - this.deployedServers.set(server.id, { ...server }); - } - } - } catch (error) { - this.logger.error({ err: error }, "Failed to sync servers to Kubernetes"); - throw error; - } - } - - private hasServerChanged(oldServer: ServerWithEnvVars, newServer: ServerWithEnvVars): boolean { - const basicPropsChanged = - oldServer.type !== newServer.type || - oldServer.listen_port !== newServer.listen_port || - oldServer.description !== newServer.description || - oldServer.service_type !== newServer.service_type || - oldServer.memory !== newServer.memory; - - if (basicPropsChanged) return true; - - const oldEnvVars = oldServer.env_variables || []; - const newEnvVars = newServer.env_variables || []; - - if (oldEnvVars.length !== newEnvVars.length) return true; - - for (const newEnv of newEnvVars) { - const oldEnv = oldEnvVars.find((e) => e.key === newEnv.key); - if (!oldEnv || oldEnv.value !== newEnv.value) { - return true; - } - } - - return false; - } -} diff --git a/packages/k8s-operator/src/crds/rbac.ts b/packages/k8s-operator/src/crds/rbac.ts deleted file mode 100644 index 3619a87..0000000 --- a/packages/k8s-operator/src/crds/rbac.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { API_GROUP, NAMESPACE } from "../config/constants"; - -export const minikuraNamespace = { - apiVersion: "v1", - kind: "Namespace", - metadata: { - name: NAMESPACE, - }, -}; - -export const minikuraServiceAccount = { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "minikura-operator", - namespace: NAMESPACE, - }, -}; - -export const minikuraClusterRole = { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "ClusterRole", - metadata: { - name: "minikura-operator-role", - }, - rules: [ - { - apiGroups: [""], - resources: ["configmaps", "services", "secrets"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], - }, - { - apiGroups: ["apps"], - resources: ["deployments", "statefulsets"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], - }, - { - apiGroups: ["networking.k8s.io"], - resources: ["ingresses"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], - }, - { - apiGroups: ["apiextensions.k8s.io"], - resources: ["customresourcedefinitions"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], - }, - { - apiGroups: [API_GROUP], - resources: ["minecraftservers", "velocityproxies"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], - }, - { - apiGroups: [API_GROUP], - resources: ["minecraftservers/status", "velocityproxies/status"], - verbs: ["get", "update", "patch"], - }, - ], -}; - -export const minikuraClusterRoleBinding = { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "ClusterRoleBinding", - metadata: { - name: "minikura-operator-role-binding", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "minikura-operator", - namespace: NAMESPACE, - }, - ], - roleRef: { - kind: "ClusterRole", - name: "minikura-operator-role", - apiGroup: "rbac.authorization.k8s.io", - }, -}; - -export const minikuraOperatorDeployment = { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { - name: "minikura-operator", - namespace: NAMESPACE, - }, - spec: { - replicas: 1, - selector: { - matchLabels: { - app: "minikura-operator", - }, - }, - template: { - metadata: { - labels: { - app: "minikura-operator", - }, - }, - spec: { - serviceAccountName: "minikura-operator", - containers: [ - { - name: "operator", - image: "${REGISTRY_URL}/minikura-operator:latest", - env: [ - { - name: "DATABASE_URL", - valueFrom: { - secretKeyRef: { - name: "minikura-operator-secrets", - key: "DATABASE_URL", - }, - }, - }, - { - name: "KUBERNETES_NAMESPACE", - value: NAMESPACE, - }, - { - name: "USE_CRDS", - value: "true", - }, - ], - resources: { - requests: { - memory: "256Mi", - cpu: "200m", - }, - limits: { - memory: "512Mi", - cpu: "500m", - }, - }, - livenessProbe: { - exec: { - command: ["bun", "-e", "console.log('Health check')"], - }, - initialDelaySeconds: 30, - periodSeconds: 30, - }, - readinessProbe: { - exec: { - command: ["bun", "-e", "console.log('Ready check')"], - }, - initialDelaySeconds: 5, - periodSeconds: 10, - }, - }, - ], - }, - }, - }, -}; diff --git a/packages/k8s-operator/src/crds/reverseProxy.ts b/packages/k8s-operator/src/crds/reverseProxy.ts deleted file mode 100644 index 51252b8..0000000 --- a/packages/k8s-operator/src/crds/reverseProxy.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { API_GROUP, API_VERSION, RESOURCE_TYPES } from "../config/constants"; - -export const REVERSE_PROXY_SERVER_CRD = { - apiVersion: "apiextensions.k8s.io/v1", - kind: "CustomResourceDefinition", - metadata: { - name: `${RESOURCE_TYPES.REVERSE_PROXY_SERVER.plural}.${API_GROUP}`, - }, - spec: { - group: API_GROUP, - versions: [ - { - name: API_VERSION, - served: true, - storage: true, - schema: { - openAPIV3Schema: { - type: "object", - properties: { - spec: { - type: "object", - required: ["id", "external_address", "external_port"], - properties: { - id: { - type: "string", - pattern: "^[a-zA-Z0-9-_]+$", - description: "ID of the reverse proxy server", - }, - description: { - type: "string", - nullable: true, - description: "Optional description of the server", - }, - external_address: { - type: "string", - description: "External address of the proxy server", - }, - external_port: { - type: "integer", - minimum: 1, - maximum: 65535, - description: "External port of the proxy server", - }, - listen_port: { - type: "integer", - minimum: 1, - maximum: 65535, - default: 25565, - nullable: true, - description: "Port the proxy server listens on internally", - }, - type: { - type: "string", - enum: ["VELOCITY", "BUNGEECORD"], - default: "VELOCITY", - nullable: true, - description: "Type of the reverse proxy server", - }, - memory: { - type: "string", - default: "512M", - nullable: true, - description: "Memory allocation for the server", - }, - environmentVariables: { - type: "array", - nullable: true, - items: { - type: "object", - required: ["key", "value"], - properties: { - key: { - type: "string", - description: "Environment variable key", - }, - value: { - type: "string", - description: "Environment variable value", - }, - }, - }, - }, - }, - }, - status: { - type: "object", - nullable: true, - properties: { - phase: { - type: "string", - enum: ["Pending", "Running", "Failed"], - description: "Current phase of the server", - }, - message: { - type: "string", - nullable: true, - description: "Detailed message about the current status", - }, - apiKey: { - type: "string", - nullable: true, - description: "API key for server communication", - }, - internalId: { - type: "string", - nullable: true, - description: "Internal ID assigned by Minikura", - }, - lastSyncedAt: { - type: "string", - nullable: true, - description: "Last time the server was synced with Kubernetes", - }, - }, - }, - }, - }, - }, - additionalPrinterColumns: [ - { - name: "Type", - type: "string", - jsonPath: ".spec.type", - }, - { - name: "External Address", - type: "string", - jsonPath: ".spec.external_address", - }, - { - name: "External Port", - type: "integer", - jsonPath: ".spec.external_port", - }, - { - name: "Status", - type: "string", - jsonPath: ".status.phase", - }, - { - name: "Age", - type: "date", - jsonPath: ".metadata.creationTimestamp", - }, - ], - }, - ], - scope: "Namespaced", - names: { - singular: RESOURCE_TYPES.REVERSE_PROXY_SERVER.singular, - plural: RESOURCE_TYPES.REVERSE_PROXY_SERVER.plural, - kind: RESOURCE_TYPES.REVERSE_PROXY_SERVER.kind, - shortNames: RESOURCE_TYPES.REVERSE_PROXY_SERVER.shortNames, - }, - }, -}; diff --git a/packages/k8s-operator/src/crds/server.ts b/packages/k8s-operator/src/crds/server.ts deleted file mode 100644 index e2ea1cc..0000000 --- a/packages/k8s-operator/src/crds/server.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { API_GROUP, API_VERSION, RESOURCE_TYPES } from "../config/constants"; - -export const MINECRAFT_SERVER_CRD = { - apiVersion: "apiextensions.k8s.io/v1", - kind: "CustomResourceDefinition", - metadata: { - name: `${RESOURCE_TYPES.MINECRAFT_SERVER.plural}.${API_GROUP}`, - }, - spec: { - group: API_GROUP, - versions: [ - { - name: API_VERSION, - served: true, - storage: true, - schema: { - openAPIV3Schema: { - type: "object", - properties: { - spec: { - type: "object", - required: ["id", "type", "listen_port"], - properties: { - id: { - type: "string", - pattern: "^[a-zA-Z0-9-_]+$", - description: "ID of the Minecraft server", - }, - description: { - type: "string", - nullable: true, - description: "Optional description of the server", - }, - listen_port: { - type: "integer", - minimum: 1, - maximum: 65535, - description: "Port the server listens on", - }, - type: { - type: "string", - enum: ["STATEFUL", "STATELESS"], - description: "Type of the server", - }, - memory: { - type: "string", - nullable: true, - default: "1G", - description: "Memory allocation for the server", - }, - environmentVariables: { - type: "array", - nullable: true, - items: { - type: "object", - required: ["key", "value"], - properties: { - key: { - type: "string", - description: "Environment variable key", - }, - value: { - type: "string", - description: "Environment variable value", - }, - }, - }, - }, - }, - }, - status: { - type: "object", - nullable: true, - properties: { - phase: { - type: "string", - enum: ["Pending", "Running", "Failed"], - description: "Current phase of the server", - }, - message: { - type: "string", - nullable: true, - description: "Detailed message about the current status", - }, - apiKey: { - type: "string", - nullable: true, - description: "API key for server communication", - }, - internalId: { - type: "string", - nullable: true, - description: "Internal ID assigned by Minikura", - }, - lastSyncedAt: { - type: "string", - nullable: true, - description: "Last time the server was synced with Kubernetes", - }, - }, - }, - }, - }, - }, - additionalPrinterColumns: [ - { - name: "Type", - type: "string", - jsonPath: ".spec.type", - }, - { - name: "Status", - type: "string", - jsonPath: ".status.phase", - }, - { - name: "Age", - type: "date", - jsonPath: ".metadata.creationTimestamp", - }, - ], - }, - ], - scope: "Namespaced", - names: { - singular: RESOURCE_TYPES.MINECRAFT_SERVER.singular, - plural: RESOURCE_TYPES.MINECRAFT_SERVER.plural, - kind: RESOURCE_TYPES.MINECRAFT_SERVER.kind, - shortNames: RESOURCE_TYPES.MINECRAFT_SERVER.shortNames, - }, - }, -}; diff --git a/packages/k8s-operator/src/index.ts b/packages/k8s-operator/src/index.ts deleted file mode 100644 index 66bbc17..0000000 --- a/packages/k8s-operator/src/index.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { dotenvLoad } from "dotenv-mono"; - -const _dotenv = dotenvLoad(); - -import { prisma } from "@minikura/db"; -import { ENABLE_CRD_REFLECTION, NAMESPACE } from "./config/constants"; -import { ReverseProxyController } from "./controllers/reverse-proxy-controller"; -import { ServerController } from "./controllers/server-controller"; -import { setupCRDRegistration } from "./utils/crd-registrar"; -import { KubernetesClient } from "./utils/k8s-client"; -import { logger } from "./utils/logger"; - -async function main() { - logger.info( - { namespace: NAMESPACE, crdReflection: ENABLE_CRD_REFLECTION }, - "Starting Minikura Kubernetes Operator" - ); - - try { - const k8sClient = KubernetesClient.getInstance(); - logger.info({ namespace: NAMESPACE }, "Successfully connected to Kubernetes cluster"); - - const serverController = new ServerController(prisma, NAMESPACE); - const reverseProxyController = new ReverseProxyController(prisma, NAMESPACE); - - serverController.startWatching(); - reverseProxyController.startWatching(); - - if (ENABLE_CRD_REFLECTION) { - logger.info("CRD reflection enabled - will create Custom Resources to mirror database state"); - try { - await setupCRDRegistration(prisma, k8sClient, NAMESPACE); - logger.info("CRD registration completed successfully"); - } catch (error: any) { - logger.error( - { - err: error, - message: error.message, - statusCode: error.response?.statusCode, - body: error.response?.body, - }, - "Failed to setup CRD registration, continuing without CRD reflection" - ); - logger.warn( - "Kubernetes resources (Deployments, Services) will still be created, but Custom Resources will not be reflected" - ); - } - } - - logger.info("Minikura Kubernetes Operator is now running and watching for changes"); - - process.on("SIGINT", gracefulShutdown); - process.on("SIGTERM", gracefulShutdown); - - function gracefulShutdown() { - logger.info("Received shutdown signal, shutting down gracefully"); - serverController.stopWatching(); - reverseProxyController.stopWatching(); - prisma.$disconnect(); - logger.info("All resources released, exiting process"); - process.exit(0); - } - } catch (error: any) { - logger.fatal( - { - err: error, - message: error.message, - statusCode: error.response?.statusCode, - body: error.response?.body, - stack: error.stack, - }, - "Failed to start Minikura Kubernetes Operator" - ); - process.exit(1); - } -} - -main().catch((error) => { - logger.fatal({ err: error }, "Unhandled error in main process"); - process.exit(1); -}); diff --git a/packages/k8s-operator/src/resources/reverseProxyServer.ts b/packages/k8s-operator/src/resources/reverseProxyServer.ts deleted file mode 100644 index da3c9d8..0000000 --- a/packages/k8s-operator/src/resources/reverseProxyServer.ts +++ /dev/null @@ -1,230 +0,0 @@ -import type * as k8s from "@kubernetes/client-node"; -import type { ReverseProxyServerType } from "@minikura/db"; -import { LABEL_PREFIX } from "../config/constants"; -import { DEFAULT_PROXY_MEMORY, JAVA_MEMORY_FACTOR } from "../config/resource-defaults"; -import type { ReverseProxyConfig } from "../types"; -import { logger } from "../utils/logger"; -import { calculateJavaMemory, convertToK8sFormat } from "../utils/memory"; -import { mapServiceType } from "../utils/service-type"; - -export async function createReverseProxyServer( - server: ReverseProxyConfig, - appsApi: k8s.AppsV1Api, - coreApi: k8s.CoreV1Api, - _networkingApi: k8s.NetworkingV1Api, - namespace: string -): Promise { - logger.debug( - { proxyId: server.id, proxyType: server.type, namespace }, - "Creating reverse proxy server" - ); - - const serverType = server.type.toLowerCase(); - const serverName = `${serverType}-${server.id}`; - - const configMap = { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { - name: `${serverName}-config`, - namespace: namespace, - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: serverType, - [`${LABEL_PREFIX}/proxy-id`]: server.id, - }, - }, - data: { - "minikura-api-key": server.apiKey, - }, - }; - - try { - await coreApi.createNamespacedConfigMap({ namespace, body: configMap }); - logger.debug({ proxyId: server.id, resource: "ConfigMap" }, "Created ConfigMap"); - } catch (error: any) { - if (error.code === 409) { - await coreApi.replaceNamespacedConfigMap({ - name: `${serverName}-config`, - namespace, - body: configMap, - }); - logger.debug({ proxyId: server.id, resource: "ConfigMap" }, "Updated ConfigMap"); - } else { - throw error; - } - } - - const service = { - apiVersion: "v1", - kind: "Service", - metadata: { - name: serverName, - namespace: namespace, - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: serverType, - [`${LABEL_PREFIX}/proxy-id`]: server.id, - }, - }, - spec: { - selector: { - app: serverName, - }, - ports: [ - { - port: server.external_port, - targetPort: server.listen_port, - protocol: "TCP", - name: "minecraft", - }, - ], - type: mapServiceType(server.service_type, "LoadBalancer"), - }, - }; - - try { - await coreApi.createNamespacedService({ namespace, body: service }); - logger.debug({ proxyId: server.id, resource: "Service" }, "Created Service"); - } catch (error: any) { - if (error.code === 409) { - await coreApi.replaceNamespacedService({ name: serverName, namespace, body: service }); - logger.debug({ proxyId: server.id, resource: "Service" }, "Updated Service"); - } else { - throw error; - } - } - - const deployment = { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { - name: serverName, - namespace: namespace, - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: serverType, - [`${LABEL_PREFIX}/proxy-id`]: server.id, - }, - }, - spec: { - replicas: 1, - selector: { - matchLabels: { - app: serverName, - }, - }, - template: { - metadata: { - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: serverType, - [`${LABEL_PREFIX}/proxy-id`]: server.id, - }, - }, - spec: { - containers: [ - { - name: serverType, - image: "itzg/mc-proxy:latest", - ports: [ - { - containerPort: server.listen_port, - name: "minecraft", - }, - ], - env: [ - { - name: "TYPE", - value: server.type, - }, - { - name: "NETWORKADDRESS_CACHE_TTL", - value: "30", - }, - { - name: "MEMORY", - value: calculateJavaMemory( - server.memory || DEFAULT_PROXY_MEMORY, - JAVA_MEMORY_FACTOR - ), - }, - ...(server.env_variables || []).map((ev) => ({ - name: ev.key, - value: ev.value, - })), - ], - readinessProbe: { - tcpSocket: { - port: server.listen_port, - }, - initialDelaySeconds: 30, - periodSeconds: 10, - }, - resources: { - requests: { - memory: convertToK8sFormat(server.memory || DEFAULT_PROXY_MEMORY), - cpu: "250m", - }, - limits: { - memory: convertToK8sFormat(server.memory || DEFAULT_PROXY_MEMORY), - cpu: "500m", - }, - }, - }, - ], - }, - }, - }, - }; - - try { - await appsApi.createNamespacedDeployment({ namespace, body: deployment }); - logger.debug({ proxyId: server.id, resource: "Deployment" }, "Created Deployment"); - } catch (error: any) { - if (error.code === 409) { - await appsApi.replaceNamespacedDeployment({ name: serverName, namespace, body: deployment }); - logger.debug({ proxyId: server.id, resource: "Deployment" }, "Updated Deployment"); - } else { - throw error; - } - } -} - -export async function deleteReverseProxyServer( - proxyId: string, - proxyType: ReverseProxyServerType, - appsApi: k8s.AppsV1Api, - coreApi: k8s.CoreV1Api, - namespace: string -): Promise { - const serverType = proxyType.toLowerCase(); - const name = `${serverType}-${proxyId}`; - - try { - await appsApi.deleteNamespacedDeployment({ name, namespace }); - logger.debug({ proxyId, resource: "Deployment" }, "Deleted Deployment"); - } catch (error: any) { - if (error.response?.statusCode !== 404) { - logger.error({ err: error, proxyId, resource: "Deployment" }, "Failed to delete Deployment"); - } - } - - try { - await coreApi.deleteNamespacedService({ name, namespace }); - logger.debug({ proxyId, resource: "Service" }, "Deleted Service"); - } catch (error: any) { - if (error.response?.statusCode !== 404) { - logger.error({ err: error, proxyId, resource: "Service" }, "Failed to delete Service"); - } - } - - try { - await coreApi.deleteNamespacedConfigMap({ name: `${name}-config`, namespace }); - logger.debug({ proxyId, resource: "ConfigMap" }, "Deleted ConfigMap"); - } catch (error: any) { - if (error.response?.statusCode !== 404) { - logger.error({ err: error, proxyId, resource: "ConfigMap" }, "Failed to delete ConfigMap"); - } - } -} diff --git a/packages/k8s-operator/src/resources/server.ts b/packages/k8s-operator/src/resources/server.ts deleted file mode 100644 index 47ff998..0000000 --- a/packages/k8s-operator/src/resources/server.ts +++ /dev/null @@ -1,426 +0,0 @@ -import type * as k8s from "@kubernetes/client-node"; -import { ServerType } from "@minikura/db"; -import { LABEL_PREFIX } from "../config/constants"; -import { DEFAULT_SERVER_MEMORY, JAVA_MEMORY_FACTOR } from "../config/resource-defaults"; -import type { ServerConfig } from "../types"; -import { logger } from "../utils/logger"; -import { calculateJavaMemory, convertToK8sFormat } from "../utils/memory"; -import { mapServiceType } from "../utils/service-type"; - -export async function createServer( - server: ServerConfig, - appsApi: k8s.AppsV1Api, - coreApi: k8s.CoreV1Api, - _networkingApi: k8s.NetworkingV1Api, - namespace: string -): Promise { - const serverName = `minecraft-${server.id}`; - - const configMap = { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { - name: `${serverName}-config`, - namespace: namespace, - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: server.type.toLowerCase(), - [`${LABEL_PREFIX}/server-id`]: server.id, - }, - }, - data: { - "server-type": server.type, - "minikura-api-key": server.apiKey, - }, - }; - - try { - await coreApi.createNamespacedConfigMap({ namespace, body: configMap }); - logger.debug({ serverId: server.id, resource: "ConfigMap" }, "Created ConfigMap"); - } catch (err: any) { - if (err.code === 409) { - await coreApi.replaceNamespacedConfigMap({ - name: `${serverName}-config`, - namespace, - body: configMap, - }); - logger.debug({ serverId: server.id, resource: "ConfigMap" }, "Updated ConfigMap"); - } else { - throw err; - } - } - - const service = { - apiVersion: "v1", - kind: "Service", - metadata: { - name: serverName, - namespace: namespace, - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: server.type.toLowerCase(), - [`${LABEL_PREFIX}/server-id`]: server.id, - }, - }, - spec: { - selector: { - app: serverName, - }, - ports: [ - { - port: server.listen_port, - targetPort: 25565, - protocol: "TCP", - name: "minecraft", - }, - ], - type: mapServiceType(server.service_type), - }, - }; - - try { - await coreApi.createNamespacedService({ namespace, body: service }); - logger.debug( - { serverId: server.id, resource: "Service", port: server.listen_port }, - "Created Service" - ); - } catch (err: any) { - if (err.code === 409) { - await coreApi.replaceNamespacedService({ name: serverName, namespace, body: service }); - logger.debug({ serverId: server.id, resource: "Service" }, "Updated Service"); - } else { - throw err; - } - } - - if (server.type === ServerType.STATELESS) { - await createDeployment(serverName, server, appsApi, namespace); - } else { - await createStatefulSet(serverName, server, appsApi, namespace); - } -} - -async function createDeployment( - serverName: string, - server: ServerConfig, - appsApi: k8s.AppsV1Api, - namespace: string -): Promise { - const deployment = { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { - name: serverName, - namespace: namespace, - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: "stateless", - [`${LABEL_PREFIX}/server-id`]: server.id, - }, - }, - spec: { - replicas: 1, - selector: { - matchLabels: { - app: serverName, - }, - }, - template: { - metadata: { - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: "stateless", - [`${LABEL_PREFIX}/server-id`]: server.id, - }, - }, - spec: { - containers: [ - { - name: "minecraft", - image: "itzg/minecraft-server", - ports: [ - { - containerPort: 25565, - name: "minecraft", - }, - ], - env: [ - { - name: "EULA", - value: "TRUE", - }, - { - name: "TYPE", - value: "VANILLA", - }, - { - name: "MEMORY", - value: calculateJavaMemory( - server.memory || DEFAULT_SERVER_MEMORY, - JAVA_MEMORY_FACTOR - ), - }, - { - name: "OPS", - value: "", - }, - { - name: "OVERRIDE_SERVER_PROPERTIES", - value: "true", - }, - { - name: "ENABLE_RCON", - value: "false", - }, - ...(server.env_variables || []).map((ev) => ({ - name: ev.key, - value: ev.value, - })), - ], - volumeMounts: [ - { - name: "config", - mountPath: "/config", - }, - ], - readinessProbe: { - tcpSocket: { - port: 25565, - }, - initialDelaySeconds: 30, - periodSeconds: 10, - }, - resources: { - requests: { - memory: convertToK8sFormat(server.memory || DEFAULT_SERVER_MEMORY), - cpu: "250m", - }, - limits: { - memory: convertToK8sFormat(server.memory || DEFAULT_SERVER_MEMORY), - cpu: "500m", - }, - }, - }, - ], - volumes: [ - { - name: "config", - configMap: { - name: `${serverName}-config`, - }, - }, - ], - }, - }, - }, - }; - - try { - await appsApi.createNamespacedDeployment({ namespace, body: deployment }); - logger.debug({ serverId: server.id, resource: "Deployment" }, "Created Deployment"); - } catch (err: any) { - if (err.code === 409) { - await appsApi.replaceNamespacedDeployment({ name: serverName, namespace, body: deployment }); - logger.debug({ serverId: server.id, resource: "Deployment" }, "Updated Deployment"); - } else { - throw err; - } - } -} - -async function createStatefulSet( - serverName: string, - server: ServerConfig, - appsApi: k8s.AppsV1Api, - namespace: string -): Promise { - const statefulSet = { - apiVersion: "apps/v1", - kind: "StatefulSet", - metadata: { - name: serverName, - namespace: namespace, - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: "stateful", - [`${LABEL_PREFIX}/server-id`]: server.id, - }, - }, - spec: { - serviceName: serverName, - replicas: 1, - selector: { - matchLabels: { - app: serverName, - }, - }, - template: { - metadata: { - labels: { - app: serverName, - [`${LABEL_PREFIX}/server-type`]: "stateful", - [`${LABEL_PREFIX}/server-id`]: server.id, - }, - }, - spec: { - containers: [ - { - name: "minecraft", - image: "itzg/minecraft-server", - ports: [ - { - containerPort: 25565, - name: "minecraft", - }, - ], - env: [ - { - name: "EULA", - value: "TRUE", - }, - { - name: "TYPE", - value: "VANILLA", - }, - { - name: "MEMORY", - value: calculateJavaMemory( - server.memory || DEFAULT_SERVER_MEMORY, - JAVA_MEMORY_FACTOR - ), - }, - { - name: "OPS", - value: "", - }, - { - name: "OVERRIDE_SERVER_PROPERTIES", - value: "true", - }, - { - name: "ENABLE_RCON", - value: "false", - }, - ...(server.env_variables || []).map((ev) => ({ - name: ev.key, - value: ev.value, - })), - ], - volumeMounts: [ - { - name: "data", - mountPath: "/data", - }, - { - name: "config", - mountPath: "/config", - }, - ], - readinessProbe: { - tcpSocket: { - port: 25565, - }, - initialDelaySeconds: 60, - periodSeconds: 10, - }, - resources: { - requests: { - memory: convertToK8sFormat(server.memory), - cpu: "250m", - }, - limits: { - memory: convertToK8sFormat(server.memory), - cpu: "500m", - }, - }, - }, - ], - volumes: [ - { - name: "config", - configMap: { - name: `${serverName}-config`, - }, - }, - ], - }, - }, - volumeClaimTemplates: [ - { - metadata: { - name: "data", - }, - spec: { - accessModes: ["ReadWriteOnce"], - resources: { - requests: { - storage: "1Gi", - }, - }, - }, - }, - ], - }, - }; - - try { - await appsApi.createNamespacedStatefulSet({ namespace, body: statefulSet }); - logger.debug({ serverId: server.id, resource: "StatefulSet" }, "Created StatefulSet"); - } catch (err: any) { - if (err.code === 409) { - await appsApi.replaceNamespacedStatefulSet({ - name: serverName, - namespace, - body: statefulSet, - }); - logger.debug({ serverId: server.id, resource: "StatefulSet" }, "Updated StatefulSet"); - } else { - throw err; - } - } -} - -export async function deleteServer( - serverId: string, - appsApi: k8s.AppsV1Api, - coreApi: k8s.CoreV1Api, - namespace: string -): Promise { - const serverName = `minecraft-${serverId}`; - - try { - await appsApi.deleteNamespacedDeployment({ name: serverName, namespace }); - logger.debug({ serverName, resource: "Deployment" }, "Deleted Deployment"); - } catch (err: any) { - if (err.response?.statusCode !== 404) { - logger.error({ err, serverName, resource: "Deployment" }, "Failed to delete Deployment"); - } - } - - try { - await appsApi.deleteNamespacedStatefulSet({ name: serverName, namespace }); - logger.debug({ serverName, resource: "StatefulSet" }, "Deleted StatefulSet"); - } catch (err: any) { - if (err.response?.statusCode !== 404) { - logger.error({ err, serverName, resource: "StatefulSet" }, "Failed to delete StatefulSet"); - } - } - - try { - await coreApi.deleteNamespacedService({ name: serverName, namespace }); - logger.debug({ serverName, resource: "Service" }, "Deleted Service"); - } catch (err: any) { - if (err.response?.statusCode !== 404) { - logger.error({ err, serverName, resource: "Service" }, "Failed to delete Service"); - } - } - - try { - await coreApi.deleteNamespacedConfigMap({ name: `${serverName}-config`, namespace }); - logger.debug({ serverName, resource: "ConfigMap" }, "Deleted ConfigMap"); - } catch (err: any) { - if (err.response?.statusCode !== 404) { - logger.error({ err, serverName, resource: "ConfigMap" }, "Failed to delete ConfigMap"); - } - } -} diff --git a/packages/k8s-operator/src/scripts/apply-crds.ts b/packages/k8s-operator/src/scripts/apply-crds.ts deleted file mode 100644 index e905ba4..0000000 --- a/packages/k8s-operator/src/scripts/apply-crds.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { PrismaClient } from "@minikura/db"; -import { dotenvLoad } from "dotenv-mono"; -import { NAMESPACE } from "../config/constants"; -import { setupCRDRegistration } from "../utils/crd-registrar"; -import { KubernetesClient } from "../utils/k8s-client"; -import { registerRBACResources } from "../utils/rbac-registrar"; - -dotenvLoad(); - -async function main() { - console.log("Starting to apply TypeScript-defined CRDs to Kubernetes cluster..."); - - try { - const k8sClient = KubernetesClient.getInstance(); - console.log(`Connected to Kubernetes cluster, using namespace: ${NAMESPACE}`); - - await registerRBACResources(k8sClient); - - console.log("Registering Custom Resource Definitions..."); - const prisma = new PrismaClient(); - await setupCRDRegistration(prisma, k8sClient, NAMESPACE); - - console.log("Successfully applied all resources to Kubernetes cluster"); - process.exit(0); - } catch (error: any) { - console.error("Failed to apply resources:", error.message); - if (error.stack) { - console.error(error.stack); - } - process.exit(1); - } -} - -main().catch((error) => { - console.error("Unhandled error:", error); - process.exit(1); -}); diff --git a/packages/k8s-operator/src/services/notification.service.ts b/packages/k8s-operator/src/services/notification.service.ts deleted file mode 100644 index 092202e..0000000 --- a/packages/k8s-operator/src/services/notification.service.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { createLogger } from "@minikura/shared"; -import pg from "pg"; - -const logger = createLogger("notification-service"); - -export class NotificationService { - private pgClient: pg.Client | null = null; - private handlers = new Map void | Promise>>(); - - async connect(connectionString: string): Promise { - if (!connectionString) { - throw new Error("Database connection string is required"); - } - - logger.info("Connecting to PostgreSQL"); - this.pgClient = new pg.Client({ connectionString }); - await this.pgClient.connect(); - - this.pgClient.on("notification", async (msg) => { - const handlers = this.handlers.get(msg.channel); - if (!handlers) return; - - try { - const payload = msg.payload ? JSON.parse(msg.payload) : {}; - logger.info({ channel: msg.channel, payload }, "Received notification"); - - for (const handler of handlers) { - try { - await handler(payload); - } catch (err) { - logger.error({ err, channel: msg.channel }, "Error in notification handler"); - } - } - } catch (err) { - logger.error({ err }, "Failed to parse notification payload"); - } - }); - - logger.info("Connected to PostgreSQL successfully"); - } - - async listen( - channel: string, - handler: (payload: unknown) => void | Promise - ): Promise { - if (!this.pgClient) { - throw new Error("NotificationService not connected"); - } - - if (!this.handlers.has(channel)) { - this.handlers.set(channel, new Set()); - await this.pgClient.query(`LISTEN ${channel}`); - logger.info({ channel }, "Listening on channel"); - } - - this.handlers.get(channel)?.add(handler); - } - - async unlisten(channel: string): Promise { - if (!this.pgClient) return; - - this.handlers.delete(channel); - await this.pgClient.query(`UNLISTEN ${channel}`); - logger.info({ channel }, "Stopped listening on channel"); - } - - async disconnect(): Promise { - if (!this.pgClient) return; - - logger.info("Disconnecting from PostgreSQL"); - await this.pgClient.end(); - this.pgClient = null; - this.handlers.clear(); - logger.info("Disconnected from PostgreSQL"); - } - - isConnected(): boolean { - return this.pgClient !== null; - } -} diff --git a/packages/k8s-operator/src/types/index.ts b/packages/k8s-operator/src/types/index.ts deleted file mode 100644 index 0507c69..0000000 --- a/packages/k8s-operator/src/types/index.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { - CustomEnvironmentVariable, - ReverseProxyServer as PrismaReverseProxyServer, - Server as PrismaServer, -} from "@minikura/db"; - -export interface CustomResource { - apiVersion: string; - kind: string; - metadata: { - name: string; - namespace?: string; - labels?: Record; - annotations?: Record; - [key: string]: any; - }; -} - -export type ServerConfig = Pick< - PrismaServer, - "id" | "description" | "type" | "listen_port" | "memory" | "service_type" -> & { - apiKey: string; - env_variables?: Array>; -}; - -export type MinecraftServerSpec = Pick< - PrismaServer, - "id" | "description" | "type" | "listen_port" | "memory" -> & { - environmentVariables?: Array>; -}; - -export interface MinecraftServerStatus { - phase: "Pending" | "Running" | "Failed"; - message?: string; - apiKey?: string; - internalId?: string; - lastSyncedAt?: string; -} - -export interface MinecraftServerCRD extends CustomResource { - spec: MinecraftServerSpec; - status?: MinecraftServerStatus; -} - -export type ReverseProxyConfig = Pick< - PrismaReverseProxyServer, - | "id" - | "description" - | "external_address" - | "external_port" - | "listen_port" - | "type" - | "memory" - | "service_type" -> & { - apiKey: string; - env_variables?: Array>; -}; - -export type ReverseProxyServerSpec = Partial< - Pick< - PrismaReverseProxyServer, - "id" | "description" | "external_address" | "external_port" | "listen_port" | "type" | "memory" - > -> & { - id: string; - external_address: string; - external_port: number; - environmentVariables?: Array>; -}; - -export interface ReverseProxyServerStatus { - phase: "Pending" | "Running" | "Failed"; - message?: string; - apiKey?: string; - internalId?: string; - lastSyncedAt?: string; -} - -export interface ReverseProxyServerCRD extends CustomResource { - spec: ReverseProxyServerSpec; - status?: ReverseProxyServerStatus; -} - -export type EnvironmentVariable = Pick; diff --git a/packages/k8s-operator/src/types/k8s-types.ts b/packages/k8s-operator/src/types/k8s-types.ts deleted file mode 100644 index 1bbe9e6..0000000 --- a/packages/k8s-operator/src/types/k8s-types.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type * as k8s from "@kubernetes/client-node"; - -export interface K8sApiError extends Error { - code?: number; - body?: string; - headers?: Record; -} - -export interface CustomResourceResponse { - metadata?: k8s.V1ObjectMeta; - spec?: T; - status?: Record; - body?: CustomResourceResponse; -} - -export interface CustomResourceListResponse { - items?: CustomResourceResponse[]; - body?: { - items?: CustomResourceResponse[]; - }; -} - -export function isK8sApiError(error: unknown): error is K8sApiError { - return error instanceof Error && ("code" in error || "body" in error || "headers" in error); -} diff --git a/packages/k8s-operator/src/utils/crd-registrar.ts b/packages/k8s-operator/src/utils/crd-registrar.ts deleted file mode 100644 index 46171df..0000000 --- a/packages/k8s-operator/src/utils/crd-registrar.ts +++ /dev/null @@ -1,633 +0,0 @@ -import type * as k8s from "@kubernetes/client-node"; -import type { PrismaClient } from "@minikura/db"; -import { API_GROUP, API_VERSION, LABEL_PREFIX } from "../config/constants"; -import { REVERSE_PROXY_SERVER_CRD } from "../crds/reverseProxy"; -import { MINECRAFT_SERVER_CRD } from "../crds/server"; -import type { CustomResourceListResponse, CustomResourceResponse } from "../types/k8s-types"; -import { isK8sApiError } from "../types/k8s-types"; -import type { KubernetesClient } from "./k8s-client"; -import { logger } from "./logger"; - -async function retryWithBackoff( - operation: () => Promise, - options: { - maxRetries?: number; - initialDelay?: number; - maxDelay?: number; - operationName?: string; - } = {} -): Promise { - const { - maxRetries = 5, - initialDelay = 1000, - maxDelay = 10000, - operationName = "operation", - } = options; - - let lastError: Error | undefined; - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - return await operation(); - } catch (error: unknown) { - lastError = error as Error; - - const is429 = isK8sApiError(error) && error.code === 429; - const isStorageInitializing = - isK8sApiError(error) && error.body?.includes("storage is (re)initializing"); - - if (!is429 && !isStorageInitializing) { - throw error; - } - - if (attempt === maxRetries) { - logger.error({ operationName, maxRetries }, "Operation failed after max retries"); - throw error; - } - - let delay = initialDelay * 2 ** attempt; - if (isK8sApiError(error) && error.headers?.["retry-after"]) { - const retryAfter = parseInt(error.headers["retry-after"], 10); - if (!Number.isNaN(retryAfter)) { - delay = retryAfter * 1000; - } - } - delay = Math.min(delay, maxDelay); - - const errorMessage = error instanceof Error ? error.message : String(error); - logger.warn( - { - operationName, - attempt: attempt + 1, - maxAttempts: maxRetries + 1, - delayMs: delay, - errorMessage, - }, - "Operation failed, retrying with backoff" - ); - - await new Promise((resolve) => setTimeout(resolve, delay)); - } - } - - throw lastError; -} - -export async function setupCRDRegistration( - prisma: PrismaClient, - k8sClient: KubernetesClient, - namespace: string -): Promise { - await registerCRDs(k8sClient); - - logger.info("Waiting for Kubernetes storage to stabilize after CRD registration"); - await new Promise((resolve) => setTimeout(resolve, 2000)); - - await startCRDReflector(prisma, k8sClient, namespace); -} - -async function registerCRDs(k8sClient: KubernetesClient): Promise { - try { - const apiExtensionsClient = k8sClient.getApiExtensionsApi(); - - logger.info( - { apiGroup: API_GROUP, apiVersion: API_VERSION }, - "Registering Custom Resource Definitions" - ); - - try { - await apiExtensionsClient.createCustomResourceDefinition({ body: MINECRAFT_SERVER_CRD }); - logger.info( - { crd: "MinecraftServer", apiGroup: API_GROUP, apiVersion: API_VERSION }, - "CRD created successfully" - ); - } catch (error: any) { - if (error.code === 409) { - logger.debug("MinecraftServer CRD already exists, skipping creation"); - } else { - logger.error({ err: error }, "Failed to create MinecraftServer CRD"); - throw error; - } - } - - try { - await apiExtensionsClient.createCustomResourceDefinition({ body: REVERSE_PROXY_SERVER_CRD }); - logger.info( - { crd: "ReverseProxyServer", apiGroup: API_GROUP, apiVersion: API_VERSION }, - "CRD created successfully" - ); - } catch (error: any) { - if (error.code === 409) { - logger.debug("ReverseProxyServer CRD already exists, skipping creation"); - } else { - logger.error({ err: error }, "Failed to create ReverseProxyServer CRD"); - throw error; - } - } - } catch (error) { - logger.error({ err: error }, "Failed to register CRDs"); - throw error; - } -} - -async function startCRDReflector( - prisma: PrismaClient, - k8sClient: KubernetesClient, - namespace: string -): Promise { - const customObjectsApi = k8sClient.getCustomObjectsApi(); - - const reflectedMinecraftServers = new Map(); - const reflectedReverseProxyServers = new Map(); - - logger.info("Starting CRD reflector to sync database state to custom resources"); - - await syncDBtoCRDs( - prisma, - customObjectsApi, - namespace, - reflectedMinecraftServers, - reflectedReverseProxyServers - ); - - setInterval(async () => { - await syncDBtoCRDs( - prisma, - customObjectsApi, - namespace, - reflectedMinecraftServers, - reflectedReverseProxyServers - ); - }, 30 * 1000); -} - -async function syncDBtoCRDs( - prisma: PrismaClient, - customObjectsApi: k8s.CustomObjectsApi, - namespace: string, - reflectedMinecraftServers: Map, - reflectedReverseProxyServers: Map -): Promise { - try { - logger.debug("Starting CRD sync operation"); - await syncMinecraftServers(prisma, customObjectsApi, namespace, reflectedMinecraftServers); - await syncReverseProxyServers( - prisma, - customObjectsApi, - namespace, - reflectedReverseProxyServers - ); - logger.debug("CRD sync operation completed successfully"); - } catch (error) { - logger.error({ err: error }, "Failed to sync database to CRDs"); - } -} - -async function syncMinecraftServers( - prisma: PrismaClient, - customObjectsApi: k8s.CustomObjectsApi, - namespace: string, - reflectedMinecraftServers: Map -): Promise { - try { - const servers = await prisma.server.findMany(); - - let existingCRs: any[] = []; - try { - const response = await retryWithBackoff( - () => - customObjectsApi.listNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "minecraftservers", - }), - { - maxRetries: 5, - initialDelay: 1000, - operationName: "List MinecraftServer CRs", - } - ); - const listResponse = response as unknown as CustomResourceListResponse; - existingCRs = listResponse.body?.items || listResponse.items || []; - } catch (error) { - logger.error( - { err: error }, - "Failed to list MinecraftServer custom resources, assuming none exist" - ); - existingCRs = []; - } - - const existingCRMap = new Map(); - const crResourceVersions = new Map(); - - for (const cr of existingCRs) { - const internalId = cr.status?.internalId; - if (internalId) { - existingCRMap.set(internalId, cr.metadata.name); - if (cr.metadata?.resourceVersion) { - crResourceVersions.set(cr.metadata.name, cr.metadata.resourceVersion); - } - } - } - - reflectedMinecraftServers.clear(); - - for (const server of servers) { - const crName = existingCRMap.get(server.id) || `${server.id.toLowerCase()}`; - - const serverCR: { - apiVersion: string; - kind: string; - metadata: { - name: string; - namespace: string; - annotations: Record; - resourceVersion?: string; - }; - spec: any; - status: any; - } = { - apiVersion: `${API_GROUP}/${API_VERSION}`, - kind: "MinecraftServer", - metadata: { - name: crName, - namespace: namespace, - annotations: { - [`${LABEL_PREFIX}/database-managed`]: "true", - [`${LABEL_PREFIX}/last-synced`]: new Date().toISOString(), - }, - }, - spec: { - id: server.id, - description: server.description, - listen_port: server.listen_port, - type: server.type, - memory: `${server.memory}M`, - }, - status: { - phase: "Running", - message: "Managed by database", - internalId: server.id, - apiKey: "[REDACTED]", - lastSyncedAt: new Date().toISOString(), - }, - }; - - try { - const existingCRName = existingCRMap.get(server.id); - if (existingCRName) { - try { - const existingResource = await customObjectsApi.getNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "minecraftservers", - name: existingCRName, - }); - - const resourceResponse = existingResource as CustomResourceResponse; - const resource = resourceResponse.body || resourceResponse; - if (resource?.metadata?.resourceVersion) { - serverCR.metadata.resourceVersion = resource.metadata.resourceVersion; - } - - await customObjectsApi.replaceNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "minecraftservers", - name: existingCRName, - body: serverCR, - }); - logger.debug( - { crName: existingCRName, serverId: server.id }, - "Updated MinecraftServer custom resource" - ); - } catch (error) { - logger.error( - { err: error, serverId: server.id }, - "Failed to get/update MinecraftServer custom resource" - ); - } - } else { - try { - await customObjectsApi.createNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "minecraftservers", - body: serverCR, - }); - logger.debug( - { crName, serverId: server.id }, - "Created MinecraftServer custom resource" - ); - } catch (createError: any) { - if (createError.code === 409) { - logger.debug({ crName }, "MinecraftServer CR already exists, updating instead"); - try { - const existingResource = await customObjectsApi.getNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "minecraftservers", - name: crName, - }); - - const resourceResponse = existingResource as CustomResourceResponse; - let resource = resourceResponse.body || resourceResponse; - if (!resource?.metadata && resourceResponse.metadata) { - resource = resourceResponse; - } - - if (resource?.metadata?.resourceVersion) { - serverCR.metadata.resourceVersion = resource.metadata.resourceVersion; - - await customObjectsApi.replaceNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "minecraftservers", - name: crName, - body: serverCR, - }); - logger.debug( - { crName, serverId: server.id }, - "Updated existing MinecraftServer custom resource" - ); - } else { - logger.error({ crName }, "Cannot update CR: no resourceVersion in response"); - } - } catch (updateError) { - logger.error( - { err: updateError, crName }, - "Failed to update MinecraftServer custom resource" - ); - throw updateError; - } - } else { - throw createError; - } - } - } - - reflectedMinecraftServers.set(server.id, crName); - } catch (error) { - logger.error( - { err: error, serverId: server.id }, - "Failed to create/update MinecraftServer custom resource" - ); - } - } - - for (const [dbId, crName] of existingCRMap.entries()) { - if (!servers.some((s) => s.id === dbId)) { - try { - await customObjectsApi.deleteNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "minecraftservers", - name: crName, - }); - logger.info( - { crName, serverId: dbId }, - "Deleted MinecraftServer CR for removed database record" - ); - } catch (error) { - logger.error({ err: error, crName }, "Failed to delete MinecraftServer custom resource"); - } - } - } - } catch (error) { - logger.error({ err: error }, "Failed to sync Minecraft servers to custom resources"); - } -} - -async function syncReverseProxyServers( - prisma: PrismaClient, - customObjectsApi: any, - namespace: string, - reflectedReverseProxyServers: Map -): Promise { - try { - const proxies = await prisma.reverseProxyServer.findMany({ - include: { - env_variables: true, - }, - }); - - let existingCRs: any[] = []; - try { - const response = await retryWithBackoff( - () => - customObjectsApi.listNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "reverseproxyservers", - }), - { - maxRetries: 5, - initialDelay: 1000, - operationName: "List ReverseProxyServer CRs", - } - ); - const listResponse = response as unknown as CustomResourceListResponse; - existingCRs = listResponse.body?.items || listResponse.items || []; - } catch (error) { - logger.error( - { err: error }, - "Failed to list ReverseProxyServer custom resources, assuming none exist" - ); - existingCRs = []; - } - - const existingCRMap = new Map(); - const crResourceVersions = new Map(); - - for (const cr of existingCRs) { - const internalId = cr.status?.internalId; - if (internalId) { - existingCRMap.set(internalId, cr.metadata.name); - if (cr.metadata?.resourceVersion) { - crResourceVersions.set(cr.metadata.name, cr.metadata.resourceVersion); - } - } - } - - reflectedReverseProxyServers.clear(); - - for (const proxy of proxies) { - const crName = existingCRMap.get(proxy.id) || `${proxy.id.toLowerCase()}`; - - const proxyCR: { - apiVersion: string; - kind: string; - metadata: { - name: string; - namespace: string; - annotations: Record; - resourceVersion?: string; - }; - spec: any; - status: any; - } = { - apiVersion: `${API_GROUP}/${API_VERSION}`, - kind: "ReverseProxyServer", - metadata: { - name: crName, - namespace: namespace, - annotations: { - [`${LABEL_PREFIX}/database-managed`]: "true", - [`${LABEL_PREFIX}/last-synced`]: new Date().toISOString(), - }, - }, - spec: { - id: proxy.id, - description: proxy.description, - external_address: proxy.external_address, - external_port: proxy.external_port, - listen_port: proxy.listen_port, - type: proxy.type, - memory: `${proxy.memory}M`, - environmentVariables: proxy.env_variables?.map((ev) => ({ - key: ev.key, - value: ev.value, - })), - }, - status: { - phase: "Running", - message: "Managed by database", - internalId: proxy.id, - apiKey: "[REDACTED]", - lastSyncedAt: new Date().toISOString(), - }, - }; - - try { - const existingCRName = existingCRMap.get(proxy.id); - if (existingCRName) { - try { - const existingResource = await customObjectsApi.getNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "reverseproxyservers", - name: existingCRName, - }); - - const resourceResponse = existingResource as CustomResourceResponse; - const resource = resourceResponse.body || resourceResponse; - if (resource?.metadata?.resourceVersion) { - proxyCR.metadata.resourceVersion = resource.metadata.resourceVersion; - } - - await customObjectsApi.replaceNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "reverseproxyservers", - name: existingCRName, - body: proxyCR, - }); - logger.debug( - { crName: existingCRName, proxyId: proxy.id }, - "Updated ReverseProxyServer custom resource" - ); - } catch (error) { - logger.error( - { err: error, proxyId: proxy.id }, - "Failed to get/update ReverseProxyServer custom resource" - ); - } - } else { - try { - await customObjectsApi.createNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "reverseproxyservers", - body: proxyCR, - }); - logger.debug( - { crName, proxyId: proxy.id }, - "Created ReverseProxyServer custom resource" - ); - } catch (createError: any) { - if (createError.code === 409) { - logger.debug({ crName }, "ReverseProxyServer CR already exists, updating instead"); - try { - const existingResource = await customObjectsApi.getNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "reverseproxyservers", - name: crName, - }); - - const resource = existingResource.body as any; - if (resource?.metadata?.resourceVersion) { - proxyCR.metadata.resourceVersion = resource.metadata.resourceVersion; - } - - await customObjectsApi.replaceNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "reverseproxyservers", - name: crName, - body: proxyCR, - }); - logger.debug( - { crName, proxyId: proxy.id }, - "Updated existing ReverseProxyServer custom resource" - ); - } catch (updateError) { - logger.error( - { err: updateError, crName }, - "Failed to update ReverseProxyServer custom resource" - ); - throw updateError; - } - } else { - throw createError; - } - } - } - - reflectedReverseProxyServers.set(proxy.id, crName); - } catch (error) { - logger.error( - { err: error, proxyId: proxy.id }, - "Failed to create/update ReverseProxyServer custom resource" - ); - } - } - - for (const [dbId, crName] of existingCRMap.entries()) { - if (!proxies.some((p) => p.id === dbId)) { - try { - await customObjectsApi.deleteNamespacedCustomObject({ - group: API_GROUP, - version: API_VERSION, - namespace, - plural: "reverseproxyservers", - name: crName, - }); - logger.info( - { crName, proxyId: dbId }, - "Deleted ReverseProxyServer CR for removed database record" - ); - } catch (error) { - logger.error( - { err: error, crName }, - "Failed to delete ReverseProxyServer custom resource" - ); - } - } - } - } catch (error) { - logger.error({ err: error }, "Failed to sync reverse proxy servers to custom resources"); - } -} diff --git a/packages/k8s-operator/src/utils/k8s-client.ts b/packages/k8s-operator/src/utils/k8s-client.ts deleted file mode 100644 index 22e7493..0000000 --- a/packages/k8s-operator/src/utils/k8s-client.ts +++ /dev/null @@ -1,71 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import { buildKubeConfig } from "@minikura/shared/kube-auth"; -import { logger } from "./logger"; - -export class KubernetesClient { - private static instance: KubernetesClient; - private kc: k8s.KubeConfig; - private appsApi!: k8s.AppsV1Api; - private coreApi!: k8s.CoreV1Api; - private networkingApi!: k8s.NetworkingV1Api; - private customObjectsApi!: k8s.CustomObjectsApi; - private apiExtensionsApi!: k8s.ApiextensionsV1Api; - - private constructor() { - this.kc = buildKubeConfig(); - this.initializeClients(); - } - - static getInstance(): KubernetesClient { - if (!KubernetesClient.instance) { - KubernetesClient.instance = new KubernetesClient(); - } - return KubernetesClient.instance; - } - - private initializeClients(): void { - this.appsApi = this.kc.makeApiClient(k8s.AppsV1Api); - this.coreApi = this.kc.makeApiClient(k8s.CoreV1Api); - this.networkingApi = this.kc.makeApiClient(k8s.NetworkingV1Api); - this.customObjectsApi = this.kc.makeApiClient(k8s.CustomObjectsApi); - this.apiExtensionsApi = this.kc.makeApiClient(k8s.ApiextensionsV1Api); - } - - getKubeConfig(): k8s.KubeConfig { - return this.kc; - } - - getAppsApi(): k8s.AppsV1Api { - return this.appsApi; - } - - getCoreApi(): k8s.CoreV1Api { - return this.coreApi; - } - - getNetworkingApi(): k8s.NetworkingV1Api { - return this.networkingApi; - } - - getCustomObjectsApi(): k8s.CustomObjectsApi { - return this.customObjectsApi; - } - - getApiExtensionsApi(): k8s.ApiextensionsV1Api { - return this.apiExtensionsApi; - } - - async handleApiError(error: any, context: string): Promise { - logger.error( - { - context, - message: error?.message, - statusCode: error?.response?.statusCode, - body: error?.response?.body, - }, - "Kubernetes API error" - ); - - throw error; - } -} diff --git a/packages/k8s-operator/src/utils/logger.ts b/packages/k8s-operator/src/utils/logger.ts deleted file mode 100644 index 5242a08..0000000 --- a/packages/k8s-operator/src/utils/logger.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { createLogger } from "@minikura/shared"; - -export { createLogger }; - -export const logger = createLogger("k8s-operator"); diff --git a/packages/k8s-operator/src/utils/memory.ts b/packages/k8s-operator/src/utils/memory.ts deleted file mode 100644 index d1a2fe1..0000000 --- a/packages/k8s-operator/src/utils/memory.ts +++ /dev/null @@ -1,32 +0,0 @@ -export function calculateJavaMemory(memory: number | string, factor: number): string { - if (typeof memory === "number") { - const calculatedValue = Math.round(memory * factor); - return `${calculatedValue}M`; - } - - const match = memory.match(/^(\d+)([MG])$/i); - if (!match) return "512M"; - - const [, valueStr, unit] = match; - const value = parseInt(valueStr, 10); - - const calculatedValue = Math.round(value * factor); - return `${calculatedValue}${unit.toUpperCase()}`; -} - -export function convertToK8sFormat(memory: number | string): string { - if (typeof memory === "number") { - return `${memory}Mi`; - } - - const match = memory.match(/^(\d+)([MG])$/i); - if (!match) return "1Gi"; - - const [, valueStr, unit] = match; - - if (unit.toUpperCase() === "G") { - return `${valueStr}Gi`; - } else { - return `${valueStr}Mi`; - } -} diff --git a/packages/k8s-operator/src/utils/rbac-registrar.ts b/packages/k8s-operator/src/utils/rbac-registrar.ts deleted file mode 100644 index ba6e01e..0000000 --- a/packages/k8s-operator/src/utils/rbac-registrar.ts +++ /dev/null @@ -1,203 +0,0 @@ -import fetch from "node-fetch"; -import { - minikuraClusterRole, - minikuraClusterRoleBinding, - minikuraNamespace, - minikuraOperatorDeployment, - minikuraServiceAccount, -} from "../crds/rbac"; -import type { KubernetesClient } from "./k8s-client"; -import { logger } from "./logger"; - -export async function registerRBACResources(k8sClient: KubernetesClient): Promise { - try { - logger.info("Starting RBAC resources registration"); - - await registerNamespace(k8sClient); - await registerServiceAccount(k8sClient); - await registerClusterRole(k8sClient); - await registerClusterRoleBinding(k8sClient); - - logger.info("RBAC resources registration completed successfully"); - } catch (error: any) { - logger.error( - { - err: error, - message: error.message, - statusCode: error.response?.statusCode, - body: error.response?.body, - }, - "Error registering RBAC resources" - ); - if (error.response) { - } - throw error; - } -} - -async function registerNamespace(k8sClient: KubernetesClient): Promise { - try { - const coreApi = k8sClient.getCoreApi(); - await coreApi.createNamespace({ body: minikuraNamespace }); - logger.info({ namespace: minikuraNamespace.metadata.name }, "Created namespace"); - } catch (error: any) { - if (error.code === 409) { - logger.debug({ namespace: minikuraNamespace.metadata.name }, "Namespace already exists"); - } else { - throw error; - } - } -} - -async function registerServiceAccount(k8sClient: KubernetesClient): Promise { - try { - const coreApi = k8sClient.getCoreApi(); - await coreApi.createNamespacedServiceAccount({ - namespace: minikuraServiceAccount.metadata.namespace, - body: minikuraServiceAccount, - }); - logger.info( - { serviceAccount: minikuraServiceAccount.metadata.name }, - "Created service account" - ); - } catch (error: any) { - if (error.code === 409) { - logger.debug(`Service account ${minikuraServiceAccount.metadata.name} already exists`); - } else { - throw error; - } - } -} - -async function registerClusterRole(k8sClient: KubernetesClient): Promise { - try { - const kc = k8sClient.getKubeConfig(); - const opts: any = {}; - await kc.applyToHTTPSOptions(opts); - - const cluster = kc.getCurrentCluster(); - if (!cluster) { - throw new Error("No active cluster found in KubeConfig"); - } - - try { - const response = await fetch( - `${cluster.server}/apis/rbac.authorization.k8s.io/v1/clusterroles`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - ...(opts as any).headers, - }, - body: JSON.stringify(minikuraClusterRole), - agent: (opts as any).agent, - } - ); - - if (response.ok) { - logger.debug(`Created cluster role ${minikuraClusterRole.metadata.name}`); - } else if (response.status === 409) { - logger.debug(`Cluster role ${minikuraClusterRole.metadata.name} already exists`); - } else { - const text = await response.text(); - throw new Error( - `Failed to create cluster role: ${response.status} ${response.statusText} - ${text}` - ); - } - } catch (error: any) { - if (error.message?.includes("already exists") || error.message?.includes("409")) { - logger.debug(`Cluster role ${minikuraClusterRole.metadata.name} already exists`); - } else { - throw error; - } - } - } catch (error: any) { - logger.error(`Error registering cluster role:`, error.message); - throw error; - } -} - -async function registerClusterRoleBinding(k8sClient: KubernetesClient): Promise { - try { - const kc = k8sClient.getKubeConfig(); - const opts: any = {}; - await kc.applyToHTTPSOptions(opts); - - const cluster = kc.getCurrentCluster(); - if (!cluster) { - throw new Error("No active cluster found in KubeConfig"); - } - - const { default: fetch } = await import("node-fetch"); - - try { - const response = await fetch( - `${cluster.server}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - ...(opts as any).headers, - }, - body: JSON.stringify(minikuraClusterRoleBinding), - agent: (opts as any).agent, - } - ); - - if (response.ok) { - logger.debug(`Created cluster role binding ${minikuraClusterRoleBinding.metadata.name}`); - } else if (response.status === 409) { - logger.debug( - `Cluster role binding ${minikuraClusterRoleBinding.metadata.name} already exists` - ); - } else { - const text = await response.text(); - throw new Error( - `Failed to create cluster role binding: ${response.status} ${response.statusText} - ${text}` - ); - } - } catch (error: any) { - if (error.message?.includes("already exists") || error.message?.includes("409")) { - logger.debug( - `Cluster role binding ${minikuraClusterRoleBinding.metadata.name} already exists` - ); - } else { - throw error; - } - } - } catch (error: any) { - logger.error(`Error registering cluster role binding:`, error.message); - throw error; - } -} - -export async function registerOperatorDeployment( - k8sClient: KubernetesClient, - registryUrl: string -): Promise { - try { - const deployment = JSON.parse( - JSON.stringify(minikuraOperatorDeployment).replace("${REGISTRY_URL}", registryUrl) - ); - - const appsApi = k8sClient.getAppsApi(); - await appsApi.createNamespacedDeployment(deployment.metadata.namespace, deployment); - logger.debug(`Created deployment ${deployment.metadata.name}`); - } catch (error: any) { - if (error.code === 409) { - logger.debug(`Deployment ${minikuraOperatorDeployment.metadata.name} already exists`); - const deployment = JSON.parse( - JSON.stringify(minikuraOperatorDeployment).replace("${REGISTRY_URL}", registryUrl) - ); - - await k8sClient.getAppsApi().replaceNamespacedDeployment({ - name: deployment.metadata.name, - namespace: deployment.metadata.namespace, - body: deployment, - }); - logger.debug(`Updated deployment ${deployment.metadata.name}`); - } else { - throw error; - } - } -} diff --git a/packages/k8s-operator/src/utils/service-type.ts b/packages/k8s-operator/src/utils/service-type.ts deleted file mode 100644 index 905e4cb..0000000 --- a/packages/k8s-operator/src/utils/service-type.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { ServiceType } from "@minikura/db"; - -export function mapServiceType( - serviceType?: ServiceType | null, - defaultType: string = "ClusterIP" -): string { - if (!serviceType) return defaultType; - - switch (serviceType) { - case "CLUSTER_IP": - return "ClusterIP"; - case "NODE_PORT": - return "NodePort"; - case "LOAD_BALANCER": - return "LoadBalancer"; - default: - return defaultType; - } -} diff --git a/packages/k8s-operator/tsconfig.json b/packages/k8s-operator/tsconfig.json deleted file mode 100644 index fd521c7..0000000 --- a/packages/k8s-operator/tsconfig.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "compilerOptions": { - // Enable latest features - "lib": ["ESNext"], - "target": "ESNext", - "module": "ESNext", - "moduleDetection": "force", - "allowJs": true, - - // Bundler mode - "moduleResolution": "bundler", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "noEmit": true, - - // Best practices - "strict": true, - "skipLibCheck": true, - "noFallthroughCasesInSwitch": true, - - // Some stricter flags (disabled by default) - "noUnusedLocals": false, - "noUnusedParameters": false, - "noPropertyAccessFromIndexSignature": false - } -} diff --git a/scripts/install.sh b/scripts/install.sh index b9ad722..98809d9 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -35,15 +35,21 @@ echo "-> Creating namespace: $NAMESPACE" kubectl create namespace $NAMESPACE --dry-run=client -o yaml | kubectl apply -f - echo "" -# Apply RBAC (single SA for both backend and operator) -echo "-> Setting up RBAC (minikura-operator ServiceAccount)" -kubectl apply -f "$PROJECT_ROOT/k8s/rbac/operator-rbac.yaml" -echo "[OK] RBAC configured" +# Apply CRDs and operator RBAC +echo "-> Installing operator CRDs" +make -C "$PROJECT_ROOT/operator" install-crds +echo "[OK] CRDs installed" echo "" -# CRD info -echo "-> Custom Resource Definitions" -echo " CRDs are auto-created when the operator starts (ENABLE_CRD_REFLECTION=true)" +echo "-> Setting up Go operator RBAC" +kubectl apply -f "$PROJECT_ROOT/operator/config/rbac/role.yaml" +kubectl apply -n "$NAMESPACE" -f "$PROJECT_ROOT/operator/config/rbac/service_account.yaml" +kubectl apply -n "$NAMESPACE" -f "$PROJECT_ROOT/operator/config/rbac/backend.yaml" +kubectl patch clusterrolebinding minikura-operator-rolebinding --type=json \ + -p="[{\"op\":\"replace\",\"path\":\"/subjects/0/namespace\",\"value\":\"$NAMESPACE\"}]" +kubectl patch clusterrolebinding minikura-backend-operator-resources --type=json \ + -p="[{\"op\":\"replace\",\"path\":\"/subjects/0/namespace\",\"value\":\"$NAMESPACE\"}]" +echo "[OK] Operator RBAC configured" echo "" echo "╔════════════════════════════════════════════════╗" @@ -57,5 +63,5 @@ echo " [OK] ClusterRole + ClusterRoleBinding" echo "" echo "Next steps:" echo " bun run dev - Start backend + web" -echo " bun run k8s:dev - Start K8s operator" +echo " bun run operator:dev - Start Go operator" echo ""