[FEATURE] Migrated to new loading sequence (#6)

Co-authored-by: Dunemask <dunemask@gmail.com> Reviewed-on: https://gitea.dunemask.dev/elysium/minecluster/pulls/6
2024-01-15 20:30:31 +00:00 · 2024-01-15 20:30:31 +00:00 · 6eb4ed3e95
commit 6eb4ed3e95
parent fb57c03ba7
53 changed files with 1349 additions and 449 deletions
--- a/lib/k8s/configs/backup-secret.yml
+++ b/lib/k8s/configs/backup-secret.yml
@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  annotations:
+    minecluster.dunemask.net/id: changeme-server-id
+  labels:
+    app: changeme-app-label
+  name: changeme-backup-secret
+type: Opaque
+data:
+  rclone.conf: ""
--- a/lib/k8s/configs/containers/ftp-server.yml
+++ b/lib/k8s/configs/containers/ftp-server.yml
@ -6,23 +6,21 @@ env:
 image: garethflowers/ftp-server
 imagePullPolicy: IfNotPresent
 livenessProbe:
-exec:
-  command: ["echo"]
-failureThreshold: 20
-initialDelaySeconds: 30
-periodSeconds: 5
-successThreshold: 1
-timeoutSeconds: 1
+  exec: { command: ["echo"] }
+  failureThreshold: 20
+  initialDelaySeconds: 0
+  periodSeconds: 5
+  successThreshold: 1
+  timeoutSeconds: 1
 name: changeme-name-ftp
 ports: [] # Programatically add all the ports for easier readability, Ports include: 20,21,40000-400009
 readinessProbe:
-  exec:
-    command: ["echo"]
-    failureThreshold: 20
-    initialDelaySeconds: 30
-    periodSeconds: 5
-    successThreshold: 1
-    timeoutSeconds: 1
+  exec: { command: ["echo"] }
+  failureThreshold: 20
+  initialDelaySeconds: 0
+  periodSeconds: 5
+  successThreshold: 1
+  timeoutSeconds: 1
 resources:
  requests:
    cpu: 50m
--- a/lib/k8s/configs/containers/minecraft-backup.yml
+++ b/lib/k8s/configs/containers/minecraft-backup.yml
@ -33,20 +33,20 @@ env:
  - name: DEST_DIR
    value: /backups
  - name: LINK_LATEST
-    value: "false"
+    value: "true"
  - name: TAR_COMPRESS_METHOD
    value: gzip
  - name: ZSTD_PARAMETERS
    value: -3 --long=25 --single-thread
  - name: RCLONE_REMOTE
-    value: mc-dunemask-net
+    value: mcl-backup-changeme
  - name: RCLONE_DEST_DIR
-    value: /minecraft-backups/deltasmp-backups
+    value: /mcl/backups/changeme
  - name: RCLONE_COMPRESS_METHOD
    value: gzip
 image: itzg/mc-backup:latest
 imagePullPolicy: IfNotPresent
-name: mcs-deltasmp-minecraft-mc-backup
+name: mcl-backup-changeme
 resources:
  requests:
    cpu: 500m
--- a/lib/k8s/configs/containers/minecraft-server.yml
+++ b/lib/k8s/configs/containers/minecraft-server.yml
@ -72,12 +72,10 @@ env:
 image: itzg/minecraft-server:latest
 imagePullPolicy: IfNotPresent
 livenessProbe:
-  exec:
-    command:
-      - mc-health
-  failureThreshold: 20
+  exec: { command: [mc-health] }
+  failureThreshold: 200
  initialDelaySeconds: 30
-  periodSeconds: 5
+  periodSeconds: 3
  successThreshold: 1
  timeoutSeconds: 1
 name: changeme-name-server
@ -88,15 +86,13 @@ ports:
  - containerPort: 25575
    name: rcon
    protocol: TCP
-readinessProbe:
-  exec:
-    command:
-      - mc-health
-  failureThreshold: 20
-  initialDelaySeconds: 30
-  periodSeconds: 5
-  successThreshold: 1
-  timeoutSeconds: 1
+# readinessProbe: # Disabling this allows for users to manipulate files even if the container is starting
+#   exec: {command: [mc-health]}
+#   failureThreshold: 200
+#   initialDelaySeconds: 30
+#   periodSeconds: 3
+#   successThreshold: 1
+#   timeoutSeconds: 1
 resources:
  requests:
    cpu: 500m
--- a/lib/k8s/configs/rcon-secret.yml
+++ b/lib/k8s/configs/rcon-secret.yml
@ -4,7 +4,7 @@ data:
 kind: Secret
 metadata:
  annotations:
-    minecluster.dunemask.net/server-name: changeme-server-name
+    minecluster.dunemask.net/id: changeme-server-id
  labels:
    app: changeme-app-label
  name: changeme-rcon-secret
--- a/lib/k8s/configs/rcon-svc.yml
+++ b/lib/k8s/configs/rcon-svc.yml
@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
  annotations:
-    minecluster.dunemask.net/server-name: changeme-server-name
+    minecluster.dunemask.net/id: changeme-server-id
  labels:
    app: changeme-app
  name: changeme-rcon
--- a/lib/k8s/configs/server-deployment.yml
+++ b/lib/k8s/configs/server-deployment.yml
@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
  annotations:
-    minecluster.dunemask.net/server-name: changeme-server-name
+    minecluster.dunemask.net/id: changeme-server-id
  name: changeme-name
  namespace: changeme-namespace
 spec:
@ -17,7 +17,7 @@ spec:
  template:
    metadata:
      annotations:
-        minecluster.dunemask.net/server-name: changeme-server-name
+        minecluster.dunemask.net/id: changeme-server-id
      labels:
        app: changeme-app
    spec:
@ -35,10 +35,10 @@ spec:
            claimName: changeme-pvc-name
        - emptyDir: {}
          name: backupdir
-        - name: rclone-config
-          secret:
-            defaultMode: 420
-            items:
-              - key: rclone.conf
-                path: rclone.conf
-            secretName: rclone-config
+        # - name: rclone-config
+        #   secret:
+        #     defaultMode: 420
+        #     items:
+        #       - key: rclone.conf
+        #         path: rclone.conf
+        #     secretName: rclone-config
--- a/lib/k8s/configs/server-pvc.yml
+++ b/lib/k8s/configs/server-pvc.yml
@ -2,7 +2,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  annotations:
-    minecluster.dunemask.net/server-name: changeme-server-name
+    minecluster.dunemask.net/id: changeme-server-id
  labels:
    service: changeme-service-name
  name: changeme-pvc-name
--- a/lib/k8s/configs/server-svc.yml
+++ b/lib/k8s/configs/server-svc.yml
@ -4,7 +4,7 @@ metadata:
  annotations:
    ingress.qumine.io/hostname: changeme-url
    ingress.qumine.io/portname: minecraft
-    minecluster.dunemask.net/server-name: changeme-server-name
+    minecluster.dunemask.net/id: changeme-server-id
  labels:
    app: changeme-app
  name: changeme-name
--- a/lib/k8s/k8s-server-control.js
+++ b/lib/k8s/k8s-server-control.js
@ -20,10 +20,10 @@ const loadYaml = (f) => yaml.load(fs.readFileSync(path.resolve(f), "utf8"));
 const mineclusterManaged = (o) =>
  o.metadata &&
  o.metadata.annotations &&
-  o.metadata.annotations["minecluster.dunemask.net/server-name"] !== undefined;
+  o.metadata.annotations["minecluster.dunemask.net/id"] !== undefined;

-export const serverMatch = (serverName) => (o) =>
-  o.metadata.annotations["minecluster.dunemask.net/server-name"] === serverName;
+export const serverMatch = (serverId) => (o) =>
+  o.metadata.annotations["minecluster.dunemask.net/id"] === serverId;

 export async function getDeployments() {
  const deploymentRes = await k8sDeps.listNamespacedDeployment(namespace);
@ -50,8 +50,8 @@ export async function getVolumes() {
  return serverVolumes;
 }

-export function getServerAssets(serverName) {
-  const serverFilter = serverMatch(serverName);
+export function getServerAssets(serverId) {
+  const serverFilter = serverMatch(serverId);
  return Promise.all([
    getDeployments(),
    getServices(),
@ -66,17 +66,18 @@ export function getServerAssets(serverName) {

      if (deployments.length > 1) throw Error("Deployment filter broken!");
      if (volumes.length > 1) throw Error("Volume filter broken!");
-      if (secrets.length > 1) throw Error("Secrets broken!");
+      if (secrets.length > 2) throw Error("Secrets broken!");
      const serverAssets = {
        deployment: deployments[0],
-        service: services.find(
-          (s) => s.metadata.name === `mcl-${serverName}-server`,
-        ),
+        service: services.find((s) => s.metadata.name.endsWith("-server")),
        volume: volumes[0],
-        rconService: services.find(
-          (s) => s.metadata.name === `mcl-${serverName}-rcon`,
+        rconService: services.find((s) => s.metadata.name.endsWith("-rcon")),
+        rconSecret: secrets.find((s) =>
+          s.metadata.name.endsWith("-rcon-secret"),
+        ),
+        backupSecret: secrets.find((s) =>
+          s.metadata.name.endsWith("-backup-secret"),
        ),
-        rconSecret: secrets[0],
      };
      for (var k in serverAssets) if (serverAssets[k]) return serverAssets;
      // If no assets exist, return nothing
@ -84,59 +85,51 @@ export function getServerAssets(serverName) {
    .catch((e) => ERR("SERVER ASSETS", e));
 }

-export async function getDeployment(serverName) {
+export async function getDeployment(serverId) {
  const servers = await getDeployments();
  const serverDeployment = servers.find(
-    (s) =>
-      s.metadata.annotations["minecluster.dunemask.net/server-name"] ===
-      serverName,
+    (s) => s.metadata.annotations["minecluster.dunemask.net/id"] === serverId,
  );
  if (!serverDeployment)
-    throw Error(`MCL Deployment '${serverName}' could not be found!`);
+    throw Error(`MCL Deployment with ID '${serverId}' could not be found!`);

  return serverDeployment;
 }

-export async function getContainers(serverName) {
-  const deployment = await getDeployment(serverName);
+export async function getContainers(serverId) {
+  const deployment = await getDeployment(serverId);
  return deployment.spec.template.spec.containers;
 }

-async function containerControl(serverName, deployment, scaleUp) {
+async function containerControl(serverSpec, deployment, scaleUp) {
  const { containers } = deployment.spec.template.spec;
  const depFtp = containers.find((c) => c.name.endsWith("-ftp"));
  const depServer = containers.find((c) => c.name.endsWith("-server"));
  const depBackup = containers.find((c) => c.name.endsWith("-backup"));
-  const serverSpec = await getServerEntry(serverName);
  const ftpContainer = depFtp ?? getFtpContainer(serverSpec);
  const serverContainer = depServer ?? getCoreServerContainer(serverSpec);
  const backupContainer = depBackup ?? getBackupContainer(serverSpec);
-  if (scaleUp) return [ftpContainer, serverContainer];
+  if (scaleUp && serverSpec.backupEnabled)
+    return [ftpContainer, serverContainer, backupContainer];
+  else if (scaleUp) return [ftpContainer, serverContainer];
  return [ftpContainer];
 }

-export async function toggleServer(serverName, scaleUp = false) {
-  const deployment = await getDeployment(serverName);
-  deployment.spec.template.spec.containers = await containerControl(
-    serverName,
-    deployment,
-    scaleUp,
-  );
-  return k8sDeps.replaceNamespacedDeployment(
-    deployment.metadata.name,
-    namespace,
-    deployment,
-  );
+export function terminationControl(containers) {
+  return containers.length > 1 ? 30 /*seconds*/ : 1 /*seconds */;
 }

-export async function scaleDeployment(serverName, scaleUp = false) {
-  const deployment = await getDeployment(serverName);
-  if (deployment.spec.replicas === 1 && scaleUp)
-    return VERB(
-      "KSC",
-      `MCL Deployment '${serverName}' is already scaled! Ignoring scale adjustment.`,
-    );
-  deployment.spec.replicas = scaleUp ? 1 : 0;
+export async function toggleServer(serverId, scaleUp = false) {
+  const [deployment, serverSpec] = await Promise.all([
+    getDeployment(serverId),
+    getServerEntry(serverId),
+  ]);
+  const containers = await containerControl(serverSpec, deployment, scaleUp);
+  const ts = terminationControl(containers);
+
+  // Speed up container termination if not running a server
+  deployment.spec.template.spec.terminationGracePeriodSeconds = ts;
+  deployment.spec.template.spec.containers = containers;

  return k8sDeps.replaceNamespacedDeployment(
    deployment.metadata.name,
--- a/lib/k8s/server-containers.js
+++ b/lib/k8s/server-containers.js
@ -4,9 +4,9 @@ import yaml from "js-yaml";
 const loadYaml = (f) => yaml.load(fs.readFileSync(path.resolve(f), "utf8"));

 export function getFtpContainer(serverSpec) {
-  const { name } = serverSpec;
+  const { mclName } = serverSpec;
  const ftpContainer = loadYaml("lib/k8s/configs/containers/ftp-server.yml");
-  ftpContainer.name = `mcl-${name}-ftp`;
+  ftpContainer.name = `mcl-${mclName}-ftp`;
  const ftpPortList = [
    { p: 20, n: "ftp-data" },
    { p: 21, n: "ftp-commands" },
@ -22,10 +22,10 @@ export function getFtpContainer(serverSpec) {
 }

 export function getCoreServerContainer(serverSpec) {
-  const { name, version, serverType, memory } = serverSpec;
+  const { mclName, version, serverType, memory } = serverSpec;
  const container = loadYaml("lib/k8s/configs/containers/minecraft-server.yml");
  // Container Updates
-  container.name = `mcl-${name}-server`;
+  container.name = `mcl-${mclName}-server`;
  container.resources.requests.memory = `${memory}Mi`;

  const findEnv = (k) => container.env.find(({ name: n }) => n === k);
@ -36,7 +36,7 @@ export function getCoreServerContainer(serverSpec) {
  updateEnv("VERSION", version);
  updateEnv("MEMORY", `${memory}M`);
  // RCON
-  const rs = `mcl-${name}-rcon-secret`;
+  const rs = `mcl-${mclName}-rcon-secret`;
  findEnv("RCON_PASSWORD").valueFrom.secretKeyRef.name = rs;
  return container;
 }
@ -50,18 +50,29 @@ export function getServerContainer(serverSpec) {
  const updateEnv = (k, v) => (findEnv(k).value = v);

  // Enviornment variables
-  updateEnv("DIFFICULTY", difficulty);
+  /*updateEnv("DIFFICULTY", difficulty);
  updateEnv("MODE", gamemode);
  updateEnv("MOTD", motd);
  updateEnv("MAX_PLAYERS", maxPlayers);
  updateEnv("SEED", seed);
  updateEnv("OPS", ops);
-  updateEnv("WHITELIST", whitelist);
+  updateEnv("WHITELIST", whitelist); */

  return container;
 }

 export function getBackupContainer(serverSpec) {
+  const { mclName, backupEnabled, backupPath } = serverSpec;
  const container = loadYaml("lib/k8s/configs/containers/minecraft-backup.yml");
+  if (!backupEnabled) return;
+  const findEnv = (k) => container.env.find(({ name: n }) => n === k);
+  const updateEnv = (k, v) => (findEnv(k).value = v);
+  updateEnv("RCLONE_REMOTE", `${mclName}-backup`);
+  updateEnv("RCLONE_DEST_DIR", backupPath);
+  container.name = `mcl-${mclName}-backup`;
+  // RCON
+  const rs = `mcl-${mclName}-rcon-secret`;
+  findEnv("RCON_PASSWORD").valueFrom.secretKeyRef.name = rs;
+
  return container;
 }
--- a/lib/k8s/server-control.js
+++ b/lib/k8s/server-control.js
@ -1,94 +0,0 @@
-import k8s from "@kubernetes/client-node";
-import {
-  getDeployment,
-  getDeployments,
-  getServerAssets,
-  scaleDeployment,
-} from "./k8s-server-control.js";
-import { ERR } from "../util/logging.js";
-import ExpressClientError from "../util/ExpressClientError.js";
-const kc = new k8s.KubeConfig();
-kc.loadFromDefault();
-
-const k8sMetrics = new k8s.Metrics(kc);
-const namespace = process.env.MCL_SERVER_NAMESPACE;
-
-export async function startServerContainer(serverSpec) {
-  const { name } = serverSpec;
-  try {
-    await scaleDeployment(name, true);
-  } catch (e) {
-    ERR("SERVER CONTROL", e);
-    throw new ExpressClientError({
-      c: 500,
-      m: `Error updating server '${name}'!\n`,
-    });
-  }
-}
-
-export async function stopServerContainer(serverSpec) {
-  const { name } = serverSpec;
-  try {
-    await scaleDeployment(name, false);
-  } catch (e) {
-    ERR("SERVER CONTROL", e);
-    throw new ExpressClientError({
-      c: 500,
-      m: `Error updating server '${name}'!`,
-    });
-  }
-}
-
-export async function getInstances() {
-  const serverDeployments = await getDeployments();
-  const podMetricsResponse = await k8sMetrics.getPodMetrics(namespace);
-  var name, metrics, services, serverAvailable, ftpAvailable;
-  const serverInstances = serverDeployments.map((s) => {
-    name = s.metadata.annotations["minecluster.dunemask.net/server-name"];
-    metrics = null;
-    const { containers } = s.spec.template.spec;
-    services = containers.map(({ name }) => name.split("-").pop());
-    const serverStatusList = s.status.conditions.map(
-      ({ type: statusType, status: sts }) => ({ statusType, sts }),
-    );
-    const deploymentAvailable =
-      serverStatusList.find(
-        (ss) => ss.statusType === "Available" && ss.sts === "True",
-      ) !== undefined;
-    serverAvailable = services.includes(`server`) && deploymentAvailable;
-    ftpAvailable = services.includes("ftp") && deploymentAvailable;
-
-    const pod = podMetricsResponse.items.find(({ metadata: md }) => {
-      return md.labels && md.labels.app && md.labels.app === `mcl-${name}-app`;
-    });
-    if (serverAvailable && pod) {
-      const podCpus = pod.containers.map(
-        ({ usage }) => parseInt(usage.cpu) / 1_000_000,
-      );
-      const podMems = pod.containers.map(
-        ({ usage }) => parseInt(usage.memory) / 1024,
-      );
-      metrics = {
-        cpu: Math.ceil(podCpus.reduce((a, b) => a + b)),
-        memory: Math.ceil(podMems.reduce((a, b) => a + b)),
-      };
-    }
-    return { name, metrics, services, serverAvailable, ftpAvailable };
-  });
-  return serverInstances;
-}
-
-export async function getNamespaceMetrics() {
-  const serverInstances = await getInstances();
-  var clusterMetrics = { cpu: 0, memory: 0 };
-  if (servers.length > 1) {
-    const clusterCpu = serverInstances
-      .map(({ metrics }) => (metrics ? metrics.cpu : 0))
-      .reduce((a, b) => a + b);
-    const clusterMem = serverInstances
-      .map(({ metrics }) => (metrics ? metrics.memory : 0))
-      .reduce((a, b) => a + b);
-    clusterMetrics = { cpu: clusterCpu, memory: clusterMem };
-  }
-  return clusterMetrics;
-}
--- a/lib/k8s/server-create.js
+++ b/lib/k8s/server-create.js
@ -4,7 +4,7 @@ import k8s from "@kubernetes/client-node";
 import yaml from "js-yaml";
 import fs from "node:fs";
 import path from "node:path";
-import ExpressClientError from "../util/ExpressClientError.js";
+
 import {
  getFtpContainer,
  getServerContainer,
@ -19,33 +19,54 @@ const namespace = process.env.MCL_SERVER_NAMESPACE;

 const loadYaml = (f) => yaml.load(fs.readFileSync(path.resolve(f), "utf8"));

+function createBackupSecret(serverSpec) {
+  if (!serverSpec.backupEnabled) return; // If backup not defined, don't create RCLONE secret
+  const { mclName, id, backupId, backupKey, backupHost } = serverSpec;
+  const backupYaml = loadYaml("lib/k8s/configs/backup-secret.yml");
+  backupYaml.metadata.labels.app = `mcl-${mclName}-app`;
+  backupYaml.metadata.name = `mcl-${mclName}-backup-secret`;
+  backupYaml.metadata.namespace = namespace;
+  backupYaml.metadata.annotations["minecluster.dunemask.net/id"] = id;
+  const rcloneConfig = [
+    `[${mclName}-backup]`,
+    "type = s3",
+    "provider = Minio",
+    "env_auth = false",
+    `access_key_id = ${backupId}`,
+    `secret_access_key = ${backupKey}`,
+    `endpoint = ${backupHost}`,
+    `acl = private`,
+  ].join("\n");
+  backupYaml.data["rclone.conf"] = Buffer.from(rcloneConfig).toString("base64");
+  return backupYaml;
+}
+
 function createRconSecret(serverSpec) {
-  const { name } = serverSpec;
+  const { mclName, id } = serverSpec;
  const rconYaml = loadYaml("lib/k8s/configs/rcon-secret.yml");
  // TODO: Dyamic rconPassword
  const rconPassword = bcrypt.hashSync(uuidv4(), 10);
  rconYaml.data["rcon-password"] = Buffer.from(rconPassword).toString("base64");
-  rconYaml.metadata.labels.app = `mcl-${name}-app`;
-  rconYaml.metadata.name = `mcl-${name}-rcon-secret`;
+  rconYaml.metadata.labels.app = `mcl-${mclName}-app`;
+  rconYaml.metadata.name = `mcl-${mclName}-rcon-secret`;
  rconYaml.metadata.namespace = namespace;
-  rconYaml.metadata.annotations["minecluster.dunemask.net/server-name"] = name;
+  rconYaml.metadata.annotations["minecluster.dunemask.net/id"] = id;
  return rconYaml;
 }

 function createServerVolume(serverSpec) {
-  const { name } = serverSpec;
+  const { mclName, id } = serverSpec;
  const volumeYaml = loadYaml("lib/k8s/configs/server-pvc.yml");
-  volumeYaml.metadata.labels.service = `mcl-${name}-server`;
-  volumeYaml.metadata.name = `mcl-${name}-volume`;
+  volumeYaml.metadata.labels.service = `mcl-${mclName}-server`;
+  volumeYaml.metadata.name = `mcl-${mclName}-volume`;
  volumeYaml.metadata.namespace = namespace;
-  volumeYaml.metadata.annotations["minecluster.dunemask.net/server-name"] =
-    name;
-  volumeYaml.spec.resources.requests.storage = "1Gi"; // TODO: Changeme
+  volumeYaml.metadata.annotations["minecluster.dunemask.net/id"] = id;
+  volumeYaml.spec.resources.requests.storage = "5Gi"; // TODO: Changeme
  return volumeYaml;
 }

 function createServerDeploy(serverSpec) {
-  const { name, host } = serverSpec;
+  const { mclName, id, backupEnabled } = serverSpec;
  const deployYaml = loadYaml("lib/k8s/configs/server-deployment.yml");
  const { metadata } = deployYaml;
  const serverContainer = getServerContainer(serverSpec);
@ -53,39 +74,54 @@ function createServerDeploy(serverSpec) {
  const ftpContainer = getFtpContainer(serverSpec);

  // Configure Metadata;
-  metadata.name = `mcl-${name}`;
+  metadata.name = `mcl-${mclName}`;
  metadata.namespace = namespace;
-  metadata.annotations["minecluster.dunemask.net/server-name"] = name;
+  metadata.annotations["minecluster.dunemask.net/id"] = id;
  deployYaml.metadata = metadata;

+  deployYaml.spec.template.spec.terminationGracePeriodSeconds = 1;
+
  // Configure Lables & Selectors
-  deployYaml.spec.selector.matchLabels.app = `mcl-${name}-app`;
-  deployYaml.spec.template.metadata.labels.app = `mcl-${name}-app`;
+  deployYaml.spec.selector.matchLabels.app = `mcl-${mclName}-app`;
+  deployYaml.spec.template.metadata.labels.app = `mcl-${mclName}-app`;
+  deployYaml.spec.template.metadata.annotations["minecluster.dunemask.net/id"] =
+    id;

  // Volumes
  deployYaml.spec.template.spec.volumes.find(
    ({ name }) => name === "datadir",
-  ).persistentVolumeClaim.claimName = `mcl-${name}-volume`;
+  ).persistentVolumeClaim.claimName = `mcl-${mclName}-volume`;
+
+  // Backups
+  if (backupEnabled) {
+    deployYaml.spec.template.spec.volumes.push({
+      name: "rclone-config",
+      secret: {
+        defaultMode: 420,
+        items: [{ key: "rclone.conf", path: "rclone.conf" }],
+        secretName: `mcl-${mclName}-backup-secret`,
+      },
+    });
+  }

  // Apply Containers  TODO: User control for autostart
-  deployYaml.spec.template.spec.containers.push(serverContainer);
+  // deployYaml.spec.template.spec.containers.push(serverContainer);
  deployYaml.spec.template.spec.containers.push(ftpContainer);
  deployYaml.spec.replicas = 1;
  return deployYaml;
 }

 function createServerService(serverSpec) {
-  const { name, host } = serverSpec;
+  const { mclName, host, id } = serverSpec;
  const serviceYaml = loadYaml("lib/k8s/configs/server-svc.yml");
  serviceYaml.metadata.annotations["ingress.qumine.io/hostname"] = host;
  serviceYaml.metadata.annotations["mc-router.itzg.me/externalServerName"] =
    host;
-  serviceYaml.metadata.labels.app = `mcl-${name}-app`;
-  serviceYaml.metadata.name = `mcl-${name}-server`;
+  serviceYaml.metadata.labels.app = `mcl-${mclName}-app`;
+  serviceYaml.metadata.name = `mcl-${mclName}-server`;
  serviceYaml.metadata.namespace = namespace;
-  serviceYaml.metadata.annotations["minecluster.dunemask.net/server-name"] =
-    name;
-  serviceYaml.spec.selector.app = `mcl-${name}-app`;
+  serviceYaml.metadata.annotations["minecluster.dunemask.net/id"] = id;
+  serviceYaml.spec.selector.app = `mcl-${mclName}-app`;
  // Port List:
  const serverPortList = [{ p: 25565, n: "minecraft" }];

@ -107,33 +143,26 @@ function createServerService(serverSpec) {
  return serviceYaml;
 }

-function createRconService(serverSpec) {
-  const { name } = serverSpec;
+function createRconService(createSpec) {
+  const { id, mclName } = createSpec;
  const rconSvcYaml = loadYaml("lib/k8s/configs/rcon-svc.yml");
-  rconSvcYaml.metadata.labels.app = `mcl-${name}-app`;
-  rconSvcYaml.metadata.name = `mcl-${name}-rcon`;
+  rconSvcYaml.metadata.labels.app = `mcl-${mclName}-app`;
+  rconSvcYaml.metadata.name = `mcl-${mclName}-rcon`;
  rconSvcYaml.metadata.namespace = namespace;
-  rconSvcYaml.metadata.annotations["minecluster.dunemask.net/server-name"] =
-    name;
-  rconSvcYaml.spec.selector.app = `mcl-${name}-app`;
+  rconSvcYaml.metadata.annotations["minecluster.dunemask.net/id"] = id;
+  rconSvcYaml.spec.selector.app = `mcl-${mclName}-app`;
  return rconSvcYaml;
 }

-export default async function createServerResources(serverSpec) {
-  const deploymentRes = await k8sDeps.listNamespacedDeployment(namespace);
-  const deployments = deploymentRes.body.items.map((i) => i.metadata.name);
-  if (deployments.includes(`mcl-${serverSpec.name}`))
-    throw new ExpressClientError({ m: "Server already exists!", c: 409 });
-  const pvcRes = await k8sCore.listNamespacedPersistentVolumeClaim(namespace);
-  const pvcs = pvcRes.body.items.map((i) => i.metadata.name);
-  if (pvcs.includes(`mcl-${serverSpec.name}-volume`))
-    throw new ExpressClientError({ m: "Server PVC already exists!", c: 409 });
-  const rconSecret = createRconSecret(serverSpec);
-  const serverVolume = createServerVolume(serverSpec);
-  const serverDeploy = createServerDeploy(serverSpec);
-  const serverService = createServerService(serverSpec);
-  const rconService = createRconService(serverSpec);
+export default async function createServerResources(createSpec) {
+  const backupSecret = createBackupSecret(createSpec);
+  const rconSecret = createRconSecret(createSpec);
+  const serverVolume = createServerVolume(createSpec);
+  const serverDeploy = createServerDeploy(createSpec);
+  const serverService = createServerService(createSpec);
+  const rconService = createRconService(createSpec);
  k8sCore.createNamespacedPersistentVolumeClaim(namespace, serverVolume);
+  if (!!backupSecret) k8sCore.createNamespacedSecret(namespace, backupSecret);
  k8sCore.createNamespacedSecret(namespace, rconSecret);
  k8sCore.createNamespacedService(namespace, serverService);
  k8sCore.createNamespacedService(namespace, rconService);
--- a/lib/k8s/server-delete.js
+++ b/lib/k8s/server-delete.js
@ -22,9 +22,9 @@ function deleteOnExist(o, fn) {
 }

 export default async function deleteServerResources(serverSpec) {
-  const { name } = serverSpec;
+  const { id } = serverSpec;
  // Ensure deployment exists
-  const server = await getServerAssets(name);
+  const server = await getServerAssets(id);
  if (!server)
    throw new ExpressClientError({
      c: 404,
@ -47,6 +47,10 @@ export default async function deleteServerResources(serverSpec) {
  const deleteRconSecret = deleteOnExist(server.rconSecret, (name) =>
    k8sCore.deleteNamespacedSecret(name, namespace),
  );
+
+  const deleteBackupSecret = deleteOnExist(server.backupSecret, (name) =>
+    k8sCore.deleteNamespacedSecret(name, namespace),
+  );
  const deleteVolume = deleteOnExist(server.volume, (name) =>
    k8sCore.deleteNamespacedPersistentVolumeClaim(name, namespace),
  );
@ -55,6 +59,7 @@ export default async function deleteServerResources(serverSpec) {
    deleteService,
    deleteRconService,
    deleteRconSecret,
+    deleteBackupSecret,
    deleteVolume,
  ]).catch(deleteError);
 }
--- a/lib/k8s/server-files.js
+++ b/lib/k8s/server-files.js
@ -34,8 +34,8 @@ export async function getFtpClient(serverService) {
 }

 export async function useServerFtp(serverSpec, fn) {
-  const { name } = serverSpec;
-  const server = await getServerAssets(name);
+  const { id } = serverSpec;
+  const server = await getServerAssets(id);
  if (!server)
    throw new ExpressClientError({
      c: 404,
--- a/lib/k8s/server-status.js
+++ b/lib/k8s/server-status.js
@ -0,0 +1,83 @@
+import k8s from "@kubernetes/client-node";
+import { getDeployments } from "./k8s-server-control.js";
+import { getServerEntries } from "../database/queries/server-queries.js";
+const kc = new k8s.KubeConfig();
+kc.loadFromDefault();
+
+const k8sMetrics = new k8s.Metrics(kc);
+const namespace = process.env.MCL_SERVER_NAMESPACE;
+
+function getServerMetrics(podMetricsRes, serverId, serverAvailable) {
+  const pod = podMetricsRes.items.find(({ metadata: md }) => {
+    return (
+      md.annotations &&
+      md.annotations["minecluster.dunemask.net/id"] === serverId
+    );
+  });
+  if (!serverAvailable || !pod) return null;
+  const podCpus = pod.containers.map(
+    ({ usage }) => parseInt(usage.cpu) / 1_000_000,
+  );
+  const podMems = pod.containers.map(
+    ({ usage }) => parseInt(usage.memory) / 1024,
+  );
+  return {
+    cpu: Math.ceil(podCpus.reduce((a, b) => a + b)),
+    memory: Math.ceil(podMems.reduce((a, b) => a + b)),
+  };
+}
+
+function getServerStatus(server) {
+  const { containers } = server.spec.template.spec;
+  const services = containers.map(({ name }) => name.split("-").pop());
+  const serverStatusList = server.status.conditions.map(
+    ({ type: statusType, status: sts }) => ({ statusType, sts }),
+  );
+  const deploymentAvailable =
+    serverStatusList.find(
+      (ss) => ss.statusType === "Available" && ss.sts === "True",
+    ) !== undefined;
+  const serverAvailable = services.includes(`server`) && deploymentAvailable;
+  const ftpAvailable = services.includes("ftp"); // TODO this needs some handling for container creation
+  return { serverAvailable, ftpAvailable, services };
+}
+
+export async function getInstances() {
+  const [serverDeployments, podMetricsRes, entries] = await Promise.all([
+    getDeployments(),
+    k8sMetrics.getPodMetrics(namespace),
+    getServerEntries(),
+  ]);
+
+  var serverId, metrics;
+  const serverInstances = serverDeployments.map((s) => {
+    serverId = s.metadata.annotations["minecluster.dunemask.net/id"];
+    const entry = entries.find((e) => e.id === serverId);
+    const { ftpAvailable, serverAvailable, services } = getServerStatus(s);
+    metrics = getServerMetrics(podMetricsRes, serverId, serverAvailable);
+    return {
+      name: !!entry ? entry.name : "Unknown",
+      id: serverId,
+      metrics,
+      services,
+      serverAvailable,
+      ftpAvailable,
+    };
+  });
+  return serverInstances;
+}
+
+export async function getNamespaceMetrics() {
+  const serverInstances = await getInstances();
+  var clusterMetrics = { cpu: 0, memory: 0 };
+  if (servers.length > 1) {
+    const clusterCpu = serverInstances
+      .map(({ metrics }) => (metrics ? metrics.cpu : 0))
+      .reduce((a, b) => a + b);
+    const clusterMem = serverInstances
+      .map(({ metrics }) => (metrics ? metrics.memory : 0))
+      .reduce((a, b) => a + b);
+    clusterMetrics = { cpu: clusterCpu, memory: clusterMem };
+  }
+  return clusterMetrics;
+}