[FEATURE] Massively increased loading time

README.md

@@ -2,3 +2,6 @@
 Minecluster or MCL is a web interface used to manage multiple instance of Minecraft Servers in Kubernetes. This app is built to be an all in one for self-hosting Minecraft server. It uses rendered helm charts based on itzg/minecraft-server 
 
 More info coming soon.
+
+## ⚠ Warning ⚠
+Development is very active and there is no garuntee for compatability or migration across versions 1/15/24

lib/controllers/lifecycle-controller.js

@@ -24,12 +24,13 @@ function payloadFilter(req, res) {
   if (!version) return res.status(400).send("Server version is required!");
   if (!serverType) return res.status(400).send("Server type is required!");
   if (!memory) return res.status(400).send("Memory is required!");
+  // TODO: Impliment non creation time backups
   if (
     !!backupHost ||
     !!backupBucket ||
     !!backupId ||
     !!backupKey ||
-    !backupInterval
+    !!backupInterval
   ) {
     // If any keys are required, all are required
     if (

lib/database/migrations/1_create_servers_table.sql

@@ -7,6 +7,7 @@ CREATE TABLE servers (
   server_type varchar(63) DEFAULT 'VANILLA',
   cpu varchar(63) DEFAULT '500',
   memory varchar(63) DEFAULT '512',
+  backup_enabled BOOLEAN DEFAULT FALSE,
   backup_host varchar(255) DEFAULT NULL,
   backup_bucket_path varchar(255) DEFAULT NULL,
   backup_id varchar(255) DEFAULT NULL,

lib/database/queries/server-queries.js

@@ -28,6 +28,7 @@ export async function createServerEntry(serverSpec) {
     version,
     server_type,
     memory,
+    backup_enabled: !!backup_interval, // We already verified the payload, so any backup key will work
     backup_host,
     backup_bucket_path,
     backup_id,
@@ -44,9 +45,29 @@ export async function createServerEntry(serverSpec) {
       version,
       server_type: serverType,
       memory,
+      backup_enabled: backupEnabled,
+      backup_host: backupHost,
+      backup_bucket_path: backupPath,
+      backup_id: backupId,
+      backup_key: backupKey,
+      backup_interval: backupInterval,
     } = entries[0];
     const mclName = getMclName(host, id);
-    return { name, mclName, id, host, version, serverType, memory };
+    return {
+      name,
+      mclName,
+      id,
+      host,
+      version,
+      serverType,
+      memory,
+      backupEnabled,
+      backupHost,
+      backupPath,
+      backupId,
+      backupKey,
+      backupInterval,
+    };
   } catch (e) {
     asExpressClientError(e);
   }
@@ -73,6 +94,7 @@ export async function getServerEntry(serverId) {
       version,
       server_type: serverType,
       memory,
+      backup_enabled: backupEnabled,
       backup_host: backupHost,
       backup_bucket_path: backupPath,
       backup_id: backupId,
@@ -88,12 +110,12 @@ export async function getServerEntry(serverId) {
       version,
       serverType,
       memory,
+      backupEnabled,
       backupHost,
       backupPath,
       backupId,
       backupKey,
-      backupInterval
-
+      backupInterval,
     };
   } catch (e) {
     asExpressClientError(e);

lib/k8s/configs/containers/ftp-server.yml

@@ -6,23 +6,21 @@ env:
 image: garethflowers/ftp-server
 imagePullPolicy: IfNotPresent
 livenessProbe:
-  exec:
-    command: ["echo"]
-failureThreshold: 20
-initialDelaySeconds: 5
-periodSeconds: 5
-successThreshold: 1
-timeoutSeconds: 1
+  exec: { command: ["echo"] }
+  failureThreshold: 20
+  initialDelaySeconds: 0
+  periodSeconds: 5
+  successThreshold: 1
+  timeoutSeconds: 1
 name: changeme-name-ftp
 ports: [] # Programatically add all the ports for easier readability, Ports include: 20,21,40000-400009
 readinessProbe:
-  exec:
-    command: ["echo"]
-    failureThreshold: 20
-    initialDelaySeconds: 5
-    periodSeconds: 5
-    successThreshold: 1
-    timeoutSeconds: 1
+  exec: { command: ["echo"] }
+  failureThreshold: 20
+  initialDelaySeconds: 0
+  periodSeconds: 5
+  successThreshold: 1
+  timeoutSeconds: 1
 resources:
   requests:
     cpu: 50m

lib/k8s/configs/containers/minecraft-server.yml

@@ -72,12 +72,10 @@ env:
 image: itzg/minecraft-server:latest
 imagePullPolicy: IfNotPresent
 livenessProbe:
-  exec:
-    # command: ["mc-health"] # This is super unsafe... but why not :)
-    command: ["echo"]
-  failureThreshold: 20
-  initialDelaySeconds: 5
-  periodSeconds: 5
+  exec: { command: [mc-health] }
+  failureThreshold: 200
+  initialDelaySeconds: 30
+  periodSeconds: 3
   successThreshold: 1
   timeoutSeconds: 1
 name: changeme-name-server
@@ -88,15 +86,13 @@ ports:
   - containerPort: 25575
     name: rcon
     protocol: TCP
-readinessProbe:
-  exec:
-    # command: ["mc-health"] # This is super unsafe... but why not :)
-    command: ["echo"]
-  failureThreshold: 20
-  initialDelaySeconds: 5
-  periodSeconds: 5
-  successThreshold: 1
-  timeoutSeconds: 1
+# readinessProbe: # Disabling this allows for users to manipulate files even if the container is starting
+#   exec: {command: [mc-health]}
+#   failureThreshold: 200
+#   initialDelaySeconds: 30
+#   periodSeconds: 3
+#   successThreshold: 1
+#   timeoutSeconds: 1
 resources:
   requests:
     cpu: 500m

lib/k8s/configs/server-deployment.yml

@@ -35,10 +35,10 @@ spec:
             claimName: changeme-pvc-name
         - emptyDir: {}
           name: backupdir
-        - name: rclone-config
-          secret:
-            defaultMode: 420
-            items:
-              - key: rclone.conf
-                path: rclone.conf
-            secretName: rclone-config
+        # - name: rclone-config
+        #   secret:
+        #     defaultMode: 420
+        #     items:
+        #       - key: rclone.conf
+        #         path: rclone.conf
+        #     secretName: rclone-config

lib/k8s/k8s-server-control.js

@@ -66,13 +66,18 @@ export function getServerAssets(serverId) {
 
       if (deployments.length > 1) throw Error("Deployment filter broken!");
       if (volumes.length > 1) throw Error("Volume filter broken!");
-      if (secrets.length > 1) throw Error("Secrets broken!");
+      if (secrets.length > 2) throw Error("Secrets broken!");
       const serverAssets = {
         deployment: deployments[0],
         service: services.find((s) => s.metadata.name.endsWith("-server")),
         volume: volumes[0],
         rconService: services.find((s) => s.metadata.name.endsWith("-rcon")),
-        rconSecret: secrets[0],
+        rconSecret: secrets.find((s) =>
+          s.metadata.name.endsWith("-rcon-secret"),
+        ),
+        backupSecret: secrets.find((s) =>
+          s.metadata.name.endsWith("-backup-secret"),
+        ),
       };
       for (var k in serverAssets) if (serverAssets[k]) return serverAssets;
       // If no assets exist, return nothing
@@ -96,26 +101,36 @@ export async function getContainers(serverId) {
   return deployment.spec.template.spec.containers;
 }
 
-async function containerControl(serverId, deployment, scaleUp) {
+async function containerControl(serverSpec, deployment, scaleUp) {
   const { containers } = deployment.spec.template.spec;
   const depFtp = containers.find((c) => c.name.endsWith("-ftp"));
   const depServer = containers.find((c) => c.name.endsWith("-server"));
   const depBackup = containers.find((c) => c.name.endsWith("-backup"));
-  const serverSpec = await getServerEntry(serverId);
   const ftpContainer = depFtp ?? getFtpContainer(serverSpec);
   const serverContainer = depServer ?? getCoreServerContainer(serverSpec);
   const backupContainer = depBackup ?? getBackupContainer(serverSpec);
-  if (scaleUp) return [ftpContainer, serverContainer];
+  if (scaleUp && serverSpec.backupEnabled)
+    return [ftpContainer, serverContainer, backupContainer];
+  else if (scaleUp) return [ftpContainer, serverContainer];
   return [ftpContainer];
 }
 
+export function terminationControl(containers) {
+  return containers.length > 1 ? 30 /*seconds*/ : 1 /*seconds */;
+}
+
 export async function toggleServer(serverId, scaleUp = false) {
-  const deployment = await getDeployment(serverId);
-  deployment.spec.template.spec.containers = await containerControl(
-    serverId,
-    deployment,
-    scaleUp,
-  );
+  const [deployment, serverSpec] = await Promise.all([
+    getDeployment(serverId),
+    getServerEntry(serverId),
+  ]);
+  const containers = await containerControl(serverSpec, deployment, scaleUp);
+  const ts = terminationControl(containers);
+
+  // Speed up container termination if not running a server
+  deployment.spec.template.spec.terminationGracePeriodSeconds = ts;
+  deployment.spec.template.spec.containers = containers;
+
   return k8sDeps.replaceNamespacedDeployment(
     deployment.metadata.name,
     namespace,

lib/k8s/server-containers.js

@@ -62,6 +62,17 @@ export function getServerContainer(serverSpec) {
 }
 
 export function getBackupContainer(serverSpec) {
+  const { mclName, backupEnabled, backupPath } = serverSpec;
   const container = loadYaml("lib/k8s/configs/containers/minecraft-backup.yml");
+  if (!backupEnabled) return;
+  const findEnv = (k) => container.env.find(({ name: n }) => n === k);
+  const updateEnv = (k, v) => (findEnv(k).value = v);
+  updateEnv("RCLONE_REMOTE", `${mclName}-backup`);
+  updateEnv("RCLONE_DEST_DIR", backupPath);
+  container.name = `mcl-${mclName}-backup`;
+  // RCON
+  const rs = `mcl-${mclName}-rcon-secret`;
+  findEnv("RCON_PASSWORD").valueFrom.secretKeyRef.name = rs;
+
   return container;
 }

lib/k8s/server-create.js

@@ -19,6 +19,28 @@ const namespace = process.env.MCL_SERVER_NAMESPACE;
 
 const loadYaml = (f) => yaml.load(fs.readFileSync(path.resolve(f), "utf8"));
 
+function createBackupSecret(serverSpec) {
+  if (!serverSpec.backupEnabled) return; // If backup not defined, don't create RCLONE secret
+  const { mclName, id, backupId, backupKey, backupHost } = serverSpec;
+  const backupYaml = loadYaml("lib/k8s/configs/backup-secret.yml");
+  backupYaml.metadata.labels.app = `mcl-${mclName}-app`;
+  backupYaml.metadata.name = `mcl-${mclName}-backup-secret`;
+  backupYaml.metadata.namespace = namespace;
+  backupYaml.metadata.annotations["minecluster.dunemask.net/id"] = id;
+  const rcloneConfig = [
+    `[${mclName}-backup]`,
+    "type = s3",
+    "provider = Minio",
+    "env_auth = false",
+    `access_key_id = ${backupId}`,
+    `secret_access_key = ${backupKey}`,
+    `endpoint = ${backupHost}`,
+    `acl = private`,
+  ].join("\n");
+  backupYaml.data["rclone.conf"] = Buffer.from(rcloneConfig).toString("base64");
+  return backupYaml;
+}
+
 function createRconSecret(serverSpec) {
   const { mclName, id } = serverSpec;
   const rconYaml = loadYaml("lib/k8s/configs/rcon-secret.yml");
@@ -39,12 +61,12 @@ function createServerVolume(serverSpec) {
   volumeYaml.metadata.name = `mcl-${mclName}-volume`;
   volumeYaml.metadata.namespace = namespace;
   volumeYaml.metadata.annotations["minecluster.dunemask.net/id"] = id;
-  volumeYaml.spec.resources.requests.storage = "1Gi"; // TODO: Changeme
+  volumeYaml.spec.resources.requests.storage = "5Gi"; // TODO: Changeme
   return volumeYaml;
 }
 
 function createServerDeploy(serverSpec) {
-  const { mclName, id } = serverSpec;
+  const { mclName, id, backupEnabled } = serverSpec;
   const deployYaml = loadYaml("lib/k8s/configs/server-deployment.yml");
   const { metadata } = deployYaml;
   const serverContainer = getServerContainer(serverSpec);
@@ -57,6 +79,8 @@ function createServerDeploy(serverSpec) {
   metadata.annotations["minecluster.dunemask.net/id"] = id;
   deployYaml.metadata = metadata;
 
+  deployYaml.spec.template.spec.terminationGracePeriodSeconds = 1;
+
   // Configure Lables & Selectors
   deployYaml.spec.selector.matchLabels.app = `mcl-${mclName}-app`;
   deployYaml.spec.template.metadata.labels.app = `mcl-${mclName}-app`;
@@ -68,6 +92,18 @@ function createServerDeploy(serverSpec) {
     ({ name }) => name === "datadir",
   ).persistentVolumeClaim.claimName = `mcl-${mclName}-volume`;
 
+  // Backups
+  if (backupEnabled) {
+    deployYaml.spec.template.spec.volumes.push({
+      name: "rclone-config",
+      secret: {
+        defaultMode: 420,
+        items: [{ key: "rclone.conf", path: "rclone.conf" }],
+        secretName: `mcl-${mclName}-backup-secret`,
+      },
+    });
+  }
+
   // Apply Containers  TODO: User control for autostart
   // deployYaml.spec.template.spec.containers.push(serverContainer);
   deployYaml.spec.template.spec.containers.push(ftpContainer);
@@ -119,12 +155,14 @@ function createRconService(createSpec) {
 }
 
 export default async function createServerResources(createSpec) {
+  const backupSecret = createBackupSecret(createSpec);
   const rconSecret = createRconSecret(createSpec);
   const serverVolume = createServerVolume(createSpec);
   const serverDeploy = createServerDeploy(createSpec);
   const serverService = createServerService(createSpec);
   const rconService = createRconService(createSpec);
   k8sCore.createNamespacedPersistentVolumeClaim(namespace, serverVolume);
+  if (!!backupSecret) k8sCore.createNamespacedSecret(namespace, backupSecret);
   k8sCore.createNamespacedSecret(namespace, rconSecret);
   k8sCore.createNamespacedService(namespace, serverService);
   k8sCore.createNamespacedService(namespace, rconService);

lib/k8s/server-delete.js

@@ -47,6 +47,10 @@ export default async function deleteServerResources(serverSpec) {
   const deleteRconSecret = deleteOnExist(server.rconSecret, (name) =>
     k8sCore.deleteNamespacedSecret(name, namespace),
   );
+
+  const deleteBackupSecret = deleteOnExist(server.backupSecret, (name) =>
+    k8sCore.deleteNamespacedSecret(name, namespace),
+  );
   const deleteVolume = deleteOnExist(server.volume, (name) =>
     k8sCore.deleteNamespacedPersistentVolumeClaim(name, namespace),
   );
@@ -55,6 +59,7 @@ export default async function deleteServerResources(serverSpec) {
     deleteService,
     deleteRconService,
     deleteRconSecret,
+    deleteBackupSecret,
     deleteVolume,
   ]).catch(deleteError);
 }

lib/k8s/server-status.js

@@ -56,7 +56,7 @@ export async function getInstances() {
     const { ftpAvailable, serverAvailable, services } = getServerStatus(s);
     metrics = getServerMetrics(podMetricsRes, serverId, serverAvailable);
     return {
-      name: entry.name,
+      name: !!entry ? entry.name : "Unknown",
       id: serverId,
       metrics,
       services,

src/pages/CreateCoreOptions.jsx

@@ -54,7 +54,7 @@ export default function CreateCoreOptions() {
   async function upsertSpec() {
     if (validateSpec() !== "validated") return;
     createServer(spec)
-      .then(() => nav("/"))
+      // .then(() => nav("/"))
       .catch(alert);
   }