diff --git a/lib/core/JobManager.js b/lib/core/JobManager.js deleted file mode 100644 index 579e2a7..0000000 --- a/lib/core/JobManager.js +++ /dev/null @@ -1,61 +0,0 @@ -import { v4 } from "uuid"; -import applyJob from "./kubernetes.js"; -import buildJob from "./job-builder.js"; - -const maxJobs = process.env.MAX_JOBS ? parseInt(process.env.MAX_JOBS) : 3; - -class JobManager { - constructor() { - this.clientMaxJobs = maxJobs; - this.clients = {}; - } - - getJob(clientId, jobId) { - return this.clients[clientId].jobs.find((j) => j.id === jobId); - } - - getJobById(jobId) { - for (var client of Object.values(this.clients)) { - const job = client.jobs.find((j) => j.id === jobId); - if (!job) continue; - return job; - } - } - - pushLog(jobId, log) { - const job = this.getJobById(jobId); - if (log instanceof Array) job.log.push(...log); - else job.log.push(log); - } - - closeJob(jobId, exitcode) { - const job = this.getJobById(jobId); - job.exitcode = exitcode; - } - - newJob(jobRequest, id) { - if (!jobRequest) throw Error("Request Must Be Object!"); - if (!this.clients[id]) this.clients[id] = { jobs: [] }; - const client = this.clients[id]; - if ( - client.jobs.filter((j) => j.exitcode === undefined).length >= - this.clientMaxJobs - ) - throw Error("Client's Active Jobs Exceeded!"); - - const job = buildJob(jobRequest, id); - job.id = v4(); - job.log = []; - this.clients[id].jobs.push(job); - applyJob(job); - return { ...job }; - } - - removeJob(clientId, id) { - this.clients[clientId].jobs = this.clients[clientId].jobs.filter( - (j) => j.id !== id - ); - } -} - -export default new JobManager(); diff --git a/lib/core/Qualiteer.js b/lib/core/Qualiteer.js index efaef7f..87e08c7 100644 --- a/lib/core/Qualiteer.js +++ b/lib/core/Qualiteer.js @@ -1,13 +1,15 @@ // Imports import fig from "figlet"; import http from "http"; +import express from "express"; import { INFO, OK, logInfo } from "../util/logging.js"; // Import Core Modules -import expressApp from "./server.js"; -import applySockets from "../sockets/handler.js"; -import jobManager from "./JobManager.js"; -import getRabbiteer from "../rabbit/rabbit-workers.js"; +import buildRoutes from "../routes/router.js"; +import buildPostgres from "../database/postgres.js"; +import injectSockets from "../sockets/socket-server.js"; +import JobManager from "../jobs/JobManager.js"; +import buildRabbiteer from "../rabbit/rabbit-workers.js"; // Constants const title = "QLTR"; @@ -17,24 +19,32 @@ const port = process.env.QUALITEER_DEV_PORT ?? 52000; export default class Qualiteer { constructor(options = {}) { for (var k in options) this[k] = options[k]; - this.jobs = jobManager; + this.jobs = JobManager; this.port = options.port ?? port; } async _preinitialize() { logInfo(fig.textSync(title, "Cyberlarge")); INFO("INIT", "Initializing..."); - this.app = expressApp; + this.app = express(); + this.pg = buildPostgres(); this.server = http.createServer(this.app); - this.sockets = applySockets(this.server, this.jobs); - this.app.set("socketio", this.sockets); - this.rabbiteer = getRabbiteer(this.sockets); + this.sockets = injectSockets(this.server, this.jobs); + this.routes = buildRoutes(this.pg, this.sockets); + this.rabbiteer = buildRabbiteer(this.pg, this.sockets); + this.app.use(this.routes); + } + + async _connect() { + await this.pg.connect(); + // await this.rabbiteer.connect(); } start() { const qt = this; return new Promise(async function init(res) { - await qt._preinitialize(); + qt._preinitialize(); + await qt._connect(); qt.server.listen(qt.port, function onStart() { OK("SERVER", `Running on ${qt.port}`); res(); diff --git a/lib/core/executor.js b/lib/core/executor.js deleted file mode 100644 index e558991..0000000 --- a/lib/core/executor.js +++ /dev/null @@ -1,9 +0,0 @@ -import Executor from "../sockets/clients/Executor.js"; - -const args = process.argv.slice(2); -const url = args[0]; -const jobId = args[1]; -const command = args.slice(2); -const job = { id: jobId, command }; -const exec = new Executor(url, job, command); -exec.runJob(); diff --git a/lib/core/internal-deploy.js b/lib/core/internal-deploy.js deleted file mode 100644 index 3590aed..0000000 --- a/lib/core/internal-deploy.js +++ /dev/null @@ -1,15 +0,0 @@ -import { INFO, ERR, OK, VERB } from "../util/logging.js"; -import Executor from "../sockets/clients/Executor.js"; -import cp from "node:child_process"; - -const jobStr = process.argv.slice(2)[0]; -const job = JSON.parse(jobStr); -const { command } = job.spec.template.spec.containers[0]; -INFO("EXEC", "Internal Executor Starting!"); -cp.exec(command, (error, stdout, stderr) => { - if (error) ERR("EXEC", error); - //if(stdout) VERB("EXEC-STDOUT", stdout); - //if(stderr) VERB("EXEC-STDERR", stderr); - OK("EXEC", "Internal Executor Finished!"); - process.exit(error ? 1 : 0); -}); diff --git a/lib/core/job-builder.js b/lib/core/job-builder.js deleted file mode 100644 index e0dd9c0..0000000 --- a/lib/core/job-builder.js +++ /dev/null @@ -1,59 +0,0 @@ -const baseCommand = "node"; -const suiteEntry = "dev/suite/runner.js"; -const pipelineMapping = [ - { - id: 0, - pipeline: [{ name: "primary" }, { name: "secondary", delay: 5000 }], - }, -]; - -const buildCommon = (jobRequest) => { - const { testName } = jobRequest; - if (!testName) throw Error("'testName' must be provided!"); - const command = [baseCommand, suiteEntry, `test=${testName}`]; - - // Apply Common Flags - command.push("isRetry=false"); - - // Return new request - return { ...jobRequest, command }; -}; - -const buildSingle = (jobReq) => jobReq; - -const buildMarker = (jobReq) => {}; - -const buildProject = (jobReq) => {}; - -const pipelineMaxLife = (testName) => { - const pipelines = pipelineMapping - .filter((m) => m.pipeline.find((t) => t.name === testName)) - .map((m) => m.pipeline); - return Math.max(pipelines.map((p) => p.length)) + 1; -}; - -const buildCompound = (jobReq, socketId) => { - const { testName, command } = jobReq; - const pipelineTriggers = jobReq.pipelineTriggers; - if (pipelineTriggers) command.push(`pipelineTriggers=${pipelineTriggers}`); - command.push(`pipelineDashboardSocket=${socketId}`); - return { ...jobReq, command }; -}; - -const nextCompound = (previousTest) => {}; - -export default function jobBuilder(jobRequest, id) { - const jobReq = buildCommon(jobRequest, id); - switch (jobRequest.type) { - case "single": - return buildSingle(jobReq); - case "marker": - return buildMarker(jobReq); - case "project": - return buildProject(jobReq); - case "compound": - return buildCompound(jobReq, id); - default: - throw Error("No Job Request Type Specified!"); - } -} diff --git a/lib/core/k8s-job.json b/lib/core/k8s-job.json deleted file mode 100644 index 01d8d4c..0000000 --- a/lib/core/k8s-job.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": { - "name": "qltr-job-test-suite-1" - }, - "spec": { - "template": { - "spec": { - "containers": [ - { - "resources": { - "requests": { - "memory": "64MI", - "cpu": "250m" - }, - "limits": { - "memory": "128MI", - "cpu": "500m" - } - }, - "name": "qltr-job-test-suite-1", - "image": "node", - "imagePullPolicy": "Always", - "command": ["node", "--version"] - } - ], - "restartPolicy": "Never" - } - }, - "backoffLimit": 4 - } -} diff --git a/lib/core/kubernetes.js b/lib/core/kubernetes.js deleted file mode 100644 index e0767c5..0000000 --- a/lib/core/kubernetes.js +++ /dev/null @@ -1,84 +0,0 @@ -import cp from "child_process"; -import fs from "fs"; -import path from "path"; - -const internalDeploy = process.env.INTERNAL_DEPLOY === "true"; -const executorUrl = process.env.EXECUTOR_URL; -const executorScriptOnly = process.env.EXECUTOR_SCRIPT_ONLY === "true"; -const executorBin = - process.env.EXECUTOR_BIN ?? `qltr-executor${executorScriptOnly ? ".js" : ""}`; - -const qualiteerUrl = - process.env.QUALITEER_URL ?? "file:///home/runner/Qualiteer/bin/executor"; - -const kubCmd = "kubectl apply -f"; -const jobsDir = "jobs/"; -const defaults = JSON.parse( - fs.readFileSync(path.resolve("./lib/core/k8s-job.json")) -); - -const wrapCommand = (jobId, command) => { - const bin = executorScriptOnly - ? `node ${executorBin}` - : `chmod +x ${executorBin} && ./${executorBin}`; - const cmd = command.map((arg) => JSON.stringify(arg)); - const curlCmd = `curl -o qltr-executor ${executorUrl} && ${bin} ${qualiteerUrl} ${jobId} ${cmd.join( - " " - )}`; - return curlCmd; -}; - -const createFile = (job) => { - const { name } = job.metadata; - const jobsPath = path.resolve(jobsDir); - if (!fs.existsSync(jobsPath)) fs.mkdirSync(jobsPath); - const filePath = path.resolve(jobsDir, `${name}.json`); - fs.writeFileSync(filePath, JSON.stringify(job)); - return filePath; -}; - -const applyFileInternally = (filePath) => { - const job = fs.readFileSync(filePath, { encoding: "utf8" }); - cp.fork(path.resolve("./lib/core/internal-deploy.js"), [job]); -}; - -const applyFile = async (filePath) => { - const command = `${kubCmd} ${filePath}`; - return new Promise((res, rej) => - cp.exec(command, (err, stdout, stderr) => (err && rej(err)) || res(stdout)) - ); -}; - -const deleteFile = (filePath) => fs.unlinkSync(filePath); - -const jobBuilder = (jobRequest) => { - const { resources, name, image, command, id: jobId } = jobRequest; - - // Safety Checks - if (!jobId) throw Error("'jobId' required!"); - if (!name) throw Error("'name' required!"); - if (!command) throw Error("'command' required!"); - if (!image) throw Error("'image' required!"); - - if (!Array.isArray(command)) throw Error("'command' must be an array!"); - - // Apply configuration - const job = { ...defaults }; - job.metadata.name = `qltr-${name}-${jobId}`; - const container = job.spec.template.spec.containers[0]; - container.name = job.metadata.name; - container.command = wrapCommand(jobId, command); - container.image = JSON.stringify(image); - - // Apply resources - job.resources = { ...job.resources, ...resources }; - return job; -}; - -export default async function createJob(jobRequest) { - const job = jobBuilder(jobRequest); - const filePath = createFile(job); - if (!internalDeploy) await applyFile(filePath); - else await applyFileInternally(filePath); - deleteFile(filePath); -} diff --git a/lib/core/server.js b/lib/core/server.js deleted file mode 100644 index ff9afbc..0000000 --- a/lib/core/server.js +++ /dev/null @@ -1,28 +0,0 @@ -// Imports -import express from "express"; - -// Routes -import results from "../routes/results-route.js"; -import alerting from "../routes/alerting-route.js"; -import react from "../routes/react-route.js"; -import catalog from "../routes/catalog-route.js"; -import jobs from "../routes/jobs-route.js"; - -import mock from "../routes/mock-route.js"; -import dev from "../routes/dev-route.js"; -const app = express(); -// Special Routes -app.all("/", (req, res) => res.redirect("/qualiteer")); -if (process.env.MOCK_ROUTES === "true") app.use(mock); -if (process.env.USE_DEV_ROUTER === "true") app.use("/api/dev", dev); - -// Middlewares - -// Routes -app.use(react); // Static Build Route -app.use("/api/results", results); -app.use("/api/alerting", alerting); -app.use("/api/catalog", catalog); -app.use("/api/jobs", jobs); - -export default app; diff --git a/lib/routes/mocks/alerting.json b/lib/database/mocks/alerting.json similarity index 100% rename from lib/routes/mocks/alerting.json rename to lib/database/mocks/alerting.json diff --git a/lib/routes/mocks/catalog.json b/lib/database/mocks/catalog.json similarity index 100% rename from lib/routes/mocks/catalog.json rename to lib/database/mocks/catalog.json diff --git a/lib/routes/mocks/results.json b/lib/database/mocks/results.json similarity index 100% rename from lib/routes/mocks/results.json rename to lib/database/mocks/results.json diff --git a/lib/database/postgres.js b/lib/database/postgres.js index ea1ecae..048c624 100644 --- a/lib/database/postgres.js +++ b/lib/database/postgres.js @@ -31,17 +31,23 @@ const dbConfig = { const migrationsDir = "lib/database/migrations"; -const configure = async () => { +const queryMock = (str) => INFO("POSTGRES MOCK", str); + +const connect = (pg) => async () => { if (pgDisabled) { WARN("POSTGRES", "Postgres Disabled!"); - return { query: (str) => INFO("POSTGRES MOCK", str) }; + return { query: queryMock }; } await migrate(dbConfig, migrationsDir); // Override the global variable DB - const pg = pgp(dbConfig); + pg = pgp(dbConfig); await pg.connect(); OK("POSTGRES", `Connected to database ${database}!`); +}; + +const buildPostgres = () => { + var pg = { query: queryMock, connect: connect(pg) }; return pg; }; -export default await configure(); +export default buildPostgres; diff --git a/lib/rabbit/rabbit-workers.js b/lib/rabbit/rabbit-workers.js index 13b1da9..b110c46 100644 --- a/lib/rabbit/rabbit-workers.js +++ b/lib/rabbit/rabbit-workers.js @@ -1,5 +1,5 @@ import Rabbiteer from "rabbiteer"; -import getWorkers from "./workers/index.js"; +import buildWorkers from "./workers/index.js"; // Pull Environment Variables const { RABBIT_HOST: host, RABBIT_USER: user, RABBIT_PASS: pass } = process.env; @@ -11,7 +11,7 @@ const rabbitConfig = { pass: pass ?? "rabbit", }; -const getRabbiteer = (skio) => - new Rabbiteer(null, getWorkers(skio), { autoRabbit: rabbitConfig }); +const buildRabbiteer = (skio) => + new Rabbiteer(null, buildWorkers(skio), { autoRabbit: rabbitConfig }); -export default getRabbiteer; +export default buildRabbiteer; diff --git a/lib/rabbit/workers/index.js b/lib/rabbit/workers/index.js index 6b9c976..b461244 100644 --- a/lib/rabbit/workers/index.js +++ b/lib/rabbit/workers/index.js @@ -1,4 +1,4 @@ import TestResultsWorker from "./TestResultsWorker.js"; -const getWorkers = (skio) => [new TestResultsWorker(skio)]; -export default getWorkers; +const buildWorkers = (skio) => [new TestResultsWorker(skio)]; +export default buildWorkers; diff --git a/lib/routes/dev-route.js b/lib/routes/dev-route.js index 74aa541..3240588 100644 --- a/lib/routes/dev-route.js +++ b/lib/routes/dev-route.js @@ -1,13 +1,13 @@ import { Router, json as jsonMiddleware } from "express"; import TestResultsWorker from "../rabbit/workers/TestResultsWorker.js"; -const router = Router(); -router.use(jsonMiddleware()); -router.post("/rabbit/TestResults", (req, res) => { - const { testResult } = req.body; - var io = req.app.get("socketio"); - new TestResultsWorker(io).onMessage(testResult); - res.sendStatus(200); -}); - -export default router; +export default function buildDevRoute(pg, skio) { + const router = Router(); + router.use(jsonMiddleware()); + router.post("/rabbit/TestResults", (req, res) => { + const { testResult } = req.body; + new TestResultsWorker(skio).onMessage(testResult); + res.sendStatus(200); + }); + return router; +} diff --git a/lib/routes/jobs-route.js b/lib/routes/jobs-route.js deleted file mode 100644 index c5149e8..0000000 --- a/lib/routes/jobs-route.js +++ /dev/null @@ -1,12 +0,0 @@ -import { Router, json as jsonMiddleware } from "express"; -import jobs from "../core/JobManager.js"; - -const router = Router(); - -router.get("/jobs", (req, res) => { - const { clients } = jobs; - const allJobs = []; - for (var c of clients) allJobs.push(...c.jobs); - res.json(allJobs); -}); -export default router; diff --git a/lib/routes/mock-route.js b/lib/routes/mock-route.js deleted file mode 100644 index a88bf2f..0000000 --- a/lib/routes/mock-route.js +++ /dev/null @@ -1,37 +0,0 @@ -import { Router } from "express"; -import { readFileSync } from "fs"; - -const router = Router(); - -const catalog = "lib/routes/mocks/catalog.json"; -const alerting = "lib/routes/mocks/alerting.json"; -const results = "lib/routes/mocks/results.json"; - -const query = async (mock) => JSON.parse(readFileSync(mock)); - -// Queries -router.get("/api/catalog/tests", (req, res) => { - query(catalog).then((catalog) => { - res.json(req.get("full") ? catalog["tests:full"] : catalog.tests); - }); -}); - -router.get("/api/results/failing", async (req, res) => { - query(results).then(async (results) => { - if (req.get("count")) res.json({ failing: results.results.length }); - else if (!req.get("full")) res.json(results.results); - else - query(catalog).then((catalog) => { - res.json( - results.results.map((r) => ({ - ...catalog["tests:full"].find((t) => t.name === r.name), - ...r, - })) - ); - }); - }); -}); - -// Mutations - -export default router; diff --git a/lib/routes/router.js b/lib/routes/router.js new file mode 100644 index 0000000..b617eb6 --- /dev/null +++ b/lib/routes/router.js @@ -0,0 +1,28 @@ +// Imports +import express from "express"; + +// Routes +import results from "../routes/results-route.js"; +import alerting from "../routes/alerting-route.js"; +import react from "../routes/react-route.js"; +import catalog from "../routes/catalog-route.js"; + +import buildDevRoute from "../routes/dev-route.js"; + +export default function buildRoutes(pg, skio) { + const router = express.Router(); + // Special Routes + router.all("/", (req, res) => res.redirect("/qualiteer")); + if (process.env.USE_DEV_ROUTER === "true") + router.use("/api/dev", buildDevRoute(pg, skio)); + + // Middlewares + + // Routes + router.use(react); // Static Build Route + router.use("/api/results", results); + router.use("/api/alerting", alerting); + router.use("/api/catalog", catalog); + + return router; +} diff --git a/lib/sockets/modifiers.js b/lib/sockets/client-listeners.js similarity index 100% rename from lib/sockets/modifiers.js rename to lib/sockets/client-listeners.js diff --git a/lib/sockets/handler.js b/lib/sockets/socket-server.js similarity index 94% rename from lib/sockets/handler.js rename to lib/sockets/socket-server.js index aad913f..703be2e 100644 --- a/lib/sockets/handler.js +++ b/lib/sockets/socket-server.js @@ -2,7 +2,7 @@ import { Server as Skio } from "socket.io"; import evt from "./events.js"; import modes from "./modes.js"; -import { initiator, executor, viewer } from "./modifiers.js"; +import { initiator, executor, viewer } from "./client-listeners.js"; const socketDrop = (io, room, id) => { const { rooms } = io.of("/").adapter;