diff --git a/.gitignore b/.gitignore index ccc5618..c981e02 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ node_modules/ build/ -jobs/ +jobs/* qltr-executor diff --git a/lib/jobs/JobManager.js b/lib/jobs/JobManager.js new file mode 100644 index 0000000..d90b6dc --- /dev/null +++ b/lib/jobs/JobManager.js @@ -0,0 +1,61 @@ +import { v4 } from "uuid"; +import applyJob from "./k8s/kubernetes.js"; +import buildJob from "./job-builder.js"; + +const maxJobs = process.env.MAX_JOBS ? parseInt(process.env.MAX_JOBS) : 3; + +class JobManager { + constructor() { + this.clientMaxJobs = maxJobs; + this.clients = {}; + } + + getJob(clientId, jobId) { + return this.clients[clientId].jobs.find((j) => j.id === jobId); + } + + getJobById(jobId) { + for (var client of Object.values(this.clients)) { + const job = client.jobs.find((j) => j.id === jobId); + if (!job) continue; + return job; + } + } + + pushLog(jobId, log) { + const job = this.getJobById(jobId); + if (log instanceof Array) job.log.push(...log); + else job.log.push(log); + } + + closeJob(jobId, exitcode) { + const job = this.getJobById(jobId); + job.exitcode = exitcode; + } + + newJob(jobRequest, id) { + if (!jobRequest) throw Error("Request Must Be Object!"); + if (!this.clients[id]) this.clients[id] = { jobs: [] }; + const client = this.clients[id]; + if ( + client.jobs.filter((j) => j.exitcode === undefined).length >= + this.clientMaxJobs + ) + throw Error("Client's Active Jobs Exceeded!"); + + const job = buildJob(jobRequest, id); + job.id = v4(); + job.log = []; + this.clients[id].jobs.push(job); + applyJob(job); + return { ...job }; + } + + removeJob(clientId, id) { + this.clients[clientId].jobs = this.clients[clientId].jobs.filter( + (j) => j.id !== id + ); + } +} + +export default new JobManager(); diff --git a/lib/jobs/executor/executor-bundler.js b/lib/jobs/executor/executor-bundler.js new file mode 100644 index 0000000..f157518 --- /dev/null +++ b/lib/jobs/executor/executor-bundler.js @@ -0,0 +1,48 @@ +import { URL } from "url"; +import loadConfigFile from "rollup/loadConfigFile"; +import path from "path"; +import { rollup } from "rollup"; +import caxa from "caxa"; + +import { verify, normalize } from "./executor-configurator.js"; +const { default: executorConfig } = await import(path.resolve("executor.config.js")); + +const __dirname = new URL(".", import.meta.url).pathname; +const { default: caxaPackage } = caxa; + +function testConfig() { + console.log("Testing config"); + verify(normalize(executorConfig([]))); +} + +async function packageBin() { + console.log("Packaging bundle into binary"); + return caxaPackage({ + input: "dist/bundles/", + output: "bin/executor", + command: [ + "{{caxa}}/node_modules/.bin/node", + "{{caxa}}/qualiteer-executor.mjs", + ], + uncompressionMessage: "Unpacking, please wait...", + }); +} + +async function rollupBundle() { + console.log("Rolling up executor into bundle"); + const { options, warnings } = await loadConfigFile( + path.resolve(__dirname, "rollup.config.js") + ); + console.log(`Rollup has ${warnings.count} warnings`); + warnings.flush(); + + for (const optionsObj of options) { + const bundle = await rollup(optionsObj); + await Promise.all(optionsObj.output.map(bundle.write)); + } +} + +testConfig(); +await rollupBundle(); +await packageBin(); +console.log("Done"); diff --git a/lib/jobs/executor/executor-configurator.js b/lib/jobs/executor/executor-configurator.js new file mode 100644 index 0000000..86ec34d --- /dev/null +++ b/lib/jobs/executor/executor-configurator.js @@ -0,0 +1,17 @@ +const funcify = (v) => ()=> v; + +export function verify(config) { + for (var k in config) { + if (typeof config[k] !== "function") + throw Error("All config options must be functions!"); + } +} + +export function normalize(conf) { + const config = { ...conf }; + for (var k in config) { + if (typeof config[k] === "function") continue; + config[k] = funcify(config[k]); + } + return config; +} diff --git a/lib/jobs/executor/executor-entrypoint.js b/lib/jobs/executor/executor-entrypoint.js new file mode 100644 index 0000000..9e51a60 --- /dev/null +++ b/lib/jobs/executor/executor-entrypoint.js @@ -0,0 +1,13 @@ +import path from "node:path"; +import Executor from "../../sockets/clients/Executor.js"; +import { normalize } from "./executor-configurator.js"; +const { default: executorConfig } = await import( + path.resolve("executor.config.js") +); + +// Load config and args +const args = process.argv.slice(2); +const config = normalize(executorConfig(args)); +// Start Executor +const exec = new Executor(args, config); +exec.runJob(); diff --git a/lib/jobs/executor/rollup.config.js b/lib/jobs/executor/rollup.config.js new file mode 100644 index 0000000..d107a3d --- /dev/null +++ b/lib/jobs/executor/rollup.config.js @@ -0,0 +1,11 @@ +import { nodeResolve } from "@rollup/plugin-node-resolve"; +import commonjs from "@rollup/plugin-commonjs"; +import { terser } from "rollup-plugin-terser"; + +export default { + input: "lib/jobs/executor/executor-entrypoint.js", + output: { + file: "dist/bundles/qualiteer-executor.mjs", + }, + plugins: [nodeResolve(), commonjs(), terser()], +}; diff --git a/lib/jobs/job-builder.js b/lib/jobs/job-builder.js new file mode 100644 index 0000000..39a894d --- /dev/null +++ b/lib/jobs/job-builder.js @@ -0,0 +1,59 @@ +const baseCommand = "node"; +const suiteEntry = "tests/assets/suite/runner.js"; +const pipelineMapping = [ + { + id: 0, + pipeline: [{ name: "primary" }, { name: "secondary", delay: 5000 }], + }, +]; + +const buildCommon = (jobRequest) => { + const { testName } = jobRequest; + if (!testName) throw Error("'testName' must be provided!"); + const command = [baseCommand, suiteEntry, `test=${testName}`]; + + // Apply Common Flags + command.push("isRetry=false"); + + // Return new request + return { ...jobRequest, command }; +}; + +const buildSingle = (jobReq) => jobReq; + +const buildMarker = (jobReq) => {}; + +const buildProject = (jobReq) => {}; + +const pipelineMaxLife = (testName) => { + const pipelines = pipelineMapping + .filter((m) => m.pipeline.find((t) => t.name === testName)) + .map((m) => m.pipeline); + return Math.max(pipelines.map((p) => p.length)) + 1; +}; + +const buildCompound = (jobReq, socketId) => { + const { testName, command } = jobReq; + const pipelineTriggers = jobReq.pipelineTriggers; + if (pipelineTriggers) command.push(`pipelineTriggers=${pipelineTriggers}`); + command.push(`pipelineDashboardSocket=${socketId}`); + return { ...jobReq, command }; +}; + +function nextCompound(previousTest) {} + +export default function jobBuilder(jobRequest, id) { + const jobReq = buildCommon(jobRequest, id); + switch (jobRequest.type) { + case "single": + return buildSingle(jobReq); + case "marker": + return buildMarker(jobReq); + case "project": + return buildProject(jobReq); + case "compound": + return buildCompound(jobReq, id); + default: + throw Error("No Job Request Type Specified!"); + } +} diff --git a/lib/jobs/job-executor.js b/lib/jobs/job-executor.js new file mode 100644 index 0000000..e558991 --- /dev/null +++ b/lib/jobs/job-executor.js @@ -0,0 +1,9 @@ +import Executor from "../sockets/clients/Executor.js"; + +const args = process.argv.slice(2); +const url = args[0]; +const jobId = args[1]; +const command = args.slice(2); +const job = { id: jobId, command }; +const exec = new Executor(url, job, command); +exec.runJob(); diff --git a/lib/jobs/k8s/k8s-bypass.js b/lib/jobs/k8s/k8s-bypass.js new file mode 100644 index 0000000..09e0a0c --- /dev/null +++ b/lib/jobs/k8s/k8s-bypass.js @@ -0,0 +1,14 @@ +import { INFO, ERR, OK, VERB } from "../../util/logging.js"; +import cp from "node:child_process"; + +const jobStr = process.argv.slice(2)[0]; +const job = JSON.parse(jobStr); +const { command } = job.spec.template.spec.containers[0]; +INFO("EXEC", "Internal Executor Starting!"); +cp.exec(command, (error, stdout, stderr) => { + if (error) ERR("EXEC", error); + //if(stdout) VERB("EXEC-STDOUT", stdout); + //if(stderr) VERB("EXEC-STDERR", stderr); + OK("EXEC", "Internal Executor Finished!"); + process.exit(error ? 1 : 0); +}); diff --git a/lib/jobs/k8s/k8s-job.json b/lib/jobs/k8s/k8s-job.json new file mode 100644 index 0000000..01d8d4c --- /dev/null +++ b/lib/jobs/k8s/k8s-job.json @@ -0,0 +1,33 @@ +{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "qltr-job-test-suite-1" + }, + "spec": { + "template": { + "spec": { + "containers": [ + { + "resources": { + "requests": { + "memory": "64MI", + "cpu": "250m" + }, + "limits": { + "memory": "128MI", + "cpu": "500m" + } + }, + "name": "qltr-job-test-suite-1", + "image": "node", + "imagePullPolicy": "Always", + "command": ["node", "--version"] + } + ], + "restartPolicy": "Never" + } + }, + "backoffLimit": 4 + } +} diff --git a/lib/jobs/k8s/kubernetes.js b/lib/jobs/k8s/kubernetes.js new file mode 100644 index 0000000..4c0756f --- /dev/null +++ b/lib/jobs/k8s/kubernetes.js @@ -0,0 +1,84 @@ +import cp from "child_process"; +import fs from "fs"; +import path from "path"; + +const internalDeploy = process.env.INTERNAL_DEPLOY === "true"; +const executorUrl = process.env.EXECUTOR_URL; +const executorScriptOnly = process.env.EXECUTOR_SCRIPT_ONLY === "true"; +const executorBin = + process.env.EXECUTOR_BIN ?? `qltr-executor${executorScriptOnly ? ".js" : ""}`; + +const qualiteerUrl = + process.env.QUALITEER_URL ?? "file:///home/runner/Qualiteer/bin/executor"; + +const kubCmd = "kubectl apply -f"; +const jobsDir = "jobs/"; +const defaults = JSON.parse( + fs.readFileSync(path.resolve("./lib/jobs/k8s/k8s-job.json")) +); + +const wrapCommand = (jobId, command) => { + const bin = executorScriptOnly + ? `node ${executorBin}` + : `chmod +x ${executorBin} && ./${executorBin}`; + const cmd = command.map((arg) => JSON.stringify(arg)); + const curlCmd = `curl -o qltr-executor ${executorUrl} && ${bin} ${qualiteerUrl} ${jobId} ${cmd.join( + " " + )}`; + return curlCmd; +}; + +const createFile = (job) => { + const { name } = job.metadata; + const jobsPath = path.resolve(jobsDir); + if (!fs.existsSync(jobsPath)) fs.mkdirSync(jobsPath); + const filePath = path.resolve(jobsDir, `${name}.json`); + fs.writeFileSync(filePath, JSON.stringify(job)); + return filePath; +}; + +const applyFileInternally = (filePath) => { + const job = fs.readFileSync(filePath, { encoding: "utf8" }); + cp.fork(path.resolve("./lib/jobs/k8s/k8s-bypass.js"), [job]); +}; + +const applyFile = async (filePath) => { + const command = `${kubCmd} ${filePath}`; + return new Promise((res, rej) => + cp.exec(command, (err, stdout, stderr) => (err && rej(err)) || res(stdout)) + ); +}; + +const deleteFile = (filePath) => fs.unlinkSync(filePath); + +const jobBuilder = (jobRequest) => { + const { resources, name, image, command, id: jobId } = jobRequest; + + // Safety Checks + if (!jobId) throw Error("'jobId' required!"); + if (!name) throw Error("'name' required!"); + if (!command) throw Error("'command' required!"); + if (!image) throw Error("'image' required!"); + + if (!Array.isArray(command)) throw Error("'command' must be an array!"); + + // Apply configuration + const job = { ...defaults }; + job.metadata.name = `qltr-${name}-${jobId}`; + const container = job.spec.template.spec.containers[0]; + container.name = job.metadata.name; + container.command = wrapCommand(jobId, command); + container.image = JSON.stringify(image); + + // Apply resources + job.resources = { ...job.resources, ...resources }; + return job; +}; + +export default async function createJob(jobRequest) { + const job = jobBuilder(jobRequest); + const filePath = createFile(job); + if (!internalDeploy) await applyFile(filePath); + else await applyFileInternally(filePath); + deleteFile(filePath); +} diff --git a/src/views/jobs/JobBox.jsx b/src/views/jobs/JobBox.jsx new file mode 100644 index 0000000..bb97db4 --- /dev/null +++ b/src/views/jobs/JobBox.jsx @@ -0,0 +1,67 @@ +import React, { useState, useContext } from "react"; +import StoreContext from "../../ctx/StoreContext.jsx"; +import JobContext, { jobStatus } from "../../ctx/JobContext.jsx"; + +import Accordion from "@mui/material/Accordion"; +import AccordionDetails from "@mui/material/AccordionDetails"; +import AccordionSummary from "@mui/material/AccordionSummary"; +import Typography from "@mui/material/Typography"; + +import IconButton from "@mui/material/IconButton"; +import CheckIcon from "@mui/icons-material/Check"; +import ClearIcon from "@mui/icons-material/Clear"; +import ViewColumnIcon from "@mui/icons-material/ViewColumn"; +import PendingIcon from "@mui/icons-material/Pending"; +import VisibilityIcon from "@mui/icons-material/Visibility"; +import DoNotDisturbIcon from "@mui/icons-material/DoNotDisturb"; + +import Box from "@mui/material/Box"; +import Stack from "@mui/material/Stack"; + +export default function JobBox(props) { + const { job } = props; + + const { name, status } = job; + + function jobIcon() { + switch (status) { + case jobStatus.OK: + return ; + case jobStatus.ERROR: + return ; + case jobStatus.PENDING: + return ; + case jobStatus.ACTIVE: + return ; + case jobStatus.CANCELED: + return ; + case jobStatus.QUEUED: + return ; + default: + return ; + } + } + + return ( + + + + {name} + + + + {jobIcon()} + + + + + ); +} diff --git a/src/views/jobs/JobLogView.jsx b/src/views/jobs/JobLogView.jsx new file mode 100644 index 0000000..65b4d5c --- /dev/null +++ b/src/views/jobs/JobLogView.jsx @@ -0,0 +1,67 @@ +import { useContext, useState, useEffect } from "react"; +import JobContext from "../../ctx/JobContext.jsx"; + +import Box from "@mui/material/Box"; +import Typography from "@mui/material/Typography"; +import Skeleton from "@mui/material/Skeleton"; +import Stack from "@mui/material/Stack"; + +export default function JobLogView(props) { + const { log } = props; + + const LoadingDot = () => ( + + ); + + return ( + + {log.map((l, i) => ( + + + + {i + 1} + + + {" "} + {l} + + + + ))} + + + + + + + ); +} diff --git a/src/views/jobs/JobTestSelector.jsx b/src/views/jobs/JobTestSelector.jsx new file mode 100644 index 0000000..cf6dbce --- /dev/null +++ b/src/views/jobs/JobTestSelector.jsx @@ -0,0 +1,49 @@ +import { useState, useEffect } from "react"; +import Box from "@mui/material/Box"; +import List from "@mui/material/List"; +import ListItem from "@mui/material/ListItem"; +import ListItemButton from "@mui/material/ListItemButton"; +import ListItemText from "@mui/material/ListItemText"; +import Checkbox from "@mui/material/Checkbox"; + +export default function JobTestSelector(props) { + const { availableTests, queued, setQueued } = props; + + useEffect(() => {}, [availableTests]); + + const queueTest = (test) => () => { + const q = [...queued]; + const testIndex = q.indexOf(test); + if (testIndex === -1) q.push(test); + else q.splice(testIndex, 1); + setQueued(q); + }; + + return ( + + + {availableTests.map((v, i) => ( + + } + disablePadding + onClick={queueTest(v)} + > + + + {v.class}#{v.name} + + } + style={{ wordBreak: "break-word" }} + /> + + + ))} + + + ); +} diff --git a/src/views/jobs/JobView.jsx b/src/views/jobs/JobView.jsx new file mode 100644 index 0000000..18030c8 --- /dev/null +++ b/src/views/jobs/JobView.jsx @@ -0,0 +1,69 @@ +import React, { useContext, useState, useEffect } from "react"; +import { useNavigate } from "react-router-dom"; +import JobContext from "../../ctx/JobContext.jsx"; +import Box from "@mui/material/Box"; +import AppBar from "@mui/material/AppBar"; +import Toolbar from "@mui/material/Toolbar"; +import Button from "@mui/material/Button"; +import Typography from "@mui/material/Typography"; + +import JobLogView from "./JobLogView.jsx"; + +export default function JobView(props) { + const navigate = useNavigate(); + const { state: jobState } = useContext(JobContext); + const { job: initJob } = props; + const [job, setJob] = useState({ log: [initJob.name] }); + + function retryJob() {} + + function downloadLog() {} + + function navigateToJobs() { + navigate("/jobs"); + } + + function onLog(d) { + const j = { ...job }; + j.log.push(d); + setJob(j); + } + + return ( + + + + + + + + {initJob.name} + + + + + + + + + + + + ); +} diff --git a/src/views/jobs/Jobs.jsx b/src/views/jobs/Jobs.jsx new file mode 100644 index 0000000..bd7935a --- /dev/null +++ b/src/views/jobs/Jobs.jsx @@ -0,0 +1,119 @@ +import { useState, useContext, useEffect } from "react"; +import { useLocation } from "react-router-dom"; + +import StoreContext from "../../ctx/StoreContext.jsx"; +import JobContext from "../../ctx/JobContext.jsx"; +import JobBox from "./JobBox.jsx"; +import JobTestSelector from "./JobTestSelector.jsx"; +import JobView from "./JobView.jsx"; + +import Button from "@mui/material/Button"; +import Dialog from "@mui/material/Dialog"; +import DialogActions from "@mui/material/DialogActions"; +import DialogContent from "@mui/material/DialogContent"; +import Toolbar from "@mui/material/Toolbar"; +import DialogTitle from "@mui/material/DialogTitle"; + +import ClickAwayListener from "@mui/material/ClickAwayListener"; +import SpeedDial from "@mui/material/SpeedDial"; +import SpeedDialAction from "@mui/material/SpeedDialAction"; +import SpeedDialIcon from "@mui/material/SpeedDialIcon"; + +import PageviewIcon from "@mui/icons-material/Pageview"; +import ViewColumnIcon from "@mui/icons-material/ViewColumn"; +import ViewCarouselIcon from "@mui/icons-material/ViewCarousel"; + +export default function Jobs() { + const { + state: jobState, + dispatch: jobDispatch, + jobBuilder, + } = useContext(JobContext); + const location = useLocation(); + const { state: store, updateStore } = useContext(StoreContext); + const [quickOpen, setQuickOpen] = useState(false); + const [jobDialogOpen, setJobDialogOpen] = useState(false); + + const actions = [ + { name: "Suite", icon: }, + { name: "Compound", icon: }, + { name: "Manual", icon: }, + ]; + + const quickOpenClick = (e) => { + e.preventDefault(); + e.stopPropagation(); + if (!store.simplifiedControls) return setQuickOpen(!quickOpen); + setJobDialogOpen(true); + }; + const quickOpenClose = () => setQuickOpen(false); + + const handleClickOpen = () => setJobDialogOpen(true); + + const [queued, setQueued] = useState([]); + useEffect(() => {}, [jobState.jobs, location]); + + const handleClose = (confirmed) => () => { + setJobDialogOpen(false); + if (!confirmed) return; + jobBuilder(queued); + }; + + return ( +
+ {location.hash === "" && + jobState.jobs.map((v, i) => ( + + + + ))} + {jobState.jobs.find((job) => job.name === location.hash.slice(1)) && ( + job.name === location.hash.slice(1))} + /> + )} + + + + New Job + + Some Selectors + + + + + + + + + + } + onClick={quickOpenClick} + open={quickOpen} + > + {actions.map((action) => ( + + ))} + + +
+ ); +}