Revamp Job flow

lib/server/database/queries/alerting.js

@@ -0,0 +1,56 @@
+import pg from "../postgres.js";
+import { silencedMock } from "../mocks/alerting-mock.js";
+import moment from "moment";
+// Imports
+import {
+  insertQuery,
+  selectWhereAnyQuery,
+  updateWhereAnyQuery,
+  deleteQuery,
+} from "../pg-query.js";
+// Constants
+const table = "alerting";
+const PG_DISABLED = process.env.POSTGRES_DISABLED;
+
+export const upsertAlertSilence = async (silence) => {
+  const {
+    id,
+    name,
+    class: className,
+    method,
+    expires: duration,
+    keepExpires,
+  } = silence;
+  const { h, m } = duration;
+  const expires = moment().add(h, "hours").add(m, "minutes").utc().format();
+  const entry = {
+    name,
+    class: className,
+    method,
+    expires: keepExpires ? undefined : expires,
+  };
+  const asUpdate = {};
+  for (var k of Object.keys(entry))
+    asUpdate[k] = entry[k] === "*" ? null : entry[k];
+  var query = id
+    ? updateWhereAnyQuery(table, asUpdate, { id })
+    : insertQuery(table, entry);
+  return pg.query(query);
+};
+
+export const deleteAlertSilence = async (silence) => {
+  const { id } = silence;
+  const query = deleteQuery(table, { id });
+  return pg.query(query);
+};
+
+// Queries
+export const getSilencedTests = async () => {
+  if (PG_DISABLED) return silencedMock();
+  const query = `SELECT * from ${table}`;
+  const silenced = await pg.query(query);
+  silenced.forEach((t, i) => {
+    for (var k of Object.keys(t)) silenced[i][k] = t[k] === null ? "*" : t[k];
+  });
+  return silenced;
+};

lib/server/database/queries/catalog.js

@@ -0,0 +1,121 @@
+import pg from "../postgres.js";
+// Imports
+import {
+  insertQuery,
+  selectWhereAnyQuery,
+  onConflictUpdate,
+} from "../pg-query.js";
+import { WARN } from "../../util/logging.js";
+
+import getFilteredTags from "../tags.js";
+import getDelay from "../delays.js";
+// Constants
+const table = "catalog";
+const PG_DISABLED = process.env.POSTGRES_DISABLED;
+import { testsMock, mappingsMock } from "../mocks/catalog-mock.js";
+// Queries
+
+export const removeDroppedTests = async (testNames) => {
+  // BUG: After dropping a test, the id jumps ridiculously high
+  const pgNames = testNames.map((tn) => `'${tn}'`).join(",");
+  const query = `DELETE FROM catalog as x where x.name not in (${pgNames});`;
+  return pg.query(query);
+};
+
+export const getTest = async (name) => {
+  const query = selectWhereAnyQuery(table, { name });
+  const results = await pg.query(query);
+  if (results.length > 1)
+    WARN("CATALOG", `More than 1 test found for '${name}'`);
+  return results[0];
+};
+
+export const getTests = async () => {
+  if (PG_DISABLED) return testsMock();
+  const query = `SELECT * from ${table}`;
+  return pg.query(query);
+};
+
+export const getPipelineMappings = async () => {
+  if (PG_DISABLED) return mappingsMock();
+  const query = `SELECT * from ${table} WHERE pipeline`;
+  const tests = await pg.query(query);
+  const mappings = [];
+  var newTrigger;
+  for (var test of tests) {
+    if (test.triggers) continue;
+    const { name, delay: delayStr } = test;
+    var triggerStack = [{ name, delay: getDelay(delayStr), delayStr }];
+    newTrigger = { name, delayStr };
+    while (
+      (newTrigger = tests.find(
+        (te) => te.triggers && te.triggers.includes(newTrigger.name)
+      )) !== null
+    ) {
+      if (!newTrigger) break;
+      triggerStack.push({
+        name: newTrigger.name,
+        delay: getDelay(newTrigger.delay),
+        delayStr: newTrigger.delay,
+      });
+    }
+    mappings.push(triggerStack.reverse());
+  }
+  return mappings;
+};
+
+export const getProjects = async () => {
+  if (PG_DISABLED) {
+    const tests = testsMock();
+  }
+};
+
+export const truncateTests = async () => {
+  if (PG_DISABLED) return console.log(`Would truncate table ${table}`);
+  const query = `TRUNCATE ${table} RESTART IDENTITY CASCADE;`;
+  return await pg.query(query);
+};
+
+export const upsertTest = async (test) => {
+  if (PG_DISABLED) return console.log("Would insert test", test);
+  const {
+    name,
+    class: className,
+    image,
+    path,
+    description,
+    type,
+    created,
+    mergeRequest,
+    tags,
+  } = test;
+
+  const filteredTags = getFilteredTags(tags);
+
+  const env =
+    filteredTags.ignore && filteredTags.env
+      ? filteredTags.env.filter((e) => !filteredTags.ignore.includes(e))
+      : filteredTags.env;
+  const catalogEntry = {
+    name,
+    class: className,
+    image,
+    path,
+    description: description ? description : null,
+    type,
+    created,
+    mr: mergeRequest,
+    tags,
+    crons: filteredTags.crons,
+    env,
+    regions: filteredTags.regions,
+    triggers: filteredTags.triggers,
+    pipeline: filteredTags.pipeline ? true : false,
+    coverage: filteredTags.coverage,
+    projects: filteredTags.projects,
+    delay: filteredTags.delay ? filteredTags.delay[0] : null,
+  };
+  const query =
+    insertQuery(table, catalogEntry) + onConflictUpdate(["name"], catalogEntry);
+  return await pg.query(query);
+};

lib/server/database/queries/results.js

@@ -0,0 +1,93 @@
+import pg from "../postgres.js";
+import { failingMock } from "../mocks/results-mock.js";
+// Imports
+import {
+  insertQuery,
+  selectWhereAnyQuery,
+  selectWhereAllQuery,
+  updateWhereAnyQuery,
+} from "../pg-query.js";
+// Constants
+const table = "results";
+const recentResultsMax = 5;
+const PG_DISABLED = process.env.POSTGRES_DISABLED;
+
+// Queries
+export const insertTestResult = (testResult) => {
+  const {
+    name,
+    class: className,
+    method,
+    env,
+    timestamp,
+    triage,
+    failed,
+    message,
+    screenshot,
+    console: cs,
+  } = testResult;
+
+  var query = insertQuery(table, {
+    name,
+    class: className,
+    method,
+    env,
+    timestamp,
+    triage,
+    failed,
+    message,
+    screenshot,
+    console: cs,
+  });
+
+  query += "\n RETURNING *";
+  return pg.query(query);
+};
+
+export const getCurrentlyFailing = async () => {
+  if (PG_DISABLED) return failingMock();
+  /* This can probably be changed into a super query, but perhaps faster/smaller */
+  const recent = `SELECT * FROM ${table} WHERE (timestamp BETWEEN NOW() - INTERVAL '24 HOURS' AND NOW()) AND NOT(failed AND triage)`;
+  const slimCatalog = `SELECT name, crons, class, type, pipeline, env AS enabled_env FROM catalog`;
+  const failing = `SELECT * FROM recent INNER JOIN slim_catalog USING(name) WHERE timestamp = (SELECT MAX(timestamp) FROM recent r2 WHERE recent.name = r2.name) AND failed`;
+  const applicableFailing = `SELECT name, count(*) as fails FROM recent WHERE recent.name IN (SELECT name FROM failing) GROUP BY name`;
+  /*const runHistory = `SELECT name, timestamp, failed  FROM (SELECT *, ROW_NUMBER() OVER(PARTITION BY name ORDER BY timestamp) as n
+				FROM ${table} WHERE name IN (SELECT name FROM failing)) as ord WHERE n <= ${recentResultsMax} ORDER BY name DESC`;*/
+  const runHistory = `SELECT name, timestamp, failed  FROM results WHERE NOT triage AND name IN (SELECT name FROM failing) ORDER BY timestamp DESC LIMIT ${recentResultsMax}`;
+  // const recentQuery = pg.query(recent);
+  const failingQuery = pg.query(
+    `WITH recent as (${recent}), slim_catalog as (${slimCatalog}) ${failing}`
+  );
+  const applicableQuery = pg.query(
+    `WITH recent as (${recent}), slim_catalog as (${slimCatalog}), failing as (${failing}) ${applicableFailing}`
+  );
+  const historyQuery = pg.query(
+    `WITH recent as (${recent}), slim_catalog as (${slimCatalog}), failing as (${failing}) ${runHistory}`
+  );
+
+  const [currentlyFailing, applicableFails, failHistory] = await Promise.all([
+    failingQuery,
+    applicableQuery,
+    historyQuery,
+  ]);
+  for (var i = 0; i < currentlyFailing.length; i++) {
+    currentlyFailing[i].dailyFails = parseInt(
+      applicableFails.find((af) => af.name === currentlyFailing[i].name).fails
+    );
+    currentlyFailing[i].recentResults = [];
+    currentlyFailing[i].enabledEnv = currentlyFailing[i].enabled_env;
+    currentlyFailing[i].isPipeline = currentlyFailing[i].pipeline;
+    delete currentlyFailing[i].enabled_env;
+    delete currentlyFailing[i].pipeline;
+    for (var fh of failHistory) {
+      if (fh.name !== currentlyFailing[i].name) continue;
+      currentlyFailing[i].recentResults.push(fh);
+    }
+  }
+  return currentlyFailing;
+};
+
+export const ignoreResult = async ({ id }) => {
+  const query = updateWhereAnyQuery(table, { failed: false }, { id });
+  return pg.query(query);
+};