Merge branch 'ep/jul26/NoSessionFileAccess' into 'master'

Removed express-session middleware & Updated to use MongoDB instead of json files

See merge request Dunemask/nubian!1
This commit is contained in:
Elijah Dunemask 2021-07-31 17:57:09 +00:00
commit 3cdab63b5f
9 changed files with 285 additions and 387 deletions

View file

@ -24,9 +24,12 @@
"body-parser": "^1.19.0",
"cors": "^2.8.5",
"express": "^4.17.1",
"express-bearer-token": "^2.4.0",
"express-session": "^1.17.2",
"install": "^0.13.0",
"lodash": "^4.17.21",
"mongodb": "^4.0.1",
"mongoose": "^5.13.3",
"multer": "^1.4.2",
"path": "^0.12.7",
"rimraf": "^3.0.2",

View file

@ -6,149 +6,146 @@ const {
readdirSync: readdir,
unlinkSync: fremove,
} = require("fs");
const AdmZip = require("adm-zip");
//Local Imports
const Pharaoh = require("../egypt/pharaoh");
const desertConfig = require("../egypt/desert_config.json");
const config = require("../config.json");
// Create the desert if it doesn't exist
if (!fexists(desertConfig.desertPath)) mkdir(desertConfig.desertPath);
//Constants
const fileStorage = new Pharaoh(
resolvePath(desertConfig.desertPath),
desertConfig.schema
);
const mongoose = require("mongoose");
mongoose.connect(`${config.Storage.NubianDatabase}/nubian`, {
useUnifiedTopology: true,
useNewUrlParser: true,
});
const users = require("../schemas/user");
const files = require("../schemas/file");
const zipDir = resolvePath(config.Storage.ZipPath);
function addFile(fileData) {
fileStorage.addEntry(fileData.fileUuid, "files", fileData);
fileStorage.updateEntry(fileData.owner, "uuid", (entry) => {
if (entry == null) this.createUser(ownerUuid);
entry.owned.push(fileData.fileUuid);
return entry;
});
}
function updateReferenceOnDelete(fileData) {
if (fileData == null) return;
//Update Users Shared List (edit)
fileData.edit.forEach((user) => {
fileStorage.updateEntry(user, "uuid", (entry) => {
if (entry == null) return;
entry.shared.splice(entry.shared.indexOf(fileData.fileUuid), 1);
return entry;
});
});
//Update Users Shared List (view)
fileData.view.forEach((user) => {
fileStorage.updateEntry(user, "uuid", (entry) => {
if (entry == null) return;
entry.shared.splice(entry.shared.indexOf(fileData.fileUuid), 1);
return entry;
});
});
fileStorage.updateEntry(fileData.owner, "uuid", (entry) => {
if (entry == null) return;
entry.owned.splice(entry.owned.indexOf(fileData.fileUuid), 1);
return entry;
function authorizedToView(userId, file) {
if (`${file.owner}` == (userId = `${userId}`) || file.public) return true;
if (file.view.includes(userId) || file.edit.includes(userId)) return true;
return false;
}
function authorizedToEdit(userId, file) {
return `${file.owner}` == `${userId}` || file.edit.includes(`${userId}`);
}
function getFile(userId, fileId) {
return files.findOne({ _id: fileId }).then((file) => {
if (authorizedToView(userId, file)) return file;
return null;
});
}
function deleteFile(fileUuid) {
const fileData = fileStorage.deleteEntry(fileUuid, "files");
return fileData;
function deleteFiles(userId, fileIds) {
return files.find({ _id: { $in: fileIds } }).then((databaseFiles) => {
var failed = [];
filesByOwner = {};
databaseFiles.forEach((file) => {
if (file.owner in filesByOwner) filesByOwner[file.owner].push(file);
else filesByOwner[file.owner] = [file];
});
for (var owner in filesByOwner) {
var deleteSize = 0;
for (var i = filesByOwner[owner].length - 1; i >= 0; i--) {
let file = filesByOwner[owner][i];
if (!authorizedToEdit(userId, file)) {
failed.push(`${file._id}`);
filesByOwner[owner].splice(i, 1);
} else deleteSize += file.size;
}
function getFile(fileUuid) {
return fileStorage.loadEntry(fileUuid, "files");
users
.updateOne(
{ _id: owner, usedStorage: { $gte: deleteSize } },
{
$pull: { owned: { $in: filesByOwner[owner] } },
$inc: {
usedStorage: -deleteSize,
},
}
function modifyFile(fileUuid, cb) {
fileStorage.updateEntry(fileUuid, "files", cb);
)
.exec();
fileIds = fileIds.filter((fileId) => !failed.includes(fileId));
rfiles = databaseFiles.filter((file) => fileIds.includes(`${file._id}`));
files
.deleteMany({
_id: { $in: fileIds },
})
.exec();
return { files: rfiles, failed };
}
function zipFiles(files) {}
function createUser(uuid) {
const userData = {
});
}
function publicfyFiles(userId, targetFiles) {
return files.find({ _id: { $in: targetFiles } }).then((databaseFiles) => {
var failed = [];
databaseFiles.forEach((file) => {
if (!authorizedToEdit(userId, file)) failed.push(`${file._id}`);
else files.updateOne({ _id: file._id }, { public: !file.public }).exec();
});
return failed;
});
}
function createUser(cairoUuid) {
return users.create(
{
cairoUuid,
usedStorage: 0,
storage: config.Storage.UserStorageSize * config.Storage.UserStorageUnit,
owned: [],
shared: [],
storage: config.Storage.UserStorageSize * config.Storage.UserStorageUnit,
usedStorage: 0,
};
fileStorage.updateEntry(uuid, "uuid", (entry) => {
if (entry != null) return;
return userData;
});
return userData;
},
(err, result) => {
if (err) return err;
return result;
}
function updateUser(ownerUuid, cb) {
fileStorage.updateEntry(ownerUuid, "uuid", cb);
);
}
function getOwnedFileList(ownerUuid) {
const owner = fileStorage.loadEntry(ownerUuid, "uuid");
if (owner == null) return;
return owner.owned;
}
function getSharedFileList(ownerUuid) {
const owner = fileStorage.loadEntry(ownerUuid, "uuid");
if (owner == null) return;
return owner.shared;
}
function setMaxStorage(ownerUuid, newMax) {
fileStorage.updateEntry(ownerUuid, "uuid", (entry) => {
if (entry == null) this.createUser(ownerUuid);
entry.storage = newMax;
return entry;
function getUserByCairoUuid(cairoUuid) {
return users.findOne({ cairoUuid: cairoUuid }, (err, result) => {
if (result == null) createUser(cairoUuid);
if (err) console.error(err);
});
}
function modifyUsedStorage(ownerUuid, cb) {
fileStorage.updateEntry(ownerUuid, "uuid", (entry) => {
if (entry == null) entry = this.createUser(ownerUuid);
const maxStorage = entry.storage;
const newUsed =
cb(entry.storage, entry.usedStorage ?? 0) ?? entry.usedStorage;
if (newUsed > maxStorage)
throw new Error("New Size Exceeds User Max Storage!");
entry.usedStorage = newUsed;
return entry;
function getUserById(userId) {
return users.findOne({ _id: userId }, (err, result) => {
if (result == null) createUser(cairoUuid);
if (err) console.error(err);
});
}
async function buildZip(ownerUuid, paths, zipUuid) {
//Create directory and build zip with adm zip
const zipPath = resolvePath(zipDir, `${zipUuid}.zip`);
var zip = {
owner: ownerUuid,
path: zipPath,
building: true,
};
fileStorage.addEntry(zipUuid, "zips", zip);
createZip(paths, zipPath).then(() => {
fileStorage.updateEntry(zipUuid, "zips", (entry) => {
if (entry == null) return;
entry.exp = Date.now() + config.Storage.ZipClickExpire;
delete entry.building;
return entry;
function uploadFile(userId, fileData) {
return getUserById(userId).then((user) => {
if (user.usedStorage + fileData.size > user.storage) return null;
return users
.updateOne({ _id: userId }, { $inc: { usedStorage: fileData.size } })
.then(() => createFile(user._id, fileData))
.then((file) => {
if (file == null) return null;
users.updateOne({ _id: userId }, { $push: { owned: file._id } }).then();
return file;
});
});
}
async function createZip(paths, zipPath) {
if (!fexists(zipDir)) mkdir(zipDir);
let zipFile = new AdmZip();
paths.forEach((filePath) => {
zipFile.addLocalFile(filePath);
function createFile(userId, fileData) {
return files.create({
path: fileData.path,
owner: userId,
name: fileData.originalname,
date: fileData.filename.substring(0, fileData.filename.indexOf("-")),
size: fileData.size,
public: false,
edit: [],
view: [],
});
setTimeout(() => zipFile.writeZip(zipPath), 0);
}
function getZipPath(ownerUuid, zipUuid) {
var zipPath, building;
fileStorage.updateEntry(zipUuid, "zips", (entry) => {
if (entry == null || (building = entry.building)) return;
entry.exp = Date.now() + config.Storage.ZipDownloadExpire;
zipPath = entry.path;
return entry;
});
if (building === true) return building;
if (zipPath == null || !fexists(zipPath)) return;
return zipPath;
}
function cleanZips() {
console.log("Would clean zips");
return;
var zipUuid;
const time = Date.now();
readdir(zipDir).forEach((file) => {
@ -164,16 +161,12 @@ function cleanZips() {
});
}
module.exports = {
addFile,
updateReferenceOnDelete,
deleteFile,
deleteFiles,
getFile,
modifyFile,
createUser,
updateUser,
getOwnedFileList,
getSharedFileList,
setMaxStorage,
modifyUsedStorage,
getUserById,
getUserByCairoUuid,
publicfyFiles,
uploadFile,
cleanZips,
};

View file

@ -6,15 +6,13 @@ const multer = require("multer");
const config = require("../config.json");
//Multer Configs
const userUploadStorage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, userUploadDestination(req));
},
destination: (req, file, cb) => cb(null, userUploadDestination(req.user._id)),
filename: (req, file, cb) => {
const n = file.originalname.replaceAll(" ", "_");
const fileName = `${Date.now()}-${n}`;
req.on("aborted", () => {
cancelUpload(resolvePath(userUploadDestination(req), fileName));
});
req.on("aborted", () =>
cancelUpload(resolvePath(userUploadDestination(req.user._id), fileName))
);
cb(null, fileName);
},
});
@ -23,10 +21,10 @@ const userUpload = multer({
}).single("user-selected-file");
//Helper Methods
function userUploadDestination(req) {
function userUploadDestination(user_id) {
if (!fs.existsSync(resolvePath(config.Storage.UploadPath)))
fs.mkdirSync(resolvePath(config.Storage.UploadPath));
const destination = resolvePath(config.Storage.UploadPath, req.session.uuid);
const destination = resolvePath(config.Storage.UploadPath, `${user_id}`);
if (!fs.existsSync(destination)) fs.mkdirSync(destination);
return destination;
}

View file

@ -5,16 +5,11 @@ const uuidGen = require("uuid-with-v6").v6;
//Local Imports
const storage = require("./storage");
const config = require("../config.json");
/**
* Generates a new uuid.v6() and reverses the uuid so the timestamp is at the end
* This should provide an additional layer of "randomness" and decrease the chances
* of duplicate uuid's being generated.
* This is reversed to force the custom DB to expand faster at first rather than
* later when there are lots of entries.
*/
function generateUuid() {
return [...uuidGen()].reverse().join("");
function load(uid) {
return storage.getUserByCairoUuid(uid);
}
/**
* Create a user with a uuid (should use Dunestorm API to login)
*/
@ -25,61 +20,7 @@ function createUser(uuid) {
* Creates file entry given aspects of a file updated
*/
function uploadFile(uuid, fileData) {
const fileUuid = generateUuid();
var sizeAccepted;
storage.modifyUsedStorage(uuid, (max, used) => {
const oldUsed = used;
used += fileData.size;
if ((sizeAccepted = used <= max)) return used;
});
if (!sizeAccepted) return;
const file = {
fileUuid,
path: fileData.path,
owner: uuid,
name: fileData.originalname,
date: fileData.filename.substring(0, fileData.filename.indexOf("-")),
size: fileData.size,
public: false,
edit: [],
view: [],
};
storage.addFile(file);
return file;
}
/**
TODO: ASYNCIFY?
Removes user references to files that are being deleted
*/
function removeEntryLinks(files) {
for (var o in files.owner) {
storage.updateUser(o, (entry) => {
if (entry == null) return;
files.owner[o].forEach((file) => {
entry.owned.splice(entry.owned.indexOf(file.fileUuid), 1);
entry.usedStorage -= file.size;
});
return entry;
});
}
for (var user in files.edit) {
storage.updateUser(user, (entry) => {
if (entry == null) return;
files.edit[user].forEach((file) => {
entry.edit.splice(entry.edit.indexOf(file), 1);
});
return entry;
});
}
for (var user in files.view) {
storage.updateUser(user, (entry) => {
if (entry == null) return;
files.view[user].forEach((file) => {
entry.view.splice(entry.view.indexOf(file), 1);
});
return entry;
});
}
return storage.uploadFile(uuid, fileData);
}
/**
* Deletes files.
@ -87,93 +28,36 @@ function removeEntryLinks(files) {
* Sorts files by user before deleting to speed up reference updates
*/
function deleteFiles(uuid, targetFiles) {
var deleteFails = [];
//Sort files by fileuuid to remove entries from the various users
var filesSortedByUser = {
owner: {},
edit: {},
view: {},
};
targetFiles.forEach((targetFile) => {
storage.modifyFile(targetFile, (entry, deleteEntry) => {
if (!authorizedToEditFile(uuid, entry)) return;
//Add owner and file size to the update object
if (filesSortedByUser.owner[entry.owner] == null)
filesSortedByUser.owner[entry.owner] = [];
filesSortedByUser.owner[entry.owner].push({
fileUuid: targetFile,
size: entry.size,
});
//Add edit members to the edit update
for (var id of entry.edit) {
if (entry.edit[id] == null) entry.edit[id] = [];
entry.edit[id].push(targetFile);
}
//Add view members to the view update
for (var id of entry.view) {
if (entry.view[id] == null) entry.view[id] = [];
entry.view[id].push(targetFile);
}
//Throw stuff in a catch, we need to make sure we delete the file physically
return storage.deleteFiles(uuid, targetFiles).then((deleteData) => {
var files = deleteData.files;
var deleteFails = deleteData.failed;
files.forEach((file) => {
try {
deleteEntry(entry);
fremove(entry.path);
fremove(file.path);
} catch (e) {
console.error("Error Deleting File", entry.name, "\nPath:", entry.path);
deleteFails.push(targetFile);
console.error("Error Deleting File", file.name, "\nPath:", file.path);
deleteFails.push(`${file._id}`);
}
});
});
//Updates user entries using the filesSortedByUser
removeEntryLinks(filesSortedByUser);
//Return the new used storage to update the database
return deleteFails.length > 0 && deleteFails;
}
/**
* Checks that a user is authourized to view the file and then
* Returns the physical filePath of a desired file (uses entry to find path)
*/
function getFilePath(uuid, targetFile) {
const fileData = storage.getFile(targetFile);
if (!authorizedToViewFile(uuid, fileData)) return;
if (fexists(fileData.path)) return fileData.path;
});
}
/**
* Returns a list of fileUuids that the user owns
*/
function getOwnedFiles(uuid) {
const fileList = storage.getOwnedFileList(uuid);
if (fileList == null) return [];
function getOwnedFiles(userId) {
return storage.getUserById(userId).then((user) => {
const fileList = user.owned;
var files = new Array(fileList.length);
fileList.forEach((file, i) => {
files[i] = storage.getFile(file);
fileList.forEach(
(file, i) =>
(files[i] = new Promise((resolve, reject) =>
storage.getFile(userId, file).then(resolve).catch(reject)
))
);
return Promise.all(files);
});
return files;
}
/**
* TODO: Impliment Zips
* Creates a zip file and returns the zipUuid to the client.
*/
async function requestZip(uuid, targetFiles, cb) {
var zipPath, fileData;
var filePaths = new Array(targetFiles.length);
for (var file of targetFiles) {
fileData = storage.getFile(file);
if (!authorizedToViewFile(uuid, fileData)) return;
if (!fexists(fileData.path)) return;
filePaths.push(fileData.path);
}
const zipUuid = generateUuid();
cb(zipUuid);
setTimeout(() => storage.buildZip(uuid, filePaths, zipUuid), 0);
return zipUuid;
}
/**
* TODO: Impliment Zips
* Returns zip path from a zipUuid
*/
function getZipPath(uuid, targetZip) {
return storage.getZipPath(uuid, targetZip);
}
/**
* TODO: Impliment Advanced Sharing
@ -190,50 +74,21 @@ function shareFile(uuid, targetFile) {
function getSharedFiles(uuid) {
return storage.getSharedFileList(uuid);
}
/**
* Checks is a user is authorized to edit a particular file
*/
function authorizedToEditFile(client, fileData) {
if (fileData == null) return false;
if (fileData.owner === client) return true;
return fileData.edit.includes(client);
}
/**
* Checks is a user is authorized to view a particular file
*/
function authorizedToViewFile(client, fileData) {
if (fileData == null) return false;
if (fileData.public === true) return true;
if (fileData.owner === client) return true;
return fileData.edit.includes(client) || fileData.view.includes(client);
}
/**
* Checks if a the user is the owner and then toggles the list of files to public
*/
function publicfyFiles(uuid, files) {
var publicfyFails = [];
files.forEach((file, i) => {
storage.modifyFile(file, (entry) => {
if (entry == null || entry.owner !== uuid) {
publicfyFails.push(file);
return;
}
entry.public = !entry.public;
return entry;
});
});
//Return the new used storage to update the database
storage.publicfyFiles(uuid, files);
return publicfyFails.length > 0 && publicfyFails;
}
module.exports = {
createUser,
uploadFile,
deleteFiles,
getFilePath,
getOwnedFiles,
publicfyFiles,
shareFile,
getSharedFiles,
requestZip,
getZipPath,
load,
};

View file

@ -1,5 +1,6 @@
{
"Storage": {
"NubianDatabase": "mongodb://localhost",
"DesertPath": "src/desert/",
"UploadPath": "src/uploads/",
"ZipPath": "zips/",

View file

@ -3,104 +3,80 @@ const axios = require("axios");
//Local Imports & Configs
const asUser = require("../api/user");
const upload = require("../api/upload");
const storage = require("../api/storage");
const config = require("../config.json");
//Establish path and create router
/** Absolute Router Path /api/stash*/
const router = express.Router();
const authMiddleware = (req, res, next) => {
if (req.session.uuid != null) return next();
var headers = {};
var bearerToken = req.get(config.Server.jwtHeader);
if (bearerToken == null) return res.sendStatus(401);
headers[config.Server.jwtHeader] = bearerToken;
const cairoMiddleware = (req, res, next) => {
if (req.token == null) return next();
else
axios
.get(config.Server.authServer, { headers })
.get(config.Server.authServer, {
headers: { authorization: `Bearer ${req.token}` },
})
.then((authRes) => {
if (authRes.status !== 200) return res.sendStatus(401);
if (authRes.data != null) {
req.session.uuid = authRes.data.uuid;
if (authRes.status !== 200) return res.status(authres.status);
if (authRes.data != null && authRes.data.uuid != null) {
asUser.load(authRes.data.uuid).then((user) => {
req.user = user;
next();
} else res.sendStatus(401);
});
} else res.status(500).json(authRes.data);
})
.catch((e) => {
if (e.response != null) res.sendStatus(e.response.status);
else res.sendStatus(401);
else res.sendStatus(500);
});
};
router.use(cairoMiddleware);
const authMiddleware = (req, res, next) => {
if (req.token == null) return res.sendStatus(401);
next();
};
router.get("/files", authMiddleware, (req, res) => {
const files = asUser.getOwnedFiles(req.session.uuid);
router.get("/files", authMiddleware, (req, res) =>
asUser.getOwnedFiles(req.user._id).then((files) => {
res.status(200).json(files);
});
})
);
router.post("/upload", authMiddleware, (req, res) => {
upload.userUpload(req, res, (err) => {
if (err || req.file == null) return res.sendStatus(500);
const fileData = asUser.uploadFile(req.session.uuid, req.file);
if (fileData == null) {
asUser.uploadFile(req.user._id, req.file).then((file) => {
if (file != null) return res.json(file);
upload.cancelUpload(req.file.path);
return res.sendStatus(500);
}
res.json(fileData);
});
});
});
router.post("/delete", authMiddleware, (req, res) => {
if (!req.body || !(req.body instanceof Array)) {
return res.sendStatus(400);
}
const failed = asUser.deleteFiles(req.session.uuid, req.body);
if (!req.body || !(req.body instanceof Array)) return res.sendStatus(400);
asUser.deleteFiles(req.user._id, req.body).then((failed) => {
if (!failed) return res.sendStatus(200);
res.status(500).json(failed);
});
});
router.get("/download", async (req, res) => {
router.get("/download", (req, res) => {
if (!req.query || (!req.query.target && !req.query.zipTarget))
return res.sendStatus(404);
if (req.query.target) {
const filePath = asUser.getFilePath(req.session.uuid, req.query.target);
if (!filePath) return res.sendStatus(404);
return res.download(filePath);
}
//ZIPS ARE NOT SUPPORTED YET
const userId = req.user == null ? null : req.user._id;
if (req.query.target)
return storage.getFile(userId, req.query.target).then((file) => {
if (file) return res.download(file.path);
return res.sendStatus(404);
if (req.session.uuid == null) return res.sendStatus(401);
if (req.query.zipTarget) {
const zipPath = asUser.getZip(req.session.uuid, req.query.zipTarget);
if (zipPath === true) return res.sendStatus(503);
if (zipPath == null) return res.sendStatus(404);
res.download(zipPath);
}
});
//TODO
router.post("/download", authMiddleware, (req, res) => {
//ZIPS ARE NOT SUPPORTED YET
return res.sendStatus(404);
if (!req.body || !(req.body instanceof Array)) {
return res.sendStatus(400);
}
asUser.requestZip(req.session.uuid, req.body, (zipUuid) => {
console.log("Client can start checking");
return res.json(zipUuid);
});
});
router.get("/raw", (req, res) => {
if (!req.query || !req.query.target) return res.sendStatus(404);
const filePath = asUser.getFilePath(req.session.uuid, req.query.target);
if (!filePath) return res.sendStatus(404);
res.sendFile(filePath);
});
router.post("/public", authMiddleware, async (req, res) => {
if (!req.body || !(req.body instanceof Array)) {
return res.sendStatus(400);
}
const failed = asUser.publicfyFiles(req.session.uuid, req.body);
if (!req.body || !(req.body instanceof Array)) return res.sendStatus(400);
const failed = asUser.publicfyFiles(req.user._id, req.body);
if (!failed) return res.sendStatus(200);
res.status(500).json(failed);
});
module.exports = router;

40
src/schemas/file.js Normal file
View file

@ -0,0 +1,40 @@
const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const ObjId = mongoose.Types.ObjectId;
const file = new Schema(
{
path: {
type: String,
required: true,
},
owner: { type: ObjId, ref: "user" },
name: {
type: String,
required: true,
},
date: {
type: String,
required: true,
},
size: {
type: Number,
required: true,
},
public: {
type: Boolean,
required: true,
},
edit: {
type: [],
required: true,
},
view: {
type: [],
required: true,
},
},
{ collection: "files" }
);
module.exports = mongoose.model("file", file);

30
src/schemas/user.js Normal file
View file

@ -0,0 +1,30 @@
const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const ObjId = mongoose.Types.ObjectId;
const user = new Schema(
{
cairoUuid: {
type: String,
required: true,
},
storage: {
type: Number,
required: true,
},
usedStorage: {
type: Number,
required: true,
},
owned: {
type: [ObjId],
required: true,
},
shared: {
type: [],
required: true,
},
},
{ collection: "users" }
);
module.exports = mongoose.model("user", user);

View file

@ -3,6 +3,7 @@ const express = require("express");
const session = require("express-session");
const cors = require("cors");
const bodyParser = require("body-parser");
const bearerToken = require('express-bearer-token');
const secret = require("uuid-with-v6").v6;
//Local Imports
const { Web, StatusCode, Server } = require("./config.json");
@ -20,6 +21,7 @@ const corsOptions = {
};
//Set Up Express session and View engine
app.use(cors(corsOptions));
app.use(bearerToken())
app.use(session({ secret: secret(), saveUninitialized: false, resave: false }));
app.use(bodyParser.json({ limit: Server.BodyLimit })); // parse application/json
app.use(bodyParser.urlencoded({ limit: Server.BodyLimit, extended: false })); // parse application/x-www-form-urlencoded