add per instance resource quotas,

move getFullInstanceConfig to utils.js,
rework resource related utilities to use new quota format

src/pve.js

@@ -79,108 +79,6 @@ export async function handleResponse (node, result, res) {
 	}
 }
 
-/**
- * Get the full config of an instance, including searching disk information.
- * @param {Object} req ProxmoxAAS API request object.
- * @param {Object} instance to get config as object containing node, type, and id.
- * @param {Array} diskprefixes Array containing prefixes for disks.
- * @returns
- */
-async function getFullInstanceConfig (req, instance, diskprefixes) {
-	const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data;
-	// fetch all instance disk and device data concurrently
-	const promises = [];
-	const mappings = [];
-	for (const key in config) {
-		if (diskprefixes.some(prefix => key.startsWith(prefix))) {
-			promises.push(getDiskInfo(instance.node, config, key));
-			mappings.push(key);
-		}
-		else if (key.startsWith("hostpci")) {
-			promises.push(getDeviceInfo(instance.node, config[key].split(",")[0]));
-			mappings.push(key);
-		}
-	}
-	const results = await Promise.all(promises);
-	results.forEach((e, i) => {
-		const key = mappings[i];
-		config[key] = e;
-	});
-	return config;
-}
-
-/**
- * Get the amount of resources used by specified user.
- * @param {Object} req ProxmoxAAS API request object.
- * @param {Object} resourceMeta data about application resources, to indicate which resources are tracked.
- * @returns {Object} k-v pairs of resource name and used amounts
- */
-export async function getUsedResources (req, resourceMeta) {
-	// get the basic resources list
-	const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data;
-
-	// setup the used object and diskPrefixes object
-	const used = {};
-	const diskprefixes = [];
-	for (const resourceName of Object.keys(resourceMeta)) {
-		if (resourceMeta[resourceName].type === "storage") {
-			used[resourceName] = 0;
-			for (const diskPrefix of resourceMeta[resourceName].disks) {
-				diskprefixes.push(diskPrefix);
-			}
-		}
-		else if (resourceMeta[resourceName].type === "list") {
-			used[resourceName] = [];
-		}
-		else {
-			used[resourceName] = 0;
-		}
-	}
-
-	// filter resources by their type, we only want lxc and qemu
-	const instances = [];
-	for (const resource of resources) {
-		if (resource.type === "lxc" || resource.type === "qemu") {
-			instances.push(resource);
-		}
-	}
-
-	const promises = [];
-	const mappings = [];
-	for (let i = 0; i < instances.length; i++) {
-		const instance = instances[i];
-		promises.push(getFullInstanceConfig(req, instance, diskprefixes));
-		mappings.push(i);
-	}
-	const configs = await Promise.all(promises);
-
-	// for each instance, sum each resource
-	for (const config of configs) {
-		for (const key of Object.keys(config)) {
-			if (Object.keys(used).includes(key) && resourceMeta[key].type === "numeric") {
-				used[key] += Number(config[key]);
-			}
-			else if (diskprefixes.some(prefix => key.startsWith(prefix))) {
-				const diskInfo = config[key];
-				if (diskInfo) { // only count if disk exists
-					used[diskInfo.storage] += Number(diskInfo.size);
-				}
-			}
-			else if (key.startsWith("net") && config[key].includes("rate=")) { // only count net instances with a rate limit
-				used.network += Number(config[key].split("rate=")[1].split(",")[0]);
-			}
-			else if (key.startsWith("hostpci")) {
-				const deviceInfo = config[key];
-				if (deviceInfo) { // only count if device exists
-					used.pci.push(deviceInfo.device_name);
-				}
-			}
-		}
-	}
-
-	return used;
-}
-
 /**
  * Get meta data for a specific disk. Adds info that is not normally available in a instance's config.
  * @param {string} node containing the query disk.

src/routes/auth.js

@@ -67,7 +67,7 @@ router.post("/password", async (req, res) => {
 		password: req.body.password,
 		userid: req.cookies.username
 	};
-	
+
 	const userRealm = params.userid.split("@").at(-1);
 	const domains = (await requestPVE("/access/domains", "GET", pveAPIToken)).data.data;
 	const realm = domains.find((e) => e.realm === userRealm);

src/routes/cluster.js

@@ -44,7 +44,7 @@ router.get(`/:node(${nodeRegexP})/pci`, async (req, res) => {
 		return;
 	}
 	// get remaining user resources
-	const userAvailPci = (await getUserResources(req, req.cookies.username)).pci;
+	const userAvailPci = (await getUserResources(req, req.cookies.username)).pci.nodes[params.node];
 	// get node avail devices
 	let nodeAvailPci = await getNodeAvailDevices(params.node, req.cookies);
 	nodeAvailPci = nodeAvailPci.filter(nodeAvail => userAvailPci.some((userAvail) => {
@@ -100,7 +100,7 @@ router.post(`${basePath}/resources`, async (req, res) => {
 		request.cpu = params.proctype;
 	}
 	// check resource approval
-	if (!await approveResources(req, req.cookies.username, request)) {
+	if (!await approveResources(req, req.cookies.username, request,  params.node)) {
 		res.status(500).send({ request, error: "Could not fulfil request." });
 		res.end();
 		return;
@@ -201,7 +201,7 @@ router.post(`${basePath}/create`, async (req, res) => {
 		}
 	}
 	// check resource approval
-	if (!await approveResources(req, req.cookies.username, request)) { // check resource approval
+	if (!await approveResources(req, req.cookies.username, request,  params.node)) { // check resource approval
 		res.status(500).send({ request, error: "Not enough resources to satisfy request." });
 		res.end();
 		return;

src/routes/cluster/disk.js

@@ -155,7 +155,7 @@ router.post("/:disk/resize", async (req, res) => {
 	const request = {};
 	request[storage] = Number(params.size * 1024 ** 3); // setup request object
 	// check request approval
-	if (!await approveResources(req, req.cookies.username, request)) {
+	if (!await approveResources(req, req.cookies.username, request,  params.node)) {
 		res.status(500).send({ request, error: `Storage ${storage} could not fulfill request of size ${params.size}G.` });
 		res.end();
 		return;
@@ -215,7 +215,7 @@ router.post("/:disk/move", async (req, res) => {
 		request[dstStorage] = Number(size); // always decrease destination storage by size
 	}
 	// check request approval
-	if (!await approveResources(req, req.cookies.username, request)) {
+	if (!await approveResources(req, req.cookies.username, request,  params.node)) {
 		res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` });
 		res.end();
 		return;
@@ -331,7 +331,7 @@ router.post("/:disk/create", async (req, res) => {
 		// setup request
 		request[params.storage] = Number(params.size * 1024 ** 3);
 		// check request approval
-		if (!await approveResources(req, req.cookies.username, request)) {
+		if (!await approveResources(req, req.cookies.username, request,  params.node)) {
 			res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` });
 			res.end();
 			return;

src/routes/cluster/net.js

@@ -57,7 +57,7 @@ router.post("/:netid/create", async (req, res) => {
 		network: Number(params.rate)
 	};
 	// check resource approval
-	if (!await approveResources(req, req.cookies.username, request)) {
+	if (!await approveResources(req, req.cookies.username, request,  params.node)) {
 		res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` });
 		res.end();
 		return;
@@ -122,7 +122,7 @@ router.post("/:netid/modify", async (req, res) => {
 		network: Number(params.rate) - Number(currentNetworkRate)
 	};
 	// check resource approval
-	if (!await approveResources(req, req.cookies.username, request)) {
+	if (!await approveResources(req, req.cookies.username, request,  params.node)) {
 		res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` });
 		res.end();
 		return;

src/routes/cluster/pci.js

@@ -108,7 +108,7 @@ router.post("/:hostpci/modify", async (req, res) => {
 		const deviceData = await getDeviceInfo(params.node, params.device);
 		const request = { pci: deviceData.device_name };
 		// check resource approval
-		if (!await approveResources(req, req.cookies.username, request)) {
+		if (!await approveResources(req, req.cookies.username, request,  params.node)) {
 			res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` });
 			res.end();
 			return;
@@ -189,7 +189,7 @@ router.post("/create", async (req, res) => {
 		pci: deviceData.device_name
 	};
 	// check resource approval
-	if (!await approveResources(req, req.cookies.username, request)) {
+	if (!await approveResources(req, req.cookies.username, request,  params.node)) {
 		res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` });
 		res.end();
 		return;

src/utils.js

@@ -3,7 +3,7 @@ import path from "path";
 import url from "url";
 import * as fs from "fs";
 
-import { getUsedResources, requestPVE } from "./pve.js";
+import { requestPVE, getDiskInfo, getDeviceInfo } from "./pve.js";
 
 /**
  * Check if a user is authorized to access a specified vm, or the cluster in general.
@@ -39,6 +39,69 @@ export async function checkAuth (cookies, res, vmpath = null) {
 	return auth;
 }
 
+/**
+ * Get the full config of an instance, including searching disk information.
+ * @param {Object} req ProxmoxAAS API request object.
+ * @param {Object} instance to get config as object containing node, type, and id.
+ * @param {Array} diskprefixes Array containing prefixes for disks.
+ * @returns
+ */
+async function getFullInstanceConfig (req, instance, diskprefixes) {
+	const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data;
+	// fetch all instance disk and device data concurrently
+	const promises = [];
+	const mappings = [];
+	for (const key in config) {
+		if (diskprefixes.some(prefix => key.startsWith(prefix))) {
+			promises.push(getDiskInfo(instance.node, config, key));
+			mappings.push(key);
+		}
+		else if (key.startsWith("hostpci")) {
+			promises.push(getDeviceInfo(instance.node, config[key].split(",")[0]));
+			mappings.push(key);
+		}
+	}
+	const results = await Promise.all(promises);
+	results.forEach((e, i) => {
+		const key = mappings[i];
+		config[key] = e;
+	});
+	config.node = instance.node;
+	return config;
+}
+
+/**
+ * Get all configs for every instance owned by the user. Uses the expanded config data from getFullInstanceConfig.
+ * @param {Object} req ProxmoxAAS API request object.
+ * @param {Object} dbResources data about application resources, to indicate which resources are tracked.
+ * @returns {Object} k-v pairs of resource name and used amounts
+ */
+async function getAllInstanceConfigs (req, diskprefixes) {
+	// get the basic resources list
+	const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data;
+
+	// filter resources by their type, we only want lxc and qemu
+	const instances = [];
+	for (const resource of resources) {
+		if (resource.type === "lxc" || resource.type === "qemu") {
+			instances.push(resource);
+		}
+	}
+
+	// get all instance configs, also include detailed disk and device info
+	const promises = [];
+	const mappings = [];
+	for (let i = 0; i < instances.length; i++) {
+		const instance = instances[i];
+		const config = getFullInstanceConfig(req, instance, diskprefixes);
+		promises.push(config);
+		mappings.push(i);
+	}
+	const configs = await Promise.all(promises);
+
+	return configs;
+}
+
 /**
  * Get user resource data including used, available, and maximum resources.
  * @param {Object} req ProxmoxAAS API request object.
@@ -48,25 +111,149 @@ export async function checkAuth (cookies, res, vmpath = null) {
 export async function getUserResources (req, username) {
 	const db = global.db;
 	const dbResources = db.getGlobal().resources;
-	const used = await getUsedResources(req, dbResources);
 	const userResources = db.getUser(username).resources;
-	Object.keys(userResources).forEach((k) => {
+
-		if (dbResources[k] && dbResources[k].type === "list") {
+	// setup disk prefixes object
-			userResources[k].forEach((listResource) => {
+	const diskprefixes = [];
-				listResource.used = 0;
+	for (const resourceName of Object.keys(dbResources)) {
-				listResource.avail = listResource.max;
+		if (dbResources[resourceName].type === "storage") {
-			});
+			for (const diskPrefix of dbResources[resourceName].disks) {
-			used[k].forEach((usedDeviceName) => {
+				diskprefixes.push(diskPrefix);
-				const index = userResources[k].findIndex((availEelement) => usedDeviceName.includes(availEelement.match));
+			}
-				userResources[k][index].used++;
+		}
-				userResources[k][index].avail--;
+	}
+
+	// setup the user resource object with used and avail for each resource and each resource pool
+	// also add a total counter for each resource (only used for display, not used to check requests)
+	for (const resourceName of Object.keys(userResources)) {
+		if (dbResources[resourceName].type === "list") {
+			userResources[resourceName].total = [];
+			userResources[resourceName].global.forEach((e) => {
+				e.used = 0;
+				e.avail = e.max;
+				const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match);
+				if (index === -1) {
+					userResources[resourceName].total.push(structuredClone(e));
+				}
+				else {
+					userResources[resourceName].total[index].max += e.max;
+					userResources[resourceName].total[index].avail += e.avail;
+				}
 			});
+			for (const nodeName of Object.keys(userResources[resourceName].nodes)) {
+				userResources[resourceName].nodes[nodeName].forEach((e) => {
+					e.used = 0;
+					e.avail = e.max;
+					const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match);
+					if (index === -1) {
+						userResources[resourceName].total.push(structuredClone(e));
+					}
+					else {
+						userResources[resourceName].total[index].max += e.max;
+						userResources[resourceName].total[index].avail += e.avail;
+					}
+				});
+			}
 		}
 		else {
-			userResources[k].used = used[k];
+			const total = {
-			userResources[k].avail = userResources[k].max - used[k];
+				max: 0,
+				used: 0,
+				avail: 0
+			};
+			userResources[resourceName].global.used = 0;
+			userResources[resourceName].global.avail = userResources[resourceName].global.max;
+			total.max += userResources[resourceName].global.max;
+			total.avail += userResources[resourceName].global.avail;
+			for (const nodeName of Object.keys(userResources[resourceName].nodes)) {
+				userResources[resourceName].nodes[nodeName].used = 0;
+				userResources[resourceName].nodes[nodeName].avail = userResources[resourceName].nodes[nodeName].max;
+				total.max += userResources[resourceName].nodes[nodeName].max;
+				total.avail += userResources[resourceName].nodes[nodeName].avail;
+			}
+			userResources[resourceName].total = total;
 		}
-	});
+	}
+
+	const configs = await getAllInstanceConfigs(req, diskprefixes);
+
+	for (const config of configs) {
+		const nodeName = config.node;
+		for (const resourceName of Object.keys(config)) {
+			// numeric resource type
+			if (resourceName in dbResources && dbResources[resourceName].type === "numeric") {
+				const val = Number(config[resourceName]);
+				// if the instance's node is restricted by this resource, add it to the instance's used value
+				if (nodeName in userResources[resourceName].nodes) {
+					userResources[resourceName].nodes[nodeName].used += val;
+					userResources[resourceName].nodes[nodeName].avail -= val;
+				}
+				// otherwise add the resource to the global pool
+				else {
+					userResources[resourceName].global.used += val;
+					userResources[resourceName].global.avail -= val;
+				}
+				userResources[resourceName].total.used += val;
+				userResources[resourceName].total.avail -= val;
+			}
+			else if (diskprefixes.some(prefix => resourceName.startsWith(prefix))) {
+				const diskInfo = config[resourceName];
+				if (diskInfo) { // only count if disk exists
+					const val = Number(diskInfo.size);
+					const storage = diskInfo.storage;
+					// if the instance's node is restricted by this resource, add it to the instance's used value
+					if (nodeName in userResources[storage].nodes) {
+						userResources[storage].nodes[nodeName].used += val;
+						userResources[storage].nodes[nodeName].avail -= val;
+					}
+					// otherwise add the resource to the global pool
+					else {
+						userResources[storage].global.used += val;
+						userResources[storage].global.avail -= val;
+					}
+					userResources[storage].total.used += val;
+					userResources[storage].total.avail -= val;
+				}
+			}
+			else if (resourceName.startsWith("net") && config[resourceName].includes("rate=")) { // only count net instances with a rate limit
+				const val = Number(config[resourceName].split("rate=")[1].split(",")[0]);
+				// if the instance's node is restricted by this resource, add it to the instance's used value
+				if (nodeName in userResources.network.nodes) {
+					userResources.network.nodes[nodeName].used += val;
+					userResources.network.nodes[nodeName].avail -= val;
+				}
+				// otherwise add the resource to the global pool
+				else {
+					userResources.network.global.used += val;
+					userResources.network.global.avail -= val;
+				}
+				userResources.network.total.used += val;
+				userResources.network.total.avail -= val;
+			}
+			else if (resourceName.startsWith("hostpci")) {
+				const deviceInfo = config[resourceName];
+				if (deviceInfo) { // only count if device exists
+					const deviceName = deviceInfo.device_name;
+					// if the instance's node is restricted by this resource, add it to the instance's used value
+					if (nodeName in userResources.pci.nodes) {
+						const index = userResources.pci.nodes[nodeName].findIndex((availEelement) => deviceName.includes(availEelement.match));
+						userResources.pci.nodes[nodeName][index].used++;
+						userResources.pci.nodes[nodeName][index].avail--;
+					}
+					// otherwise add the resource to the global pool
+					else {
+						const index = userResources.pci.global.findIndex((availEelement) => deviceName.includes(availEelement.match));
+						userResources.pci.global[index].used++;
+						userResources.pci.global[index].avail--;
+					}
+					const index = userResources.pci.total.findIndex((availEelement) => deviceName.includes(availEelement.match));
+					userResources.pci.total[index].used++;
+					userResources.pci.total[index].avail--;
+				}
+			}
+		}
+	}
+
 	return userResources;
 }
 
@@ -77,29 +264,43 @@ export async function getUserResources (req, username) {
  * @param {Object} request k-v pairs of resources and requested amounts
  * @returns {boolean} true if the available resources can fullfill the requested resources, false otherwise.
  */
-export async function approveResources (req, username, request) {
+export async function approveResources (req, username, request, node) {
 	const db = global.db;
 	const dbResources = db.getGlobal().resources;
 	const userResources = await getUserResources(req, username);
 	let approved = true;
-	Object.keys(request).forEach((key) => {
+	Object.keys(request).every((key) => {
-		if (!(key in userResources)) { // if requested resource is not in avail, block
+		// if requested resource is not specified in user resources, assume it's not allowed
+		if (!(key in userResources)) {
 			approved = false;
+			return false;
 		}
-		else if (dbResources[key].type === "list") { // if the resource type is list, check if the requested resource exists in the list
+
-			const index = userResources[key].findIndex((availElement) => request[key].includes(availElement.match));
+		const inNode = node in userResources[key].nodes;
+		const resourceData = inNode ? userResources[key].nodes[node] : userResources[key].global;
+
+		// if the resource type is list, check if the requested resource exists in the list
+		if (dbResources[key].type === "list") {
+			const index = resourceData.findIndex((availElement) => request[key].includes(availElement.match));
 			// if no matching resource when index == -1, then remaining is -1 otherwise use the remaining value
-			const avail = index === -1 ? false : userResources[key][index].avail > 0;
+			const avail = index === -1 ? false : resourceData[index].avail > 0;
 			if (avail !== dbResources[key].whitelist) {
 				approved = false;
+				return false;
 			}
 		}
-		else if (isNaN(userResources[key].avail) || isNaN(request[key])) { // if either the requested or avail resource is NaN, block
+		// if either the requested or avail resource is NaN, block
+		else if (isNaN(resourceData.avail) || isNaN(request[key])) {
 			approved = false;
+			return false;
 		}
-		else if (userResources[key].avail - request[key] < 0) { // if the avail resources is less than the requested resources, block
+		// if the avail resources is less than the requested resources, block
+		else if (resourceData.avail - request[key] < 0) {
 			approved = false;
+			return false;
 		}
+
+		return true;
 	});
 	return approved; // if all requested resources pass, allow
 }