From f40d1aee79872d757e96e100c90ceb9934f97e02 Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Wed, 15 Nov 2023 19:57:59 +0000 Subject: [PATCH] add per instance resource quotas, move getFullInstanceConfig to utils.js, rework resource related utilities to use new quota format --- src/pve.js | 102 --------------- src/routes/auth.js | 2 +- src/routes/cluster.js | 6 +- src/routes/cluster/disk.js | 6 +- src/routes/cluster/net.js | 4 +- src/routes/cluster/pci.js | 4 +- src/utils.js | 247 +++++++++++++++++++++++++++++++++---- 7 files changed, 235 insertions(+), 136 deletions(-) diff --git a/src/pve.js b/src/pve.js index 8f274fc..fdbfe72 100644 --- a/src/pve.js +++ b/src/pve.js @@ -79,108 +79,6 @@ export async function handleResponse (node, result, res) { } } -/** - * Get the full config of an instance, including searching disk information. - * @param {Object} req ProxmoxAAS API request object. - * @param {Object} instance to get config as object containing node, type, and id. - * @param {Array} diskprefixes Array containing prefixes for disks. - * @returns - */ -async function getFullInstanceConfig (req, instance, diskprefixes) { - const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data; - // fetch all instance disk and device data concurrently - const promises = []; - const mappings = []; - for (const key in config) { - if (diskprefixes.some(prefix => key.startsWith(prefix))) { - promises.push(getDiskInfo(instance.node, config, key)); - mappings.push(key); - } - else if (key.startsWith("hostpci")) { - promises.push(getDeviceInfo(instance.node, config[key].split(",")[0])); - mappings.push(key); - } - } - const results = await Promise.all(promises); - results.forEach((e, i) => { - const key = mappings[i]; - config[key] = e; - }); - return config; -} - -/** - * Get the amount of resources used by specified user. - * @param {Object} req ProxmoxAAS API request object. - * @param {Object} resourceMeta data about application resources, to indicate which resources are tracked. - * @returns {Object} k-v pairs of resource name and used amounts - */ -export async function getUsedResources (req, resourceMeta) { - // get the basic resources list - const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data; - - // setup the used object and diskPrefixes object - const used = {}; - const diskprefixes = []; - for (const resourceName of Object.keys(resourceMeta)) { - if (resourceMeta[resourceName].type === "storage") { - used[resourceName] = 0; - for (const diskPrefix of resourceMeta[resourceName].disks) { - diskprefixes.push(diskPrefix); - } - } - else if (resourceMeta[resourceName].type === "list") { - used[resourceName] = []; - } - else { - used[resourceName] = 0; - } - } - - // filter resources by their type, we only want lxc and qemu - const instances = []; - for (const resource of resources) { - if (resource.type === "lxc" || resource.type === "qemu") { - instances.push(resource); - } - } - - const promises = []; - const mappings = []; - for (let i = 0; i < instances.length; i++) { - const instance = instances[i]; - promises.push(getFullInstanceConfig(req, instance, diskprefixes)); - mappings.push(i); - } - const configs = await Promise.all(promises); - - // for each instance, sum each resource - for (const config of configs) { - for (const key of Object.keys(config)) { - if (Object.keys(used).includes(key) && resourceMeta[key].type === "numeric") { - used[key] += Number(config[key]); - } - else if (diskprefixes.some(prefix => key.startsWith(prefix))) { - const diskInfo = config[key]; - if (diskInfo) { // only count if disk exists - used[diskInfo.storage] += Number(diskInfo.size); - } - } - else if (key.startsWith("net") && config[key].includes("rate=")) { // only count net instances with a rate limit - used.network += Number(config[key].split("rate=")[1].split(",")[0]); - } - else if (key.startsWith("hostpci")) { - const deviceInfo = config[key]; - if (deviceInfo) { // only count if device exists - used.pci.push(deviceInfo.device_name); - } - } - } - } - - return used; -} - /** * Get meta data for a specific disk. Adds info that is not normally available in a instance's config. * @param {string} node containing the query disk. diff --git a/src/routes/auth.js b/src/routes/auth.js index 530a312..ad18995 100644 --- a/src/routes/auth.js +++ b/src/routes/auth.js @@ -67,7 +67,7 @@ router.post("/password", async (req, res) => { password: req.body.password, userid: req.cookies.username }; - + const userRealm = params.userid.split("@").at(-1); const domains = (await requestPVE("/access/domains", "GET", pveAPIToken)).data.data; const realm = domains.find((e) => e.realm === userRealm); diff --git a/src/routes/cluster.js b/src/routes/cluster.js index fe0121a..fe6c332 100644 --- a/src/routes/cluster.js +++ b/src/routes/cluster.js @@ -44,7 +44,7 @@ router.get(`/:node(${nodeRegexP})/pci`, async (req, res) => { return; } // get remaining user resources - const userAvailPci = (await getUserResources(req, req.cookies.username)).pci; + const userAvailPci = (await getUserResources(req, req.cookies.username)).pci.nodes[params.node]; // get node avail devices let nodeAvailPci = await getNodeAvailDevices(params.node, req.cookies); nodeAvailPci = nodeAvailPci.filter(nodeAvail => userAvailPci.some((userAvail) => { @@ -100,7 +100,7 @@ router.post(`${basePath}/resources`, async (req, res) => { request.cpu = params.proctype; } // check resource approval - if (!await approveResources(req, req.cookies.username, request)) { + if (!await approveResources(req, req.cookies.username, request, params.node)) { res.status(500).send({ request, error: "Could not fulfil request." }); res.end(); return; @@ -201,7 +201,7 @@ router.post(`${basePath}/create`, async (req, res) => { } } // check resource approval - if (!await approveResources(req, req.cookies.username, request)) { // check resource approval + if (!await approveResources(req, req.cookies.username, request, params.node)) { // check resource approval res.status(500).send({ request, error: "Not enough resources to satisfy request." }); res.end(); return; diff --git a/src/routes/cluster/disk.js b/src/routes/cluster/disk.js index 11d514d..616adb2 100644 --- a/src/routes/cluster/disk.js +++ b/src/routes/cluster/disk.js @@ -155,7 +155,7 @@ router.post("/:disk/resize", async (req, res) => { const request = {}; request[storage] = Number(params.size * 1024 ** 3); // setup request object // check request approval - if (!await approveResources(req, req.cookies.username, request)) { + if (!await approveResources(req, req.cookies.username, request, params.node)) { res.status(500).send({ request, error: `Storage ${storage} could not fulfill request of size ${params.size}G.` }); res.end(); return; @@ -215,7 +215,7 @@ router.post("/:disk/move", async (req, res) => { request[dstStorage] = Number(size); // always decrease destination storage by size } // check request approval - if (!await approveResources(req, req.cookies.username, request)) { + if (!await approveResources(req, req.cookies.username, request, params.node)) { res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` }); res.end(); return; @@ -331,7 +331,7 @@ router.post("/:disk/create", async (req, res) => { // setup request request[params.storage] = Number(params.size * 1024 ** 3); // check request approval - if (!await approveResources(req, req.cookies.username, request)) { + if (!await approveResources(req, req.cookies.username, request, params.node)) { res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` }); res.end(); return; diff --git a/src/routes/cluster/net.js b/src/routes/cluster/net.js index 9f8ca1d..878b54c 100644 --- a/src/routes/cluster/net.js +++ b/src/routes/cluster/net.js @@ -57,7 +57,7 @@ router.post("/:netid/create", async (req, res) => { network: Number(params.rate) }; // check resource approval - if (!await approveResources(req, req.cookies.username, request)) { + if (!await approveResources(req, req.cookies.username, request, params.node)) { res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` }); res.end(); return; @@ -122,7 +122,7 @@ router.post("/:netid/modify", async (req, res) => { network: Number(params.rate) - Number(currentNetworkRate) }; // check resource approval - if (!await approveResources(req, req.cookies.username, request)) { + if (!await approveResources(req, req.cookies.username, request, params.node)) { res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` }); res.end(); return; diff --git a/src/routes/cluster/pci.js b/src/routes/cluster/pci.js index d96e904..5dc1a60 100644 --- a/src/routes/cluster/pci.js +++ b/src/routes/cluster/pci.js @@ -108,7 +108,7 @@ router.post("/:hostpci/modify", async (req, res) => { const deviceData = await getDeviceInfo(params.node, params.device); const request = { pci: deviceData.device_name }; // check resource approval - if (!await approveResources(req, req.cookies.username, request)) { + if (!await approveResources(req, req.cookies.username, request, params.node)) { res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` }); res.end(); return; @@ -189,7 +189,7 @@ router.post("/create", async (req, res) => { pci: deviceData.device_name }; // check resource approval - if (!await approveResources(req, req.cookies.username, request)) { + if (!await approveResources(req, req.cookies.username, request, params.node)) { res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` }); res.end(); return; diff --git a/src/utils.js b/src/utils.js index 9346b33..af63b7a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -3,7 +3,7 @@ import path from "path"; import url from "url"; import * as fs from "fs"; -import { getUsedResources, requestPVE } from "./pve.js"; +import { requestPVE, getDiskInfo, getDeviceInfo } from "./pve.js"; /** * Check if a user is authorized to access a specified vm, or the cluster in general. @@ -39,6 +39,69 @@ export async function checkAuth (cookies, res, vmpath = null) { return auth; } +/** + * Get the full config of an instance, including searching disk information. + * @param {Object} req ProxmoxAAS API request object. + * @param {Object} instance to get config as object containing node, type, and id. + * @param {Array} diskprefixes Array containing prefixes for disks. + * @returns + */ +async function getFullInstanceConfig (req, instance, diskprefixes) { + const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data; + // fetch all instance disk and device data concurrently + const promises = []; + const mappings = []; + for (const key in config) { + if (diskprefixes.some(prefix => key.startsWith(prefix))) { + promises.push(getDiskInfo(instance.node, config, key)); + mappings.push(key); + } + else if (key.startsWith("hostpci")) { + promises.push(getDeviceInfo(instance.node, config[key].split(",")[0])); + mappings.push(key); + } + } + const results = await Promise.all(promises); + results.forEach((e, i) => { + const key = mappings[i]; + config[key] = e; + }); + config.node = instance.node; + return config; +} + +/** + * Get all configs for every instance owned by the user. Uses the expanded config data from getFullInstanceConfig. + * @param {Object} req ProxmoxAAS API request object. + * @param {Object} dbResources data about application resources, to indicate which resources are tracked. + * @returns {Object} k-v pairs of resource name and used amounts + */ +async function getAllInstanceConfigs (req, diskprefixes) { + // get the basic resources list + const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data; + + // filter resources by their type, we only want lxc and qemu + const instances = []; + for (const resource of resources) { + if (resource.type === "lxc" || resource.type === "qemu") { + instances.push(resource); + } + } + + // get all instance configs, also include detailed disk and device info + const promises = []; + const mappings = []; + for (let i = 0; i < instances.length; i++) { + const instance = instances[i]; + const config = getFullInstanceConfig(req, instance, diskprefixes); + promises.push(config); + mappings.push(i); + } + const configs = await Promise.all(promises); + + return configs; +} + /** * Get user resource data including used, available, and maximum resources. * @param {Object} req ProxmoxAAS API request object. @@ -48,25 +111,149 @@ export async function checkAuth (cookies, res, vmpath = null) { export async function getUserResources (req, username) { const db = global.db; const dbResources = db.getGlobal().resources; - const used = await getUsedResources(req, dbResources); const userResources = db.getUser(username).resources; - Object.keys(userResources).forEach((k) => { - if (dbResources[k] && dbResources[k].type === "list") { - userResources[k].forEach((listResource) => { - listResource.used = 0; - listResource.avail = listResource.max; - }); - used[k].forEach((usedDeviceName) => { - const index = userResources[k].findIndex((availEelement) => usedDeviceName.includes(availEelement.match)); - userResources[k][index].used++; - userResources[k][index].avail--; + + // setup disk prefixes object + const diskprefixes = []; + for (const resourceName of Object.keys(dbResources)) { + if (dbResources[resourceName].type === "storage") { + for (const diskPrefix of dbResources[resourceName].disks) { + diskprefixes.push(diskPrefix); + } + } + } + + // setup the user resource object with used and avail for each resource and each resource pool + // also add a total counter for each resource (only used for display, not used to check requests) + for (const resourceName of Object.keys(userResources)) { + if (dbResources[resourceName].type === "list") { + userResources[resourceName].total = []; + userResources[resourceName].global.forEach((e) => { + e.used = 0; + e.avail = e.max; + const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match); + if (index === -1) { + userResources[resourceName].total.push(structuredClone(e)); + } + else { + userResources[resourceName].total[index].max += e.max; + userResources[resourceName].total[index].avail += e.avail; + } }); + for (const nodeName of Object.keys(userResources[resourceName].nodes)) { + userResources[resourceName].nodes[nodeName].forEach((e) => { + e.used = 0; + e.avail = e.max; + const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match); + if (index === -1) { + userResources[resourceName].total.push(structuredClone(e)); + } + else { + userResources[resourceName].total[index].max += e.max; + userResources[resourceName].total[index].avail += e.avail; + } + }); + } } else { - userResources[k].used = used[k]; - userResources[k].avail = userResources[k].max - used[k]; + const total = { + max: 0, + used: 0, + avail: 0 + }; + userResources[resourceName].global.used = 0; + userResources[resourceName].global.avail = userResources[resourceName].global.max; + total.max += userResources[resourceName].global.max; + total.avail += userResources[resourceName].global.avail; + for (const nodeName of Object.keys(userResources[resourceName].nodes)) { + userResources[resourceName].nodes[nodeName].used = 0; + userResources[resourceName].nodes[nodeName].avail = userResources[resourceName].nodes[nodeName].max; + total.max += userResources[resourceName].nodes[nodeName].max; + total.avail += userResources[resourceName].nodes[nodeName].avail; + } + userResources[resourceName].total = total; } - }); + } + + const configs = await getAllInstanceConfigs(req, diskprefixes); + + for (const config of configs) { + const nodeName = config.node; + for (const resourceName of Object.keys(config)) { + // numeric resource type + if (resourceName in dbResources && dbResources[resourceName].type === "numeric") { + const val = Number(config[resourceName]); + // if the instance's node is restricted by this resource, add it to the instance's used value + if (nodeName in userResources[resourceName].nodes) { + userResources[resourceName].nodes[nodeName].used += val; + userResources[resourceName].nodes[nodeName].avail -= val; + } + // otherwise add the resource to the global pool + else { + userResources[resourceName].global.used += val; + userResources[resourceName].global.avail -= val; + } + userResources[resourceName].total.used += val; + userResources[resourceName].total.avail -= val; + } + else if (diskprefixes.some(prefix => resourceName.startsWith(prefix))) { + const diskInfo = config[resourceName]; + if (diskInfo) { // only count if disk exists + const val = Number(diskInfo.size); + const storage = diskInfo.storage; + // if the instance's node is restricted by this resource, add it to the instance's used value + if (nodeName in userResources[storage].nodes) { + userResources[storage].nodes[nodeName].used += val; + userResources[storage].nodes[nodeName].avail -= val; + } + // otherwise add the resource to the global pool + else { + userResources[storage].global.used += val; + userResources[storage].global.avail -= val; + } + userResources[storage].total.used += val; + userResources[storage].total.avail -= val; + } + } + else if (resourceName.startsWith("net") && config[resourceName].includes("rate=")) { // only count net instances with a rate limit + const val = Number(config[resourceName].split("rate=")[1].split(",")[0]); + // if the instance's node is restricted by this resource, add it to the instance's used value + if (nodeName in userResources.network.nodes) { + userResources.network.nodes[nodeName].used += val; + userResources.network.nodes[nodeName].avail -= val; + } + // otherwise add the resource to the global pool + else { + userResources.network.global.used += val; + userResources.network.global.avail -= val; + } + userResources.network.total.used += val; + userResources.network.total.avail -= val; + } + else if (resourceName.startsWith("hostpci")) { + const deviceInfo = config[resourceName]; + if (deviceInfo) { // only count if device exists + const deviceName = deviceInfo.device_name; + // if the instance's node is restricted by this resource, add it to the instance's used value + if (nodeName in userResources.pci.nodes) { + const index = userResources.pci.nodes[nodeName].findIndex((availEelement) => deviceName.includes(availEelement.match)); + userResources.pci.nodes[nodeName][index].used++; + userResources.pci.nodes[nodeName][index].avail--; + } + // otherwise add the resource to the global pool + else { + const index = userResources.pci.global.findIndex((availEelement) => deviceName.includes(availEelement.match)); + userResources.pci.global[index].used++; + userResources.pci.global[index].avail--; + } + const index = userResources.pci.total.findIndex((availEelement) => deviceName.includes(availEelement.match)); + userResources.pci.total[index].used++; + userResources.pci.total[index].avail--; + } + } + } + } + return userResources; } @@ -77,29 +264,43 @@ export async function getUserResources (req, username) { * @param {Object} request k-v pairs of resources and requested amounts * @returns {boolean} true if the available resources can fullfill the requested resources, false otherwise. */ -export async function approveResources (req, username, request) { +export async function approveResources (req, username, request, node) { const db = global.db; const dbResources = db.getGlobal().resources; const userResources = await getUserResources(req, username); let approved = true; - Object.keys(request).forEach((key) => { - if (!(key in userResources)) { // if requested resource is not in avail, block + Object.keys(request).every((key) => { + // if requested resource is not specified in user resources, assume it's not allowed + if (!(key in userResources)) { approved = false; + return false; } - else if (dbResources[key].type === "list") { // if the resource type is list, check if the requested resource exists in the list - const index = userResources[key].findIndex((availElement) => request[key].includes(availElement.match)); + + const inNode = node in userResources[key].nodes; + const resourceData = inNode ? userResources[key].nodes[node] : userResources[key].global; + + // if the resource type is list, check if the requested resource exists in the list + if (dbResources[key].type === "list") { + const index = resourceData.findIndex((availElement) => request[key].includes(availElement.match)); // if no matching resource when index == -1, then remaining is -1 otherwise use the remaining value - const avail = index === -1 ? false : userResources[key][index].avail > 0; + const avail = index === -1 ? false : resourceData[index].avail > 0; if (avail !== dbResources[key].whitelist) { approved = false; + return false; } } - else if (isNaN(userResources[key].avail) || isNaN(request[key])) { // if either the requested or avail resource is NaN, block + // if either the requested or avail resource is NaN, block + else if (isNaN(resourceData.avail) || isNaN(request[key])) { approved = false; + return false; } - else if (userResources[key].avail - request[key] < 0) { // if the avail resources is less than the requested resources, block + // if the avail resources is less than the requested resources, block + else if (resourceData.avail - request[key] < 0) { approved = false; + return false; } + + return true; }); return approved; // if all requested resources pass, allow }