add per instance resource quotas,

move getFullInstanceConfig to utils.js,
rework resource related utilities to use new quota format
This commit is contained in:
Arthur Lu 2023-11-15 19:57:59 +00:00
parent 02764c8f6a
commit c2ab19b6d6
7 changed files with 235 additions and 136 deletions

View File

@ -79,108 +79,6 @@ export async function handleResponse (node, result, res) {
}
}
/**
* Get the full config of an instance, including searching disk information.
* @param {Object} req ProxmoxAAS API request object.
* @param {Object} instance to get config as object containing node, type, and id.
* @param {Array} diskprefixes Array containing prefixes for disks.
* @returns
*/
async function getFullInstanceConfig (req, instance, diskprefixes) {
const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data;
// fetch all instance disk and device data concurrently
const promises = [];
const mappings = [];
for (const key in config) {
if (diskprefixes.some(prefix => key.startsWith(prefix))) {
promises.push(getDiskInfo(instance.node, config, key));
mappings.push(key);
}
else if (key.startsWith("hostpci")) {
promises.push(getDeviceInfo(instance.node, config[key].split(",")[0]));
mappings.push(key);
}
}
const results = await Promise.all(promises);
results.forEach((e, i) => {
const key = mappings[i];
config[key] = e;
});
return config;
}
/**
* Get the amount of resources used by specified user.
* @param {Object} req ProxmoxAAS API request object.
* @param {Object} resourceMeta data about application resources, to indicate which resources are tracked.
* @returns {Object} k-v pairs of resource name and used amounts
*/
export async function getUsedResources (req, resourceMeta) {
// get the basic resources list
const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data;
// setup the used object and diskPrefixes object
const used = {};
const diskprefixes = [];
for (const resourceName of Object.keys(resourceMeta)) {
if (resourceMeta[resourceName].type === "storage") {
used[resourceName] = 0;
for (const diskPrefix of resourceMeta[resourceName].disks) {
diskprefixes.push(diskPrefix);
}
}
else if (resourceMeta[resourceName].type === "list") {
used[resourceName] = [];
}
else {
used[resourceName] = 0;
}
}
// filter resources by their type, we only want lxc and qemu
const instances = [];
for (const resource of resources) {
if (resource.type === "lxc" || resource.type === "qemu") {
instances.push(resource);
}
}
const promises = [];
const mappings = [];
for (let i = 0; i < instances.length; i++) {
const instance = instances[i];
promises.push(getFullInstanceConfig(req, instance, diskprefixes));
mappings.push(i);
}
const configs = await Promise.all(promises);
// for each instance, sum each resource
for (const config of configs) {
for (const key of Object.keys(config)) {
if (Object.keys(used).includes(key) && resourceMeta[key].type === "numeric") {
used[key] += Number(config[key]);
}
else if (diskprefixes.some(prefix => key.startsWith(prefix))) {
const diskInfo = config[key];
if (diskInfo) { // only count if disk exists
used[diskInfo.storage] += Number(diskInfo.size);
}
}
else if (key.startsWith("net") && config[key].includes("rate=")) { // only count net instances with a rate limit
used.network += Number(config[key].split("rate=")[1].split(",")[0]);
}
else if (key.startsWith("hostpci")) {
const deviceInfo = config[key];
if (deviceInfo) { // only count if device exists
used.pci.push(deviceInfo.device_name);
}
}
}
}
return used;
}
/**
* Get meta data for a specific disk. Adds info that is not normally available in a instance's config.
* @param {string} node containing the query disk.

View File

@ -44,7 +44,7 @@ router.get(`/:node(${nodeRegexP})/pci`, async (req, res) => {
return;
}
// get remaining user resources
const userAvailPci = (await getUserResources(req, req.cookies.username)).pci;
const userAvailPci = (await getUserResources(req, req.cookies.username)).pci.nodes[params.node];
// get node avail devices
let nodeAvailPci = await getNodeAvailDevices(params.node, req.cookies);
nodeAvailPci = nodeAvailPci.filter(nodeAvail => userAvailPci.some((userAvail) => {
@ -100,7 +100,7 @@ router.post(`${basePath}/resources`, async (req, res) => {
request.cpu = params.proctype;
}
// check resource approval
if (!await approveResources(req, req.cookies.username, request)) {
if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: "Could not fulfil request." });
res.end();
return;
@ -201,7 +201,7 @@ router.post(`${basePath}/create`, async (req, res) => {
}
}
// check resource approval
if (!await approveResources(req, req.cookies.username, request)) { // check resource approval
if (!await approveResources(req, req.cookies.username, request, params.node)) { // check resource approval
res.status(500).send({ request, error: "Not enough resources to satisfy request." });
res.end();
return;

View File

@ -155,7 +155,7 @@ router.post("/:disk/resize", async (req, res) => {
const request = {};
request[storage] = Number(params.size * 1024 ** 3); // setup request object
// check request approval
if (!await approveResources(req, req.cookies.username, request)) {
if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Storage ${storage} could not fulfill request of size ${params.size}G.` });
res.end();
return;
@ -215,7 +215,7 @@ router.post("/:disk/move", async (req, res) => {
request[dstStorage] = Number(size); // always decrease destination storage by size
}
// check request approval
if (!await approveResources(req, req.cookies.username, request)) {
if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` });
res.end();
return;
@ -331,7 +331,7 @@ router.post("/:disk/create", async (req, res) => {
// setup request
request[params.storage] = Number(params.size * 1024 ** 3);
// check request approval
if (!await approveResources(req, req.cookies.username, request)) {
if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` });
res.end();
return;

View File

@ -57,7 +57,7 @@ router.post("/:netid/create", async (req, res) => {
network: Number(params.rate)
};
// check resource approval
if (!await approveResources(req, req.cookies.username, request)) {
if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` });
res.end();
return;
@ -122,7 +122,7 @@ router.post("/:netid/modify", async (req, res) => {
network: Number(params.rate) - Number(currentNetworkRate)
};
// check resource approval
if (!await approveResources(req, req.cookies.username, request)) {
if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` });
res.end();
return;

View File

@ -108,7 +108,7 @@ router.post("/:hostpci/modify", async (req, res) => {
const deviceData = await getDeviceInfo(params.node, params.device);
const request = { pci: deviceData.device_name };
// check resource approval
if (!await approveResources(req, req.cookies.username, request)) {
if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` });
res.end();
return;
@ -189,7 +189,7 @@ router.post("/create", async (req, res) => {
pci: deviceData.device_name
};
// check resource approval
if (!await approveResources(req, req.cookies.username, request)) {
if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` });
res.end();
return;

View File

@ -3,7 +3,7 @@ import path from "path";
import url from "url";
import * as fs from "fs";
import { getUsedResources, requestPVE } from "./pve.js";
import { requestPVE, getDiskInfo, getDeviceInfo } from "./pve.js";
/**
* Check if a user is authorized to access a specified vm, or the cluster in general.
@ -39,6 +39,69 @@ export async function checkAuth (cookies, res, vmpath = null) {
return auth;
}
/**
* Get the full config of an instance, including searching disk information.
* @param {Object} req ProxmoxAAS API request object.
* @param {Object} instance to get config as object containing node, type, and id.
* @param {Array} diskprefixes Array containing prefixes for disks.
* @returns
*/
async function getFullInstanceConfig (req, instance, diskprefixes) {
const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data;
// fetch all instance disk and device data concurrently
const promises = [];
const mappings = [];
for (const key in config) {
if (diskprefixes.some(prefix => key.startsWith(prefix))) {
promises.push(getDiskInfo(instance.node, config, key));
mappings.push(key);
}
else if (key.startsWith("hostpci")) {
promises.push(getDeviceInfo(instance.node, config[key].split(",")[0]));
mappings.push(key);
}
}
const results = await Promise.all(promises);
results.forEach((e, i) => {
const key = mappings[i];
config[key] = e;
});
config.node = instance.node;
return config;
}
/**
* Get all configs for every instance owned by the user. Uses the expanded config data from getFullInstanceConfig.
* @param {Object} req ProxmoxAAS API request object.
* @param {Object} dbResources data about application resources, to indicate which resources are tracked.
* @returns {Object} k-v pairs of resource name and used amounts
*/
async function getAllInstanceConfigs (req, diskprefixes) {
// get the basic resources list
const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data;
// filter resources by their type, we only want lxc and qemu
const instances = [];
for (const resource of resources) {
if (resource.type === "lxc" || resource.type === "qemu") {
instances.push(resource);
}
}
// get all instance configs, also include detailed disk and device info
const promises = [];
const mappings = [];
for (let i = 0; i < instances.length; i++) {
const instance = instances[i];
const config = getFullInstanceConfig(req, instance, diskprefixes);
promises.push(config);
mappings.push(i);
}
const configs = await Promise.all(promises);
return configs;
}
/**
* Get user resource data including used, available, and maximum resources.
* @param {Object} req ProxmoxAAS API request object.
@ -48,25 +111,149 @@ export async function checkAuth (cookies, res, vmpath = null) {
export async function getUserResources (req, username) {
const db = global.db;
const dbResources = db.getGlobal().resources;
const used = await getUsedResources(req, dbResources);
const userResources = db.getUser(username).resources;
Object.keys(userResources).forEach((k) => {
if (dbResources[k] && dbResources[k].type === "list") {
userResources[k].forEach((listResource) => {
listResource.used = 0;
listResource.avail = listResource.max;
});
used[k].forEach((usedDeviceName) => {
const index = userResources[k].findIndex((availEelement) => usedDeviceName.includes(availEelement.match));
userResources[k][index].used++;
userResources[k][index].avail--;
});
// setup disk prefixes object
const diskprefixes = [];
for (const resourceName of Object.keys(dbResources)) {
if (dbResources[resourceName].type === "storage") {
for (const diskPrefix of dbResources[resourceName].disks) {
diskprefixes.push(diskPrefix);
}
}
}
// setup the user resource object with used and avail for each resource and each resource pool
// also add a total counter for each resource (only used for display, not used to check requests)
for (const resourceName of Object.keys(userResources)) {
if (dbResources[resourceName].type === "list") {
userResources[resourceName].total = [];
userResources[resourceName].global.forEach((e) => {
e.used = 0;
e.avail = e.max;
const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match);
if (index === -1) {
userResources[resourceName].total.push(structuredClone(e));
}
else {
userResources[k].used = used[k];
userResources[k].avail = userResources[k].max - used[k];
userResources[resourceName].total[index].max += e.max;
userResources[resourceName].total[index].avail += e.avail;
}
});
for (const nodeName of Object.keys(userResources[resourceName].nodes)) {
userResources[resourceName].nodes[nodeName].forEach((e) => {
e.used = 0;
e.avail = e.max;
const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match);
if (index === -1) {
userResources[resourceName].total.push(structuredClone(e));
}
else {
userResources[resourceName].total[index].max += e.max;
userResources[resourceName].total[index].avail += e.avail;
}
});
}
}
else {
const total = {
max: 0,
used: 0,
avail: 0
};
userResources[resourceName].global.used = 0;
userResources[resourceName].global.avail = userResources[resourceName].global.max;
total.max += userResources[resourceName].global.max;
total.avail += userResources[resourceName].global.avail;
for (const nodeName of Object.keys(userResources[resourceName].nodes)) {
userResources[resourceName].nodes[nodeName].used = 0;
userResources[resourceName].nodes[nodeName].avail = userResources[resourceName].nodes[nodeName].max;
total.max += userResources[resourceName].nodes[nodeName].max;
total.avail += userResources[resourceName].nodes[nodeName].avail;
}
userResources[resourceName].total = total;
}
}
const configs = await getAllInstanceConfigs(req, diskprefixes);
for (const config of configs) {
const nodeName = config.node;
for (const resourceName of Object.keys(config)) {
// numeric resource type
if (resourceName in dbResources && dbResources[resourceName].type === "numeric") {
const val = Number(config[resourceName]);
// if the instance's node is restricted by this resource, add it to the instance's used value
if (nodeName in userResources[resourceName].nodes) {
userResources[resourceName].nodes[nodeName].used += val;
userResources[resourceName].nodes[nodeName].avail -= val;
}
// otherwise add the resource to the global pool
else {
userResources[resourceName].global.used += val;
userResources[resourceName].global.avail -= val;
}
userResources[resourceName].total.used += val;
userResources[resourceName].total.avail -= val;
}
else if (diskprefixes.some(prefix => resourceName.startsWith(prefix))) {
const diskInfo = config[resourceName];
if (diskInfo) { // only count if disk exists
const val = Number(diskInfo.size);
const storage = diskInfo.storage;
// if the instance's node is restricted by this resource, add it to the instance's used value
if (nodeName in userResources[storage].nodes) {
userResources[storage].nodes[nodeName].used += val;
userResources[storage].nodes[nodeName].avail -= val;
}
// otherwise add the resource to the global pool
else {
userResources[storage].global.used += val;
userResources[storage].global.avail -= val;
}
userResources[storage].total.used += val;
userResources[storage].total.avail -= val;
}
}
else if (resourceName.startsWith("net") && config[resourceName].includes("rate=")) { // only count net instances with a rate limit
const val = Number(config[resourceName].split("rate=")[1].split(",")[0]);
// if the instance's node is restricted by this resource, add it to the instance's used value
if (nodeName in userResources.network.nodes) {
userResources.network.nodes[nodeName].used += val;
userResources.network.nodes[nodeName].avail -= val;
}
// otherwise add the resource to the global pool
else {
userResources.network.global.used += val;
userResources.network.global.avail -= val;
}
userResources.network.total.used += val;
userResources.network.total.avail -= val;
}
else if (resourceName.startsWith("hostpci")) {
const deviceInfo = config[resourceName];
if (deviceInfo) { // only count if device exists
const deviceName = deviceInfo.device_name;
// if the instance's node is restricted by this resource, add it to the instance's used value
if (nodeName in userResources.pci.nodes) {
const index = userResources.pci.nodes[nodeName].findIndex((availEelement) => deviceName.includes(availEelement.match));
userResources.pci.nodes[nodeName][index].used++;
userResources.pci.nodes[nodeName][index].avail--;
}
// otherwise add the resource to the global pool
else {
const index = userResources.pci.global.findIndex((availEelement) => deviceName.includes(availEelement.match));
userResources.pci.global[index].used++;
userResources.pci.global[index].avail--;
}
const index = userResources.pci.total.findIndex((availEelement) => deviceName.includes(availEelement.match));
userResources.pci.total[index].used++;
userResources.pci.total[index].avail--;
}
}
}
}
return userResources;
}
@ -77,29 +264,43 @@ export async function getUserResources (req, username) {
* @param {Object} request k-v pairs of resources and requested amounts
* @returns {boolean} true if the available resources can fullfill the requested resources, false otherwise.
*/
export async function approveResources (req, username, request) {
export async function approveResources (req, username, request, node) {
const db = global.db;
const dbResources = db.getGlobal().resources;
const userResources = await getUserResources(req, username);
let approved = true;
Object.keys(request).forEach((key) => {
if (!(key in userResources)) { // if requested resource is not in avail, block
Object.keys(request).every((key) => {
// if requested resource is not specified in user resources, assume it's not allowed
if (!(key in userResources)) {
approved = false;
return false;
}
else if (dbResources[key].type === "list") { // if the resource type is list, check if the requested resource exists in the list
const index = userResources[key].findIndex((availElement) => request[key].includes(availElement.match));
const inNode = node in userResources[key].nodes;
const resourceData = inNode ? userResources[key].nodes[node] : userResources[key].global;
// if the resource type is list, check if the requested resource exists in the list
if (dbResources[key].type === "list") {
const index = resourceData.findIndex((availElement) => request[key].includes(availElement.match));
// if no matching resource when index == -1, then remaining is -1 otherwise use the remaining value
const avail = index === -1 ? false : userResources[key][index].avail > 0;
const avail = index === -1 ? false : resourceData[index].avail > 0;
if (avail !== dbResources[key].whitelist) {
approved = false;
return false;
}
}
else if (isNaN(userResources[key].avail) || isNaN(request[key])) { // if either the requested or avail resource is NaN, block
// if either the requested or avail resource is NaN, block
else if (isNaN(resourceData.avail) || isNaN(request[key])) {
approved = false;
return false;
}
else if (userResources[key].avail - request[key] < 0) { // if the avail resources is less than the requested resources, block
// if the avail resources is less than the requested resources, block
else if (resourceData.avail - request[key] < 0) {
approved = false;
return false;
}
return true;
});
return approved; // if all requested resources pass, allow
}