add per instance resource quotas,
move getFullInstanceConfig to utils.js, rework resource related utilities to use new quota format
This commit is contained in:
Arthur Lu
parent
02764c8f6a
commit
c2ab19b6d6
102
src/pve.js
102
src/pve.js
@ -79,108 +79,6 @@ export async function handleResponse (node, result, res) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full config of an instance, including searching disk information.
|
||||
* @param {Object} req ProxmoxAAS API request object.
|
||||
* @param {Object} instance to get config as object containing node, type, and id.
|
||||
* @param {Array} diskprefixes Array containing prefixes for disks.
|
||||
* @returns
|
||||
*/
|
||||
async function getFullInstanceConfig (req, instance, diskprefixes) {
|
||||
const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data;
|
||||
// fetch all instance disk and device data concurrently
|
||||
const promises = [];
|
||||
const mappings = [];
|
||||
for (const key in config) {
|
||||
if (diskprefixes.some(prefix => key.startsWith(prefix))) {
|
||||
promises.push(getDiskInfo(instance.node, config, key));
|
||||
mappings.push(key);
|
||||
}
|
||||
else if (key.startsWith("hostpci")) {
|
||||
promises.push(getDeviceInfo(instance.node, config[key].split(",")[0]));
|
||||
mappings.push(key);
|
||||
}
|
||||
}
|
||||
const results = await Promise.all(promises);
|
||||
results.forEach((e, i) => {
|
||||
const key = mappings[i];
|
||||
config[key] = e;
|
||||
});
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the amount of resources used by specified user.
|
||||
* @param {Object} req ProxmoxAAS API request object.
|
||||
* @param {Object} resourceMeta data about application resources, to indicate which resources are tracked.
|
||||
* @returns {Object} k-v pairs of resource name and used amounts
|
||||
*/
|
||||
export async function getUsedResources (req, resourceMeta) {
|
||||
// get the basic resources list
|
||||
const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data;
|
||||
|
||||
// setup the used object and diskPrefixes object
|
||||
const used = {};
|
||||
const diskprefixes = [];
|
||||
for (const resourceName of Object.keys(resourceMeta)) {
|
||||
if (resourceMeta[resourceName].type === "storage") {
|
||||
used[resourceName] = 0;
|
||||
for (const diskPrefix of resourceMeta[resourceName].disks) {
|
||||
diskprefixes.push(diskPrefix);
|
||||
}
|
||||
}
|
||||
else if (resourceMeta[resourceName].type === "list") {
|
||||
used[resourceName] = [];
|
||||
}
|
||||
else {
|
||||
used[resourceName] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// filter resources by their type, we only want lxc and qemu
|
||||
const instances = [];
|
||||
for (const resource of resources) {
|
||||
if (resource.type === "lxc" || resource.type === "qemu") {
|
||||
instances.push(resource);
|
||||
}
|
||||
}
|
||||
|
||||
const promises = [];
|
||||
const mappings = [];
|
||||
for (let i = 0; i < instances.length; i++) {
|
||||
const instance = instances[i];
|
||||
promises.push(getFullInstanceConfig(req, instance, diskprefixes));
|
||||
mappings.push(i);
|
||||
}
|
||||
const configs = await Promise.all(promises);
|
||||
|
||||
// for each instance, sum each resource
|
||||
for (const config of configs) {
|
||||
for (const key of Object.keys(config)) {
|
||||
if (Object.keys(used).includes(key) && resourceMeta[key].type === "numeric") {
|
||||
used[key] += Number(config[key]);
|
||||
}
|
||||
else if (diskprefixes.some(prefix => key.startsWith(prefix))) {
|
||||
const diskInfo = config[key];
|
||||
if (diskInfo) { // only count if disk exists
|
||||
used[diskInfo.storage] += Number(diskInfo.size);
|
||||
}
|
||||
}
|
||||
else if (key.startsWith("net") && config[key].includes("rate=")) { // only count net instances with a rate limit
|
||||
used.network += Number(config[key].split("rate=")[1].split(",")[0]);
|
||||
}
|
||||
else if (key.startsWith("hostpci")) {
|
||||
const deviceInfo = config[key];
|
||||
if (deviceInfo) { // only count if device exists
|
||||
used.pci.push(deviceInfo.device_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return used;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get meta data for a specific disk. Adds info that is not normally available in a instance's config.
|
||||
* @param {string} node containing the query disk.
|
||||
|
||||
@ -44,7 +44,7 @@ router.get(`/:node(${nodeRegexP})/pci`, async (req, res) => {
|
||||
return;
|
||||
}
|
||||
// get remaining user resources
|
||||
const userAvailPci = (await getUserResources(req, req.cookies.username)).pci;
|
||||
const userAvailPci = (await getUserResources(req, req.cookies.username)).pci.nodes[params.node];
|
||||
// get node avail devices
|
||||
let nodeAvailPci = await getNodeAvailDevices(params.node, req.cookies);
|
||||
nodeAvailPci = nodeAvailPci.filter(nodeAvail => userAvailPci.some((userAvail) => {
|
||||
@ -100,7 +100,7 @@ router.post(`${basePath}/resources`, async (req, res) => {
|
||||
request.cpu = params.proctype;
|
||||
}
|
||||
// check resource approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) {
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) {
|
||||
res.status(500).send({ request, error: "Could not fulfil request." });
|
||||
res.end();
|
||||
return;
|
||||
@ -201,7 +201,7 @@ router.post(`${basePath}/create`, async (req, res) => {
|
||||
}
|
||||
}
|
||||
// check resource approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) { // check resource approval
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) { // check resource approval
|
||||
res.status(500).send({ request, error: "Not enough resources to satisfy request." });
|
||||
res.end();
|
||||
return;
|
||||
|
||||
@ -155,7 +155,7 @@ router.post("/:disk/resize", async (req, res) => {
|
||||
const request = {};
|
||||
request[storage] = Number(params.size * 1024 ** 3); // setup request object
|
||||
// check request approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) {
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) {
|
||||
res.status(500).send({ request, error: `Storage ${storage} could not fulfill request of size ${params.size}G.` });
|
||||
res.end();
|
||||
return;
|
||||
@ -215,7 +215,7 @@ router.post("/:disk/move", async (req, res) => {
|
||||
request[dstStorage] = Number(size); // always decrease destination storage by size
|
||||
}
|
||||
// check request approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) {
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) {
|
||||
res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` });
|
||||
res.end();
|
||||
return;
|
||||
@ -331,7 +331,7 @@ router.post("/:disk/create", async (req, res) => {
|
||||
// setup request
|
||||
request[params.storage] = Number(params.size * 1024 ** 3);
|
||||
// check request approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) {
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) {
|
||||
res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` });
|
||||
res.end();
|
||||
return;
|
||||
|
||||
@ -57,7 +57,7 @@ router.post("/:netid/create", async (req, res) => {
|
||||
network: Number(params.rate)
|
||||
};
|
||||
// check resource approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) {
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) {
|
||||
res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` });
|
||||
res.end();
|
||||
return;
|
||||
@ -122,7 +122,7 @@ router.post("/:netid/modify", async (req, res) => {
|
||||
network: Number(params.rate) - Number(currentNetworkRate)
|
||||
};
|
||||
// check resource approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) {
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) {
|
||||
res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` });
|
||||
res.end();
|
||||
return;
|
||||
|
||||
@ -108,7 +108,7 @@ router.post("/:hostpci/modify", async (req, res) => {
|
||||
const deviceData = await getDeviceInfo(params.node, params.device);
|
||||
const request = { pci: deviceData.device_name };
|
||||
// check resource approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) {
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) {
|
||||
res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` });
|
||||
res.end();
|
||||
return;
|
||||
@ -189,7 +189,7 @@ router.post("/create", async (req, res) => {
|
||||
pci: deviceData.device_name
|
||||
};
|
||||
// check resource approval
|
||||
if (!await approveResources(req, req.cookies.username, request)) {
|
||||
if (!await approveResources(req, req.cookies.username, request, params.node)) {
|
||||
res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` });
|
||||
res.end();
|
||||
return;
|
||||
|
||||
247
src/utils.js
247
src/utils.js
@ -3,7 +3,7 @@ import path from "path";
|
||||
import url from "url";
|
||||
import * as fs from "fs";
|
||||
|
||||
import { getUsedResources, requestPVE } from "./pve.js";
|
||||
import { requestPVE, getDiskInfo, getDeviceInfo } from "./pve.js";
|
||||
|
||||
/**
|
||||
* Check if a user is authorized to access a specified vm, or the cluster in general.
|
||||
@ -39,6 +39,69 @@ export async function checkAuth (cookies, res, vmpath = null) {
|
||||
return auth;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full config of an instance, including searching disk information.
|
||||
* @param {Object} req ProxmoxAAS API request object.
|
||||
* @param {Object} instance to get config as object containing node, type, and id.
|
||||
* @param {Array} diskprefixes Array containing prefixes for disks.
|
||||
* @returns
|
||||
*/
|
||||
async function getFullInstanceConfig (req, instance, diskprefixes) {
|
||||
const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data;
|
||||
// fetch all instance disk and device data concurrently
|
||||
const promises = [];
|
||||
const mappings = [];
|
||||
for (const key in config) {
|
||||
if (diskprefixes.some(prefix => key.startsWith(prefix))) {
|
||||
promises.push(getDiskInfo(instance.node, config, key));
|
||||
mappings.push(key);
|
||||
}
|
||||
else if (key.startsWith("hostpci")) {
|
||||
promises.push(getDeviceInfo(instance.node, config[key].split(",")[0]));
|
||||
mappings.push(key);
|
||||
}
|
||||
}
|
||||
const results = await Promise.all(promises);
|
||||
results.forEach((e, i) => {
|
||||
const key = mappings[i];
|
||||
config[key] = e;
|
||||
});
|
||||
config.node = instance.node;
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all configs for every instance owned by the user. Uses the expanded config data from getFullInstanceConfig.
|
||||
* @param {Object} req ProxmoxAAS API request object.
|
||||
* @param {Object} dbResources data about application resources, to indicate which resources are tracked.
|
||||
* @returns {Object} k-v pairs of resource name and used amounts
|
||||
*/
|
||||
async function getAllInstanceConfigs (req, diskprefixes) {
|
||||
// get the basic resources list
|
||||
const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data;
|
||||
|
||||
// filter resources by their type, we only want lxc and qemu
|
||||
const instances = [];
|
||||
for (const resource of resources) {
|
||||
if (resource.type === "lxc" || resource.type === "qemu") {
|
||||
instances.push(resource);
|
||||
}
|
||||
}
|
||||
|
||||
// get all instance configs, also include detailed disk and device info
|
||||
const promises = [];
|
||||
const mappings = [];
|
||||
for (let i = 0; i < instances.length; i++) {
|
||||
const instance = instances[i];
|
||||
const config = getFullInstanceConfig(req, instance, diskprefixes);
|
||||
promises.push(config);
|
||||
mappings.push(i);
|
||||
}
|
||||
const configs = await Promise.all(promises);
|
||||
|
||||
return configs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get user resource data including used, available, and maximum resources.
|
||||
* @param {Object} req ProxmoxAAS API request object.
|
||||
@ -48,25 +111,149 @@ export async function checkAuth (cookies, res, vmpath = null) {
|
||||
export async function getUserResources (req, username) {
|
||||
const db = global.db;
|
||||
const dbResources = db.getGlobal().resources;
|
||||
const used = await getUsedResources(req, dbResources);
|
||||
const userResources = db.getUser(username).resources;
|
||||
Object.keys(userResources).forEach((k) => {
|
||||
if (dbResources[k] && dbResources[k].type === "list") {
|
||||
userResources[k].forEach((listResource) => {
|
||||
listResource.used = 0;
|
||||
listResource.avail = listResource.max;
|
||||
});
|
||||
used[k].forEach((usedDeviceName) => {
|
||||
const index = userResources[k].findIndex((availEelement) => usedDeviceName.includes(availEelement.match));
|
||||
userResources[k][index].used++;
|
||||
userResources[k][index].avail--;
|
||||
|
||||
// setup disk prefixes object
|
||||
const diskprefixes = [];
|
||||
for (const resourceName of Object.keys(dbResources)) {
|
||||
if (dbResources[resourceName].type === "storage") {
|
||||
for (const diskPrefix of dbResources[resourceName].disks) {
|
||||
diskprefixes.push(diskPrefix);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setup the user resource object with used and avail for each resource and each resource pool
|
||||
// also add a total counter for each resource (only used for display, not used to check requests)
|
||||
for (const resourceName of Object.keys(userResources)) {
|
||||
if (dbResources[resourceName].type === "list") {
|
||||
userResources[resourceName].total = [];
|
||||
userResources[resourceName].global.forEach((e) => {
|
||||
e.used = 0;
|
||||
e.avail = e.max;
|
||||
const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match);
|
||||
if (index === -1) {
|
||||
userResources[resourceName].total.push(structuredClone(e));
|
||||
}
|
||||
else {
|
||||
userResources[resourceName].total[index].max += e.max;
|
||||
userResources[resourceName].total[index].avail += e.avail;
|
||||
}
|
||||
});
|
||||
for (const nodeName of Object.keys(userResources[resourceName].nodes)) {
|
||||
userResources[resourceName].nodes[nodeName].forEach((e) => {
|
||||
e.used = 0;
|
||||
e.avail = e.max;
|
||||
const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match);
|
||||
if (index === -1) {
|
||||
userResources[resourceName].total.push(structuredClone(e));
|
||||
}
|
||||
else {
|
||||
userResources[resourceName].total[index].max += e.max;
|
||||
userResources[resourceName].total[index].avail += e.avail;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
else {
|
||||
userResources[k].used = used[k];
|
||||
userResources[k].avail = userResources[k].max - used[k];
|
||||
const total = {
|
||||
max: 0,
|
||||
used: 0,
|
||||
avail: 0
|
||||
};
|
||||
userResources[resourceName].global.used = 0;
|
||||
userResources[resourceName].global.avail = userResources[resourceName].global.max;
|
||||
total.max += userResources[resourceName].global.max;
|
||||
total.avail += userResources[resourceName].global.avail;
|
||||
for (const nodeName of Object.keys(userResources[resourceName].nodes)) {
|
||||
userResources[resourceName].nodes[nodeName].used = 0;
|
||||
userResources[resourceName].nodes[nodeName].avail = userResources[resourceName].nodes[nodeName].max;
|
||||
total.max += userResources[resourceName].nodes[nodeName].max;
|
||||
total.avail += userResources[resourceName].nodes[nodeName].avail;
|
||||
}
|
||||
userResources[resourceName].total = total;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const configs = await getAllInstanceConfigs(req, diskprefixes);
|
||||
|
||||
for (const config of configs) {
|
||||
const nodeName = config.node;
|
||||
for (const resourceName of Object.keys(config)) {
|
||||
// numeric resource type
|
||||
if (resourceName in dbResources && dbResources[resourceName].type === "numeric") {
|
||||
const val = Number(config[resourceName]);
|
||||
// if the instance's node is restricted by this resource, add it to the instance's used value
|
||||
if (nodeName in userResources[resourceName].nodes) {
|
||||
userResources[resourceName].nodes[nodeName].used += val;
|
||||
userResources[resourceName].nodes[nodeName].avail -= val;
|
||||
}
|
||||
// otherwise add the resource to the global pool
|
||||
else {
|
||||
userResources[resourceName].global.used += val;
|
||||
userResources[resourceName].global.avail -= val;
|
||||
}
|
||||
userResources[resourceName].total.used += val;
|
||||
userResources[resourceName].total.avail -= val;
|
||||
}
|
||||
else if (diskprefixes.some(prefix => resourceName.startsWith(prefix))) {
|
||||
const diskInfo = config[resourceName];
|
||||
if (diskInfo) { // only count if disk exists
|
||||
const val = Number(diskInfo.size);
|
||||
const storage = diskInfo.storage;
|
||||
// if the instance's node is restricted by this resource, add it to the instance's used value
|
||||
if (nodeName in userResources[storage].nodes) {
|
||||
userResources[storage].nodes[nodeName].used += val;
|
||||
userResources[storage].nodes[nodeName].avail -= val;
|
||||
}
|
||||
// otherwise add the resource to the global pool
|
||||
else {
|
||||
userResources[storage].global.used += val;
|
||||
userResources[storage].global.avail -= val;
|
||||
}
|
||||
userResources[storage].total.used += val;
|
||||
userResources[storage].total.avail -= val;
|
||||
}
|
||||
}
|
||||
else if (resourceName.startsWith("net") && config[resourceName].includes("rate=")) { // only count net instances with a rate limit
|
||||
const val = Number(config[resourceName].split("rate=")[1].split(",")[0]);
|
||||
// if the instance's node is restricted by this resource, add it to the instance's used value
|
||||
if (nodeName in userResources.network.nodes) {
|
||||
userResources.network.nodes[nodeName].used += val;
|
||||
userResources.network.nodes[nodeName].avail -= val;
|
||||
}
|
||||
// otherwise add the resource to the global pool
|
||||
else {
|
||||
userResources.network.global.used += val;
|
||||
userResources.network.global.avail -= val;
|
||||
}
|
||||
userResources.network.total.used += val;
|
||||
userResources.network.total.avail -= val;
|
||||
}
|
||||
else if (resourceName.startsWith("hostpci")) {
|
||||
const deviceInfo = config[resourceName];
|
||||
if (deviceInfo) { // only count if device exists
|
||||
const deviceName = deviceInfo.device_name;
|
||||
// if the instance's node is restricted by this resource, add it to the instance's used value
|
||||
if (nodeName in userResources.pci.nodes) {
|
||||
const index = userResources.pci.nodes[nodeName].findIndex((availEelement) => deviceName.includes(availEelement.match));
|
||||
userResources.pci.nodes[nodeName][index].used++;
|
||||
userResources.pci.nodes[nodeName][index].avail--;
|
||||
}
|
||||
// otherwise add the resource to the global pool
|
||||
else {
|
||||
const index = userResources.pci.global.findIndex((availEelement) => deviceName.includes(availEelement.match));
|
||||
userResources.pci.global[index].used++;
|
||||
userResources.pci.global[index].avail--;
|
||||
}
|
||||
const index = userResources.pci.total.findIndex((availEelement) => deviceName.includes(availEelement.match));
|
||||
userResources.pci.total[index].used++;
|
||||
userResources.pci.total[index].avail--;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return userResources;
|
||||
}
|
||||
|
||||
@ -77,29 +264,43 @@ export async function getUserResources (req, username) {
|
||||
* @param {Object} request k-v pairs of resources and requested amounts
|
||||
* @returns {boolean} true if the available resources can fullfill the requested resources, false otherwise.
|
||||
*/
|
||||
export async function approveResources (req, username, request) {
|
||||
export async function approveResources (req, username, request, node) {
|
||||
const db = global.db;
|
||||
const dbResources = db.getGlobal().resources;
|
||||
const userResources = await getUserResources(req, username);
|
||||
let approved = true;
|
||||
Object.keys(request).forEach((key) => {
|
||||
if (!(key in userResources)) { // if requested resource is not in avail, block
|
||||
Object.keys(request).every((key) => {
|
||||
// if requested resource is not specified in user resources, assume it's not allowed
|
||||
if (!(key in userResources)) {
|
||||
approved = false;
|
||||
return false;
|
||||
}
|
||||
else if (dbResources[key].type === "list") { // if the resource type is list, check if the requested resource exists in the list
|
||||
const index = userResources[key].findIndex((availElement) => request[key].includes(availElement.match));
|
||||
|
||||
const inNode = node in userResources[key].nodes;
|
||||
const resourceData = inNode ? userResources[key].nodes[node] : userResources[key].global;
|
||||
|
||||
// if the resource type is list, check if the requested resource exists in the list
|
||||
if (dbResources[key].type === "list") {
|
||||
const index = resourceData.findIndex((availElement) => request[key].includes(availElement.match));
|
||||
// if no matching resource when index == -1, then remaining is -1 otherwise use the remaining value
|
||||
const avail = index === -1 ? false : userResources[key][index].avail > 0;
|
||||
const avail = index === -1 ? false : resourceData[index].avail > 0;
|
||||
if (avail !== dbResources[key].whitelist) {
|
||||
approved = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (isNaN(userResources[key].avail) || isNaN(request[key])) { // if either the requested or avail resource is NaN, block
|
||||
// if either the requested or avail resource is NaN, block
|
||||
else if (isNaN(resourceData.avail) || isNaN(request[key])) {
|
||||
approved = false;
|
||||
return false;
|
||||
}
|
||||
else if (userResources[key].avail - request[key] < 0) { // if the avail resources is less than the requested resources, block
|
||||
// if the avail resources is less than the requested resources, block
|
||||
else if (resourceData.avail - request[key] < 0) {
|
||||
approved = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
return approved; // if all requested resources pass, allow
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user