tronnet/ProxmoxAAS-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add per instance resource quotas,

Browse Source 
move getFullInstanceConfig to utils.js,
rework resource related utilities to use new quota format
This commit is contained in:
Arthur Lu 2023-11-15 19:57:59 +00:00
parent 23cb635b75
commit f40d1aee79
7 changed files with 235 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
src/pve.js
View File

@ -79,108 +79,6 @@ export async function handleResponse (node, result, res) {
} }
} }
/**
* Get the full config of an instance, including searching disk information.
* @param {Object} req ProxmoxAAS API request object.
* @param {Object} instance to get config as object containing node, type, and id.
* @param {Array} diskprefixes Array containing prefixes for disks.
* @returns
*/
async function getFullInstanceConfig (req, instance, diskprefixes) {
const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data;
// fetch all instance disk and device data concurrently
const promises = [];
const mappings = [];
for (const key in config) {
if (diskprefixes.some(prefix => key.startsWith(prefix))) {
promises.push(getDiskInfo(instance.node, config, key));
mappings.push(key);
}
else if (key.startsWith("hostpci")) {
promises.push(getDeviceInfo(instance.node, config[key].split(",")[0]));
mappings.push(key);
}
}
const results = await Promise.all(promises);
results.forEach((e, i) => {
const key = mappings[i];
config[key] = e;
});
return config;
}
/**
* Get the amount of resources used by specified user.
* @param {Object} req ProxmoxAAS API request object.
* @param {Object} resourceMeta data about application resources, to indicate which resources are tracked.
* @returns {Object} k-v pairs of resource name and used amounts
*/
export async function getUsedResources (req, resourceMeta) {
// get the basic resources list
const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data;
// setup the used object and diskPrefixes object
const used = {};
const diskprefixes = [];
for (const resourceName of Object.keys(resourceMeta)) {
if (resourceMeta[resourceName].type === "storage") {
used[resourceName] = 0;
for (const diskPrefix of resourceMeta[resourceName].disks) {
diskprefixes.push(diskPrefix);
}
}
else if (resourceMeta[resourceName].type === "list") {
used[resourceName] = [];
}
else {
used[resourceName] = 0;
}
}
// filter resources by their type, we only want lxc and qemu
const instances = [];
for (const resource of resources) {
if (resource.type === "lxc" || resource.type === "qemu") {
instances.push(resource);
}
}
const promises = [];
const mappings = [];
for (let i = 0; i < instances.length; i++) {
const instance = instances[i];
promises.push(getFullInstanceConfig(req, instance, diskprefixes));
mappings.push(i);
}
const configs = await Promise.all(promises);
// for each instance, sum each resource
for (const config of configs) {
for (const key of Object.keys(config)) {
if (Object.keys(used).includes(key) && resourceMeta[key].type === "numeric") {
used[key] += Number(config[key]);
}
else if (diskprefixes.some(prefix => key.startsWith(prefix))) {
const diskInfo = config[key];
if (diskInfo) { // only count if disk exists
used[diskInfo.storage] += Number(diskInfo.size);
}
}
else if (key.startsWith("net") && config[key].includes("rate=")) { // only count net instances with a rate limit
used.network += Number(config[key].split("rate=")[1].split(",")[0]);
}
else if (key.startsWith("hostpci")) {
const deviceInfo = config[key];
if (deviceInfo) { // only count if device exists
used.pci.push(deviceInfo.device_name);
}
}
}
}
return used;
}
/** /**
* Get meta data for a specific disk. Adds info that is not normally available in a instance's config. * Get meta data for a specific disk. Adds info that is not normally available in a instance's config.
* @param {string} node containing the query disk. * @param {string} node containing the query disk.

2
src/routes/auth.js
View File

@ -67,7 +67,7 @@ router.post("/password", async (req, res) => {
password: req.body.password, password: req.body.password,
userid: req.cookies.username userid: req.cookies.username
}; };
const userRealm = params.userid.split("@").at(-1); const userRealm = params.userid.split("@").at(-1);
const domains = (await requestPVE("/access/domains", "GET", pveAPIToken)).data.data; const domains = (await requestPVE("/access/domains", "GET", pveAPIToken)).data.data;
const realm = domains.find((e) => e.realm === userRealm); const realm = domains.find((e) => e.realm === userRealm);

6
src/routes/cluster.js
View File

@ -44,7 +44,7 @@ router.get(`/:node(${nodeRegexP})/pci`, async (req, res) => {
return; return;
} }
// get remaining user resources // get remaining user resources
const userAvailPci = (await getUserResources(req, req.cookies.username)).pci; const userAvailPci = (await getUserResources(req, req.cookies.username)).pci.nodes[params.node];
// get node avail devices // get node avail devices
let nodeAvailPci = await getNodeAvailDevices(params.node, req.cookies); let nodeAvailPci = await getNodeAvailDevices(params.node, req.cookies);
nodeAvailPci = nodeAvailPci.filter(nodeAvail => userAvailPci.some((userAvail) => { nodeAvailPci = nodeAvailPci.filter(nodeAvail => userAvailPci.some((userAvail) => {
@ -100,7 +100,7 @@ router.post(`${basePath}/resources`, async (req, res) => {
request.cpu = params.proctype; request.cpu = params.proctype;
} }
// check resource approval // check resource approval
if (!await approveResources(req, req.cookies.username, request)) { if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: "Could not fulfil request." }); res.status(500).send({ request, error: "Could not fulfil request." });
res.end(); res.end();
return; return;
@ -201,7 +201,7 @@ router.post(`${basePath}/create`, async (req, res) => {
} }
} }
// check resource approval // check resource approval
if (!await approveResources(req, req.cookies.username, request)) { // check resource approval if (!await approveResources(req, req.cookies.username, request, params.node)) { // check resource approval
res.status(500).send({ request, error: "Not enough resources to satisfy request." }); res.status(500).send({ request, error: "Not enough resources to satisfy request." });
res.end(); res.end();
return; return;

6
src/routes/cluster/disk.js
View File

@ -155,7 +155,7 @@ router.post("/:disk/resize", async (req, res) => {
const request = {}; const request = {};
request[storage] = Number(params.size * 1024 ** 3); // setup request object request[storage] = Number(params.size * 1024 ** 3); // setup request object
// check request approval // check request approval
if (!await approveResources(req, req.cookies.username, request)) { if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Storage ${storage} could not fulfill request of size ${params.size}G.` }); res.status(500).send({ request, error: `Storage ${storage} could not fulfill request of size ${params.size}G.` });
res.end(); res.end();
return; return;
@ -215,7 +215,7 @@ router.post("/:disk/move", async (req, res) => {
request[dstStorage] = Number(size); // always decrease destination storage by size request[dstStorage] = Number(size); // always decrease destination storage by size
} }
// check request approval // check request approval
if (!await approveResources(req, req.cookies.username, request)) { if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` }); res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` });
res.end(); res.end();
return; return;
@ -331,7 +331,7 @@ router.post("/:disk/create", async (req, res) => {
// setup request // setup request
request[params.storage] = Number(params.size * 1024 ** 3); request[params.storage] = Number(params.size * 1024 ** 3);
// check request approval // check request approval
if (!await approveResources(req, req.cookies.username, request)) { if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` }); res.status(500).send({ request, error: `Storage ${params.storage} could not fulfill request of size ${params.size}G.` });
res.end(); res.end();
return; return;

4
src/routes/cluster/net.js
View File

@ -57,7 +57,7 @@ router.post("/:netid/create", async (req, res) => {
network: Number(params.rate) network: Number(params.rate)
}; };
// check resource approval // check resource approval
if (!await approveResources(req, req.cookies.username, request)) { if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` }); res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` });
res.end(); res.end();
return; return;
@ -122,7 +122,7 @@ router.post("/:netid/modify", async (req, res) => {
network: Number(params.rate) - Number(currentNetworkRate) network: Number(params.rate) - Number(currentNetworkRate)
}; };
// check resource approval // check resource approval
if (!await approveResources(req, req.cookies.username, request)) { if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` }); res.status(500).send({ request, error: `Could not fulfil network request of ${params.rate}MB/s.` });
res.end(); res.end();
return; return;

4
src/routes/cluster/pci.js
View File

@ -108,7 +108,7 @@ router.post("/:hostpci/modify", async (req, res) => {
const deviceData = await getDeviceInfo(params.node, params.device); const deviceData = await getDeviceInfo(params.node, params.device);
const request = { pci: deviceData.device_name }; const request = { pci: deviceData.device_name };
// check resource approval // check resource approval
if (!await approveResources(req, req.cookies.username, request)) { if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` }); res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` });
res.end(); res.end();
return; return;
@ -189,7 +189,7 @@ router.post("/create", async (req, res) => {
pci: deviceData.device_name pci: deviceData.device_name
}; };
// check resource approval // check resource approval
if (!await approveResources(req, req.cookies.username, request)) { if (!await approveResources(req, req.cookies.username, request, params.node)) {
res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` }); res.status(500).send({ request, error: `Could not fulfil request for ${deviceData.device_name}.` });
res.end(); res.end();
return; return;

247
src/utils.js
View File

@ -3,7 +3,7 @@ import path from "path";
import url from "url"; import url from "url";
import * as fs from "fs"; import * as fs from "fs";
import { getUsedResources, requestPVE } from "./pve.js"; import { requestPVE, getDiskInfo, getDeviceInfo } from "./pve.js";
/** /**
* Check if a user is authorized to access a specified vm, or the cluster in general. * Check if a user is authorized to access a specified vm, or the cluster in general.
@ -39,6 +39,69 @@ export async function checkAuth (cookies, res, vmpath = null) {
return auth; return auth;
} }
/**
* Get the full config of an instance, including searching disk information.
* @param {Object} req ProxmoxAAS API request object.
* @param {Object} instance to get config as object containing node, type, and id.
* @param {Array} diskprefixes Array containing prefixes for disks.
* @returns
*/
async function getFullInstanceConfig (req, instance, diskprefixes) {
const config = (await requestPVE(`/nodes/${instance.node}/${instance.type}/${instance.vmid}/config`, "GET", { cookies: req.cookies })).data.data;
// fetch all instance disk and device data concurrently
const promises = [];
const mappings = [];
for (const key in config) {
if (diskprefixes.some(prefix => key.startsWith(prefix))) {
promises.push(getDiskInfo(instance.node, config, key));
mappings.push(key);
}
else if (key.startsWith("hostpci")) {
promises.push(getDeviceInfo(instance.node, config[key].split(",")[0]));
mappings.push(key);
}
}
const results = await Promise.all(promises);
results.forEach((e, i) => {
const key = mappings[i];
config[key] = e;
});
config.node = instance.node;
return config;
}
/**
* Get all configs for every instance owned by the user. Uses the expanded config data from getFullInstanceConfig.
* @param {Object} req ProxmoxAAS API request object.
* @param {Object} dbResources data about application resources, to indicate which resources are tracked.
* @returns {Object} k-v pairs of resource name and used amounts
*/
async function getAllInstanceConfigs (req, diskprefixes) {
// get the basic resources list
const resources = (await requestPVE("/cluster/resources", "GET", { cookies: req.cookies })).data.data;
// filter resources by their type, we only want lxc and qemu
const instances = [];
for (const resource of resources) {
if (resource.type === "lxc" || resource.type === "qemu") {
instances.push(resource);
}
}
// get all instance configs, also include detailed disk and device info
const promises = [];
const mappings = [];
for (let i = 0; i < instances.length; i++) {
const instance = instances[i];
const config = getFullInstanceConfig(req, instance, diskprefixes);
promises.push(config);
mappings.push(i);
}
const configs = await Promise.all(promises);
return configs;
}
/** /**
* Get user resource data including used, available, and maximum resources. * Get user resource data including used, available, and maximum resources.
* @param {Object} req ProxmoxAAS API request object. * @param {Object} req ProxmoxAAS API request object.
@ -48,25 +111,149 @@ export async function checkAuth (cookies, res, vmpath = null) {
export async function getUserResources (req, username) { export async function getUserResources (req, username) {
const db = global.db; const db = global.db;
const dbResources = db.getGlobal().resources; const dbResources = db.getGlobal().resources;
const used = await getUsedResources(req, dbResources);
const userResources = db.getUser(username).resources; const userResources = db.getUser(username).resources;
Object.keys(userResources).forEach((k) => {
if (dbResources[k] && dbResources[k].type === "list") { // setup disk prefixes object
userResources[k].forEach((listResource) => { const diskprefixes = [];
listResource.used = 0; for (const resourceName of Object.keys(dbResources)) {
listResource.avail = listResource.max; if (dbResources[resourceName].type === "storage") {
}); for (const diskPrefix of dbResources[resourceName].disks) {
used[k].forEach((usedDeviceName) => { diskprefixes.push(diskPrefix);
const index = userResources[k].findIndex((availEelement) => usedDeviceName.includes(availEelement.match)); }
userResources[k][index].used++; }
userResources[k][index].avail--; }
// setup the user resource object with used and avail for each resource and each resource pool
// also add a total counter for each resource (only used for display, not used to check requests)
for (const resourceName of Object.keys(userResources)) {
if (dbResources[resourceName].type === "list") {
userResources[resourceName].total = [];
userResources[resourceName].global.forEach((e) => {
e.used = 0;
e.avail = e.max;
const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match);
if (index === -1) {
userResources[resourceName].total.push(structuredClone(e));
}
else {
userResources[resourceName].total[index].max += e.max;
userResources[resourceName].total[index].avail += e.avail;
}
}); });
for (const nodeName of Object.keys(userResources[resourceName].nodes)) {
userResources[resourceName].nodes[nodeName].forEach((e) => {
e.used = 0;
e.avail = e.max;
const index = userResources[resourceName].total.findIndex((availEelement) => e.match === availEelement.match);
if (index === -1) {
userResources[resourceName].total.push(structuredClone(e));
}
else {
userResources[resourceName].total[index].max += e.max;
userResources[resourceName].total[index].avail += e.avail;
}
});
}
} }
else { else {
userResources[k].used = used[k]; const total = {
userResources[k].avail = userResources[k].max - used[k]; max: 0,
used: 0,
avail: 0
};
userResources[resourceName].global.used = 0;
userResources[resourceName].global.avail = userResources[resourceName].global.max;
total.max += userResources[resourceName].global.max;
total.avail += userResources[resourceName].global.avail;
for (const nodeName of Object.keys(userResources[resourceName].nodes)) {
userResources[resourceName].nodes[nodeName].used = 0;
userResources[resourceName].nodes[nodeName].avail = userResources[resourceName].nodes[nodeName].max;
total.max += userResources[resourceName].nodes[nodeName].max;
total.avail += userResources[resourceName].nodes[nodeName].avail;
}
userResources[resourceName].total = total;
} }
}); }
const configs = await getAllInstanceConfigs(req, diskprefixes);
for (const config of configs) {
const nodeName = config.node;
for (const resourceName of Object.keys(config)) {
// numeric resource type
if (resourceName in dbResources && dbResources[resourceName].type === "numeric") {
const val = Number(config[resourceName]);
// if the instance's node is restricted by this resource, add it to the instance's used value
if (nodeName in userResources[resourceName].nodes) {
userResources[resourceName].nodes[nodeName].used += val;
userResources[resourceName].nodes[nodeName].avail -= val;
}
// otherwise add the resource to the global pool
else {
userResources[resourceName].global.used += val;
userResources[resourceName].global.avail -= val;
}
userResources[resourceName].total.used += val;
userResources[resourceName].total.avail -= val;
}
else if (diskprefixes.some(prefix => resourceName.startsWith(prefix))) {
const diskInfo = config[resourceName];
if (diskInfo) { // only count if disk exists
const val = Number(diskInfo.size);
const storage = diskInfo.storage;
// if the instance's node is restricted by this resource, add it to the instance's used value
if (nodeName in userResources[storage].nodes) {
userResources[storage].nodes[nodeName].used += val;
userResources[storage].nodes[nodeName].avail -= val;
}
// otherwise add the resource to the global pool
else {
userResources[storage].global.used += val;
userResources[storage].global.avail -= val;
}
userResources[storage].total.used += val;
userResources[storage].total.avail -= val;
}
}
else if (resourceName.startsWith("net") && config[resourceName].includes("rate=")) { // only count net instances with a rate limit
const val = Number(config[resourceName].split("rate=")[1].split(",")[0]);
// if the instance's node is restricted by this resource, add it to the instance's used value
if (nodeName in userResources.network.nodes) {
userResources.network.nodes[nodeName].used += val;
userResources.network.nodes[nodeName].avail -= val;
}
// otherwise add the resource to the global pool
else {
userResources.network.global.used += val;
userResources.network.global.avail -= val;
}
userResources.network.total.used += val;
userResources.network.total.avail -= val;
}
else if (resourceName.startsWith("hostpci")) {
const deviceInfo = config[resourceName];
if (deviceInfo) { // only count if device exists
const deviceName = deviceInfo.device_name;
// if the instance's node is restricted by this resource, add it to the instance's used value
if (nodeName in userResources.pci.nodes) {
const index = userResources.pci.nodes[nodeName].findIndex((availEelement) => deviceName.includes(availEelement.match));
userResources.pci.nodes[nodeName][index].used++;
userResources.pci.nodes[nodeName][index].avail--;
}
// otherwise add the resource to the global pool
else {
const index = userResources.pci.global.findIndex((availEelement) => deviceName.includes(availEelement.match));
userResources.pci.global[index].used++;
userResources.pci.global[index].avail--;
}
const index = userResources.pci.total.findIndex((availEelement) => deviceName.includes(availEelement.match));
userResources.pci.total[index].used++;
userResources.pci.total[index].avail--;
}
}
}
}
return userResources; return userResources;
} }
@ -77,29 +264,43 @@ export async function getUserResources (req, username) {
* @param {Object} request k-v pairs of resources and requested amounts * @param {Object} request k-v pairs of resources and requested amounts
* @returns {boolean} true if the available resources can fullfill the requested resources, false otherwise. * @returns {boolean} true if the available resources can fullfill the requested resources, false otherwise.
*/ */
export async function approveResources (req, username, request) { export async function approveResources (req, username, request, node) {
const db = global.db; const db = global.db;
const dbResources = db.getGlobal().resources; const dbResources = db.getGlobal().resources;
const userResources = await getUserResources(req, username); const userResources = await getUserResources(req, username);
let approved = true; let approved = true;
Object.keys(request).forEach((key) => { Object.keys(request).every((key) => {
if (!(key in userResources)) { // if requested resource is not in avail, block // if requested resource is not specified in user resources, assume it's not allowed
if (!(key in userResources)) {
approved = false; approved = false;
return false;
} }
else if (dbResources[key].type === "list") { // if the resource type is list, check if the requested resource exists in the list
const index = userResources[key].findIndex((availElement) => request[key].includes(availElement.match)); const inNode = node in userResources[key].nodes;
const resourceData = inNode ? userResources[key].nodes[node] : userResources[key].global;
// if the resource type is list, check if the requested resource exists in the list
if (dbResources[key].type === "list") {
const index = resourceData.findIndex((availElement) => request[key].includes(availElement.match));
// if no matching resource when index == -1, then remaining is -1 otherwise use the remaining value // if no matching resource when index == -1, then remaining is -1 otherwise use the remaining value
const avail = index === -1 ? false : userResources[key][index].avail > 0; const avail = index === -1 ? false : resourceData[index].avail > 0;
if (avail !== dbResources[key].whitelist) { if (avail !== dbResources[key].whitelist) {
approved = false; approved = false;
return false;
} }
} }
else if (isNaN(userResources[key].avail) || isNaN(request[key])) { // if either the requested or avail resource is NaN, block // if either the requested or avail resource is NaN, block
else if (isNaN(resourceData.avail) || isNaN(request[key])) {
approved = false; approved = false;
return false;
} }
else if (userResources[key].avail - request[key] < 0) { // if the avail resources is less than the requested resources, block // if the avail resources is less than the requested resources, block
else if (resourceData.avail - request[key] < 0) {
approved = false; approved = false;
return false;
} }
return true;
}); });
return approved; // if all requested resources pass, allow return approved; // if all requested resources pass, allow
} }