diff --git a/packages/grid_client/docs/deployment_flow.md b/packages/grid_client/docs/deployment_flow.md new file mode 100644 index 0000000000..3eacabbdf2 --- /dev/null +++ b/packages/grid_client/docs/deployment_flow.md @@ -0,0 +1,117 @@ +# Deployment Flow + +This document outlines the process for determining whether to deploy on a zos3 or a zos4 node. + +## Machine Model Initialization + +The deployment process begins with initializing the machine model as follows: + +```ts +const vms: MachinesModel = { + name: "newMY", + network: { + name: "hellotest", + ip_range: "10.249.0.0/16", + myceliumSeeds: [ + { + nodeId: 168, + seed: "050d109829d8492d48bfb33b711056080571c69e46bfde6b4294c4c5bf468a76", //(HexSeed of length 32) + }, + ], + }, + machines: [ + { + name: "testvmMY", + node_id: 168, + disks: [ + { + name: "wedDisk", + size: 8, + mountpoint: "/testdisk", + }, + ], + public_ip: false, + public_ip6: false, + planetary: true, + mycelium: true, + myceliumSeed: "1e1404279b3d", //(HexSeed of length 6) + cpu: 1, + memory: 1024 * 2, + rootfs_size: 0, + flist: "https://hub.grid.tf/tf-official-apps/base:latest.flist", + entrypoint: "/sbin/zinit init", + env: { + SSH_KEY: config.ssh_key, + }, + }, + ], + metadata: "", + description: "test deploying single VM with mycelium via ts grid3 client", +}; +``` + +## Deployment Execution + +- The next step is invoking the `deploy` function: + - Takes the `MachinesModel` object as a parameter + - Checks if a machine with the same name already exists and if so throws an error + - If not it calls the `_createDeployment` function along some othe functions + - Finally it returns the created contracts and the wiregaurd configuration + +```ts +await client.machines.deploy(vms); +``` + +- The `_createDeployment` function: + - Takes the `MachinesModel` object as a parameter + - Retrieves the features of the node using: `await this.rmb.request([nodeTwinId], "zos.system.node_features_get", "", 20, 3);` + - Examines the retrieved features to determine the network's primitive type (`Network` or `ZNetworkLight`) and initializes it accordingly. + - Sets the contractMetadata based on the network type. + - Invokes the `create` function + +```ts +await this._createDeployment(options); +``` + +- The `create` function + - Validates or assigns IP addresses based on the network type. + - Determines network type (`network` or `network-light`) based on node features. + - Adds access points and updates network configurations as necessary. + - Initialize the VM primitive (`VMPrimitive` or `VMLightPrimitive`) based on the network type + - Configures the VM with networking, storage, and environment variables. + - Generates a Mycelium seed if not provided. + - Generate the deployments + - Returns the deployments and wiregaurd configurations + +```ts +await this.vm.create( + machine.name, + machine.node_id, + machine.flist, + machine.cpu, + machine.memory, + machine.rootfs_size, + machine.disks!, + machine.public_ip, + machine.public_ip6!, + machine.planetary, + machine.mycelium, + machine.myceliumSeed!, + network, + options.network.myceliumSeeds!, + machine.entrypoint, + machine.env, + contractMetadata, + options.metadata, + options.description, + machine.qsfs_disks, + this.config.projectName, + options.network.addAccess, + options.network.accessNodeId, + machine.ip, + machine.corex, + machine.solutionProviderId!, + machine.zlogsOutput, + machine.gpus, +); +``` diff --git a/packages/grid_client/scripts/applications/casperlabs.ts b/packages/grid_client/scripts/applications/casperlabs.ts index eccd0f3f58..424db2d056 100644 --- a/packages/grid_client/scripts/applications/casperlabs.ts +++ b/packages/grid_client/scripts/applications/casperlabs.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -55,6 +55,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/discourse.ts b/packages/grid_client/scripts/applications/discourse.ts index dad2a7cfda..749610676c 100644 --- a/packages/grid_client/scripts/applications/discourse.ts +++ b/packages/grid_client/scripts/applications/discourse.ts @@ -1,7 +1,7 @@ import { Buffer } from "buffer"; import TweetNACL from "tweetnacl"; -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -63,6 +63,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.ip, Features.ipv4, Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/funkwhale.ts b/packages/grid_client/scripts/applications/funkwhale.ts index 6595362b53..77d91d5430 100644 --- a/packages/grid_client/scripts/applications/funkwhale.ts +++ b/packages/grid_client/scripts/applications/funkwhale.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -55,6 +55,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/mattermost.ts b/packages/grid_client/scripts/applications/mattermost.ts index 39fda2510f..0dab539c01 100644 --- a/packages/grid_client/scripts/applications/mattermost.ts +++ b/packages/grid_client/scripts/applications/mattermost.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -47,6 +47,8 @@ async function main() { const grid3 = await getClient(`mattermost/${name}`); const subdomain = "mm" + grid3.twinId + name; const instanceCapacity = { cru: 1, mru: 2, sru: 15 }; // Update the instance capacity values according to your requirements. + // Change to true if smtp is configured + const smtp = false; //VMNode Selection const vmQueryOptions: FilterOptions = { @@ -55,6 +57,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: smtp ? [Features.ip, Features.ipv4, Features.wireguard] : [], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/nextcloud.ts b/packages/grid_client/scripts/applications/nextcloud.ts index 9fb330a67e..20a554186f 100644 --- a/packages/grid_client/scripts/applications/nextcloud.ts +++ b/packages/grid_client/scripts/applications/nextcloud.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -55,6 +55,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/nodepilot.ts b/packages/grid_client/scripts/applications/nodepilot.ts index d6bf363919..fbc28ecf1e 100644 --- a/packages/grid_client/scripts/applications/nodepilot.ts +++ b/packages/grid_client/scripts/applications/nodepilot.ts @@ -1,4 +1,4 @@ -import { FilterOptions, MachinesModel } from "../../src"; +import { Features, FilterOptions, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -35,6 +35,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.ip, Features.ipv4, Features.wireguard], }; const nodes = await grid3.capacity.filterNodes(vmQueryOptions); const vmNode = await pingNodes(grid3, nodes); diff --git a/packages/grid_client/scripts/applications/peertube.ts b/packages/grid_client/scripts/applications/peertube.ts index f1c13608ba..09895bb552 100644 --- a/packages/grid_client/scripts/applications/peertube.ts +++ b/packages/grid_client/scripts/applications/peertube.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -55,6 +55,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/static_website.ts b/packages/grid_client/scripts/applications/static_website.ts index ac67cad3da..037001b7f9 100644 --- a/packages/grid_client/scripts/applications/static_website.ts +++ b/packages/grid_client/scripts/applications/static_website.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, GridClient, MachinesModel, NodeInfo } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, GridClient, MachinesModel, NodeInfo } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -55,6 +55,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/subsquid.ts b/packages/grid_client/scripts/applications/subsquid.ts index 4e864ba232..8fe1a1cb90 100644 --- a/packages/grid_client/scripts/applications/subsquid.ts +++ b/packages/grid_client/scripts/applications/subsquid.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -55,6 +55,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/taiga.ts b/packages/grid_client/scripts/applications/taiga.ts index df751e7972..3e6de700b6 100644 --- a/packages/grid_client/scripts/applications/taiga.ts +++ b/packages/grid_client/scripts/applications/taiga.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -55,6 +55,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/applications/wordpress.ts b/packages/grid_client/scripts/applications/wordpress.ts index 064aa5d57d..4f084c83f3 100644 --- a/packages/grid_client/scripts/applications/wordpress.ts +++ b/packages/grid_client/scripts/applications/wordpress.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; +import { Features, FilterOptions, GatewayNameModel, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -56,6 +56,7 @@ async function main() { sru: instanceCapacity.sru, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; //GatewayNode Selection const gatewayQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/orchestrators/caprover_leader.ts b/packages/grid_client/scripts/orchestrators/caprover_leader.ts index c26e258143..f4ec32d57a 100644 --- a/packages/grid_client/scripts/orchestrators/caprover_leader.ts +++ b/packages/grid_client/scripts/orchestrators/caprover_leader.ts @@ -1,4 +1,4 @@ -import { FilterOptions, MachinesModel } from "../../src"; +import { Features, FilterOptions, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log } from "../utils"; @@ -34,6 +34,7 @@ async function main() { sru: 10, farmId: 1, availableFor: grid3.twinId, + features: [Features.wireguard, Features.ip, Features.ipv4], }; const vms: MachinesModel = { diff --git a/packages/grid_client/scripts/orchestrators/caprover_worker.ts b/packages/grid_client/scripts/orchestrators/caprover_worker.ts index ce8d466272..5db31f3677 100644 --- a/packages/grid_client/scripts/orchestrators/caprover_worker.ts +++ b/packages/grid_client/scripts/orchestrators/caprover_worker.ts @@ -1,4 +1,4 @@ -import { FilterOptions, MachinesModel } from "../../src"; +import { Features, FilterOptions, MachinesModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log } from "../utils"; @@ -32,6 +32,7 @@ async function main() { mru: 4, // GB sru: 10, farmId: 1, + features: [Features.wireguard, Features.ip, Features.ipv4], }; const vms: MachinesModel = { diff --git a/packages/grid_client/scripts/orchestrators/kubernetes_leader.ts b/packages/grid_client/scripts/orchestrators/kubernetes_leader.ts index e44600b0e0..aca51e25ec 100644 --- a/packages/grid_client/scripts/orchestrators/kubernetes_leader.ts +++ b/packages/grid_client/scripts/orchestrators/kubernetes_leader.ts @@ -1,4 +1,4 @@ -import { FilterOptions, K8SModel } from "../../src"; +import { Features, FilterOptions, K8SModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log } from "../utils"; @@ -33,6 +33,7 @@ async function main() { sru: 6, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; const workerQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/orchestrators/kubernetes_with_qsfs.ts b/packages/grid_client/scripts/orchestrators/kubernetes_with_qsfs.ts index 051f1e71db..6bfd8786ad 100644 --- a/packages/grid_client/scripts/orchestrators/kubernetes_with_qsfs.ts +++ b/packages/grid_client/scripts/orchestrators/kubernetes_with_qsfs.ts @@ -1,4 +1,4 @@ -import { FilterOptions, GridClient, K8SModel, QSFSZDBSModel } from "../../src"; +import { Features, FilterOptions, GridClient, K8SModel, QSFSZDBSModel } from "../../src"; import { config, getClient } from "../client_loader"; import { log, pingNodes } from "../utils"; @@ -50,6 +50,7 @@ async function main() { sru: 6, availableFor: grid3.twinId, farmId: 1, + features: [Features.wireguard], }; const qsfsQueryOptions: FilterOptions = { diff --git a/packages/grid_client/scripts/orchestrators/kubernetes_worker.ts b/packages/grid_client/scripts/orchestrators/kubernetes_worker.ts index 85ca42b3d7..027e45a2f3 100644 --- a/packages/grid_client/scripts/orchestrators/kubernetes_worker.ts +++ b/packages/grid_client/scripts/orchestrators/kubernetes_worker.ts @@ -1,4 +1,4 @@ -import { AddWorkerModel, FilterOptions } from "../../src"; +import { AddWorkerModel, Features, FilterOptions } from "../../src"; import { getClient } from "../client_loader"; import { log } from "../utils"; @@ -34,6 +34,7 @@ async function main() { mru: 1, // GB sru: 10, farmId: 1, + features: [Features.wireguard], }; const worker: AddWorkerModel = { diff --git a/packages/grid_client/scripts/single_vm_zos4.ts b/packages/grid_client/scripts/single_vm_zos4.ts new file mode 100644 index 0000000000..921030bd81 --- /dev/null +++ b/packages/grid_client/scripts/single_vm_zos4.ts @@ -0,0 +1,115 @@ +import { Features, FilterOptions, generateRandomHexSeed, GridClient, MachinesDeleteModel, MachinesModel } from "../src"; +import { config, getClient } from "./client_loader"; +import { log, pingNodes } from "./utils"; + +async function deploy(client, vms) { + const resultVM = await client.machines.deploy(vms); + log("================= Deploying VM ================="); + log(resultVM); + log("================= Deploying VM ================="); +} + +async function getDeployment(client, vms) { + const resultVM = await client.machines.getObj(vms.name); + log("================= Getting deployment information ================="); + log(resultVM); + log("================= Getting deployment information ================="); +} + +async function cancel(client, vms) { + const resultVM = await client.machines.delete(vms); + log("================= Canceling the deployment ================="); + log(resultVM); + log("================= Canceling the deployment ================="); +} + +async function main() { + const name = "newalgorand"; + const grid3 = await getClient(`algorand/${name}`); + const instanceCapacity = { cru: 2, mru: 4, sru: 100 }; // Update the instance capacity values according to your requirements. + + //VMNode Selection + const vmQueryOptions: FilterOptions = { + cru: instanceCapacity.cru, + mru: instanceCapacity.mru, + sru: instanceCapacity.sru, + availableFor: grid3.twinId, + farmId: 1, + features: [Features.zmachinelight, Features.networklight, Features.mycelium], + }; + const nodes = await grid3.capacity.filterNodes(vmQueryOptions); + const vmNode = await pingNodes(grid3, nodes); + + const vms: MachinesModel = { + name, + network: { + name: "vmNode", + ip_range: "10.249.0.0/16", + myceliumSeeds: [ + { + nodeId: vmNode, + /** + * ### Mycelium Network Seed: + * - The `seed` is an optional field used to provide a specific seed for the Mycelium network. + * - If not provided, the `GridClient` will generate a seed automatically when the `mycelium` flag is enabled. + * - **Use Case:** If you need the new machine to have the same IP address as a previously deleted machine, set the `seed` field to the old seed value. + */ + seed: generateRandomHexSeed(32), + }, + ], + }, + machines: [ + { + name: "testvmMY", + node_id: vmNode, + disks: [ + { + name: "wedDisk", + size: instanceCapacity.sru, + mountpoint: "/testdisk", + }, + ], + planetary: false, + public_ip: false, + public_ip6: false, + /** + * ### Mycelium Flag Behavior: + * - When the `mycelium` flag is enabled, there’s no need to manually provide the `myceliumSeed` flag. + * - The `GridClient` will automatically generate the necessary seed for you. + * - **However**, if you have **an existing seed** from a previously deleted machine and wish to deploy a new machine that retains the same IP address, + * - **you can simply pass in the old seed during deployment instead of calling the `generateRandomHexSeed()` function**. + */ + mycelium: true, + /** + * ### Mycelium Seed: + * - The `myceliumSeed` is an optional field used to provide a specific seed for the Mycelium network. + * - If not provided, the `GridClient` will generate a seed automatically when the `mycelium` flag is enabled. + * - **Use Case:** If you need the new machine to have the same IP address as a previously deleted machine, set the `seed` field to the old seed value. */ + myceliumSeed: generateRandomHexSeed(6), // (HexSeed of length 6) + cpu: instanceCapacity.cru, + memory: 1024 * instanceCapacity.mru, + rootfs_size: 0, + flist: "https://hub.grid.tf/tf-official-apps/base:latest.flist", + entrypoint: "/sbin/zinit init", + env: { + SSH_KEY: config.ssh_key, + }, + }, + ], + metadata: "", + description: "test deploying single ZOS4 VM with mycelium via ts grid3 client", + }; + + //Deploy VMs + await deploy(grid3, vms); + + //Get the deployment + await getDeployment(grid3, vms); + + //Uncomment the line below to cancel the deployment + // await cancel(grid3, { name }); + + await grid3.disconnect(); +} + +main(); diff --git a/packages/grid_client/src/helpers/types.ts b/packages/grid_client/src/helpers/types.ts index 912e955f3d..03e06b3d9b 100644 --- a/packages/grid_client/src/helpers/types.ts +++ b/packages/grid_client/src/helpers/types.ts @@ -1,4 +1,4 @@ -import { PublicIPResult, ResultStates } from "../zos"; +import { PublicIPResult, ResultStates, WorkloadTypes } from "../zos"; interface NetworkInterface { /** The network identifier */ @@ -44,10 +44,29 @@ interface ExtendedMountData extends Partial { // Union type for the mount data type MountData = BaseMountData | ExtendedMountData; - +enum Features { + ipv4 = "ipv4", + ip = "ip", + mycelium = "mycelium", + wireguard = "wireguard", + yggdrasil = "yggdrasil", + gatewayfqdnproxy = "gateway-fqdn-proxy", + gatewaynameproxy = "gateway-name-proxy", + zmachine = "zmachine", + zmount = "zmount", + volume = "volume", + network = "network", + zdb = "zdb", + qsfs = "qsfs", + zlogs = "zlogs", + networklight = "network-light", + zmachinelight = "zmachine-light", +} interface ZmachineData { /** The version of the workload */ version: number; + /** The type of the workload */ + type: WorkloadTypes; /** The contract ID associated with the workload */ contractId: number; /** The node ID where the workload is deployed */ @@ -112,4 +131,4 @@ interface GatewayDeploymentData { description: string; } -export { ZmachineData, VM, ExtendedMountData, GatewayDeploymentData }; +export { ZmachineData, VM, ExtendedMountData, GatewayDeploymentData, Features }; diff --git a/packages/grid_client/src/high_level/base.ts b/packages/grid_client/src/high_level/base.ts index 0f1310509c..6902b5734c 100644 --- a/packages/grid_client/src/high_level/base.ts +++ b/packages/grid_client/src/high_level/base.ts @@ -7,6 +7,7 @@ import { events } from "../helpers/events"; import { Operations, TwinDeployment } from "../high_level/models"; import { DeploymentFactory } from "../primitives/deployment"; import { Network } from "../primitives/network"; +import { ZNetworkLight } from "../primitives/networklight"; import { Nodes } from "../primitives/nodes"; import { Deployment } from "../zos/deployment"; import { Workload, WorkloadTypes } from "../zos/workload"; @@ -27,6 +28,7 @@ class HighLevelBase { WorkloadTypes.ip, WorkloadTypes.ipv4, // TODO: remove deprecated WorkloadTypes.zmachine, + WorkloadTypes.zmachinelight, WorkloadTypes.zmount, WorkloadTypes.volume, WorkloadTypes.zdb, @@ -38,35 +40,48 @@ class HighLevelBase { ): [Workload[], Workload[]] { let deletedMachineWorkloads: Workload[] = []; if (names.length === 0) { - deletedMachineWorkloads = deployment.workloads.filter(item => item.type === WorkloadTypes.zmachine); + deletedMachineWorkloads = deployment.workloads.filter( + item => item.type === WorkloadTypes.zmachine || item.type === WorkloadTypes.zmachinelight, + ); } - if (names.length !== 0 && types.includes(WorkloadTypes.zmachine)) { - const Workloads = deployment.workloads.filter(item => item.type === WorkloadTypes.zmachine); - for (const workload of Workloads) { - if (!names.includes(workload.name)) { - continue; - } - for (const mount of workload.data["mounts"]) { - names.push(mount.name); - } + if (names.length !== 0 && (types.includes(WorkloadTypes.zmachine) || types.includes(WorkloadTypes.zmachinelight))) { + const workloadTypes = [WorkloadTypes.zmachine, WorkloadTypes.zmachinelight]; + + for (const type of workloadTypes) { + if (!types.includes(type)) continue; + + const Workloads = deployment.workloads.filter(item => item.type === type); + + for (const workload of Workloads) { + if (!names.includes(workload.name)) { + continue; + } + + for (const mount of workload.data["mounts"]) { + names.push(mount.name); + } + + const toRemoveZlogs = deployment.workloads.filter(item => { + const zlog = item.type === WorkloadTypes.zlogs; + const workloadtypename = + (item.data as any)[type === WorkloadTypes.zmachine ? "zmachine" : "zmachine-light"] === workload.name; + return zlog && workloadtypename; + }); - const toRemoveZlogs = deployment.workloads.filter(item => { - const x = item.type === WorkloadTypes.zlogs; - const y = (item.data as any)["zmachine"] === workload.name; - return x && y; - }); + names.push(...toRemoveZlogs.map(zlog => zlog.name)); - names.push(...toRemoveZlogs.map(x => x.name)); + if (type === WorkloadTypes.zmachine) { + names.push(workload.data["network"].public_ip); + } - names.push(workload.data["network"].public_ip); - deletedMachineWorkloads.push(workload); + deletedMachineWorkloads.push(workload); + } } } - const remainingWorkloads: Workload[] = []; for (const workload of deployment.workloads) { - if (workload.type === WorkloadTypes.network) { + if (workload.type === WorkloadTypes.network || workload.type === WorkloadTypes.networklight) { remainingWorkloads.push(workload); continue; } @@ -86,17 +101,23 @@ class HighLevelBase { remainingWorkloads: Workload[], deletedMachineWorkloads: Workload[], node_id: number, - ): Promise<[TwinDeployment[], Workload[], number[], string[], Network | null]> { + ): Promise<[TwinDeployment[], Workload[], number[], string[], Network | ZNetworkLight | null]> { const twinDeployments: TwinDeployment[] = []; const deletedNodes: number[] = []; const deletedIps: string[] = []; const deploymentFactory = new DeploymentFactory(this.config); - let network: Network | null = null; + let network: Network | ZNetworkLight | null = null; + let network_contract_id; + for (const workload of deletedMachineWorkloads) { if (!network) { const networkName = workload.data["network"].interfaces[0].network; const networkIpRange = Addr(workload.data["network"].interfaces[0].ip).mask(16).toString(); - network = new Network(networkName, networkIpRange, this.config); + if (workload.type === WorkloadTypes.zmachinelight) { + network = new ZNetworkLight(networkName, networkIpRange, this.config); + } else { + network = new Network(networkName, networkIpRange, this.config); + } await network.load(); } const machineIp = workload.data["network"].interfaces[0].ip; @@ -111,17 +132,19 @@ class HighLevelBase { deletedIps.push(deletedIp); continue; } - const hasAccessPoint = network.hasAccessPoint(node_id); - if (hasAccessPoint && network.nodes.length !== 1) { - console.log( - `network ${network.name} still has access point:${hasAccessPoint} and number of nodes ${network.nodes.length}`, - ); - deletedIps.push(deletedIp); - continue; + if (network instanceof Network) { + const hasAccessPoint = network.hasAccessPoint(node_id); + if (hasAccessPoint && network.nodes.length !== 1) { + console.log( + `network ${network.name} still has access point:${hasAccessPoint} and number of nodes ${network.nodes.length}`, + ); + deletedIps.push(deletedIp); + continue; + } } + network_contract_id = await network.deleteNode(node_id); - const contract_id = await network.deleteNode(node_id); - if (contract_id === deployment.contract_id) { + if (network_contract_id === deployment.contract_id) { if (remainingWorkloads.length === 1) { twinDeployments.push(new TwinDeployment(deployment, Operations.delete, 0, 0, "", network)); remainingWorkloads = []; @@ -134,7 +157,7 @@ class HighLevelBase { // check that the deployment doesn't have another workloads for (let d of network.deployments) { d = await deploymentFactory.fromObj(d); - if (d.contract_id !== contract_id) { + if (d.contract_id !== network_contract_id) { continue; } if (d.workloads.length === 1) { @@ -146,7 +169,11 @@ class HighLevelBase { } } // in case of the network got more accesspoints on different nodes this won't be valid - if (network.nodes.length === 1 && network.getNodeReservedIps(network.nodes[0].node_id).length === 0) { + if ( + network instanceof Network && + network.nodes.length === 1 && + network.getNodeReservedIps(network.nodes[0].node_id).length === 0 + ) { const contract_id = await network.deleteNode(network.nodes[0].node_id); for (let d of network.deployments) { d = await deploymentFactory.fromObj(d); @@ -172,6 +199,7 @@ class HighLevelBase { WorkloadTypes.ip, WorkloadTypes.ipv4, // TODO: remove deprecated WorkloadTypes.zmachine, + WorkloadTypes.zmachinelight, WorkloadTypes.zmount, WorkloadTypes.volume, WorkloadTypes.zdb, @@ -181,7 +209,7 @@ class HighLevelBase { WorkloadTypes.zlogs, ], ): Promise { - if (types.includes(WorkloadTypes.network)) { + if (types.includes(WorkloadTypes.network) || types.includes(WorkloadTypes.networklight)) { throw new GridClientErrors.Workloads.WorkloadDeleteError("Network workload can't be deleted."); } let twinDeployments: TwinDeployment[] = []; @@ -201,7 +229,11 @@ class HighLevelBase { await this._deleteMachineNetwork(deployment, remainingWorkloads, deletedMachineWorkloads, node_id); twinDeployments = twinDeployments.concat(newTwinDeployments); remainingWorkloads = newRemainingWorkloads; - if (remainingWorkloads.length !== 0 && remainingWorkloads.length < numberOfWorkloads) { + if ( + network instanceof Network && + remainingWorkloads.length !== 0 && + remainingWorkloads.length < numberOfWorkloads + ) { for (const deleteNode of deletedNodes) { await network!.deleteNode(deleteNode); } diff --git a/packages/grid_client/src/high_level/kubernetes.ts b/packages/grid_client/src/high_level/kubernetes.ts index a9c5e523b9..15a48a0f62 100644 --- a/packages/grid_client/src/high_level/kubernetes.ts +++ b/packages/grid_client/src/high_level/kubernetes.ts @@ -2,6 +2,7 @@ import { events } from "../helpers/events"; import { VMHL } from "../high_level//machine"; import { MyceliumNetworkModel, QSFSDiskModel } from "../modules/models"; import { Network } from "../primitives/network"; +import { ZNetworkLight } from "../primitives/networklight"; import { Deployment } from "../zos/deployment"; import { WorkloadTypes } from "../zos/workload"; import { HighLevelBase } from "./base"; @@ -22,7 +23,7 @@ class KubernetesHL extends HighLevelBase { planetary: boolean, mycelium: boolean, myceliumSeed: string, - network: Network, + network: Network | ZNetworkLight, myceliumNetworkSeeds: MyceliumNetworkModel[] = [], sshKey: string, contractMetadata: string, @@ -100,7 +101,7 @@ class KubernetesHL extends HighLevelBase { planetary: boolean, mycelium: boolean, myceliumSeed: string, - network: Network, + network: Network | ZNetworkLight, myceliumNetworkSeeds: MyceliumNetworkModel[] = [], sshKey: string, contractMetadata, @@ -168,6 +169,7 @@ class KubernetesHL extends HighLevelBase { async delete(deployment: Deployment, names: string[]) { return await this._delete(deployment, names, [ WorkloadTypes.zmachine, + WorkloadTypes.zmachinelight, WorkloadTypes.zmount, WorkloadTypes.volume, WorkloadTypes.ip, diff --git a/packages/grid_client/src/high_level/machine.ts b/packages/grid_client/src/high_level/machine.ts index 0de7b154c1..ef9cc4dac4 100644 --- a/packages/grid_client/src/high_level/machine.ts +++ b/packages/grid_client/src/high_level/machine.ts @@ -13,9 +13,11 @@ import { Network, Nodes, PublicIPPrimitive, + VMLightPrimitive, VMPrimitive, ZlogsPrimitive, } from "../primitives/index"; +import { ZNetworkLight } from "../primitives/networklight"; import { QSFSPrimitive } from "../primitives/qsfs"; import { Mount, ZdbGroup } from "../zos"; import { Deployment } from "../zos/deployment"; @@ -37,7 +39,7 @@ class VMHL extends HighLevelBase { planetary: boolean, mycelium: boolean, myceliumSeed: string, - network: Network, + network: Network | ZNetworkLight, myceliumNetworkSeeds: MyceliumNetworkModel[] = [], entrypoint: string, env: Record, @@ -60,6 +62,9 @@ class VMHL extends HighLevelBase { RAMInMegaBytes: memory, }); } + const nodeInfo = await this.nodes.getNode(nodeId); + // Checks if the node is a zos4 node + const isZOS4 = nodeInfo.features.some(item => item.includes("zmachine-light") || item.includes("network-light")); const deployments: TwinDeployment[] = []; const workloads: Workload[] = []; let totalDisksSize = rootfs_size; @@ -86,7 +91,6 @@ class VMHL extends HighLevelBase { ); } else { // If Available for twinId (dedicated), check it's not in grace period - const nodeInfo = await this.nodes.getNode(nodeId); if (nodeInfo.rentContractId !== 0) { const contract = await this.config.tfclient.contracts.get({ id: nodeInfo.rentContractId }); if (contract && contract.state.gracePeriod) { @@ -149,9 +153,13 @@ class VMHL extends HighLevelBase { } // ipv4 + // Reject the deployment if the node is a zos4 node, as it doesn't support ipv4. let ipName = ""; let publicIps = 0; if (publicIp || publicIp6) { + if (isZOS4) { + throw new GridClientErrors.Farms.InvalidResourcesError(`Zmachine Light doesn't support public ips.`); + } const ip = new PublicIPPrimitive(); ipName = `${name}_pubip`; workloads.push(ip.create(ipName, "", description, 0, publicIp, publicIp6)); @@ -168,7 +176,6 @@ class VMHL extends HighLevelBase { publicIps++; } } - if (gpus && gpus.length > 0) { const nodeTwinId = await this.nodes.getNodeTwinId(nodeId); const gpuList = await this.rmb.request([nodeTwinId], "zos.gpu.list", ""); @@ -200,58 +207,76 @@ class VMHL extends HighLevelBase { let accessNodeSubnet; if (ip) { userIPsubnet = network.ValidateFreeSubnet(Addr(ip).mask(24).toString()); - accessNodeSubnet = network.getFreeSubnet(); + + // If the node is not a zos4 node get accessNodeSubnet + if (!isZOS4) { + accessNodeSubnet = network.getFreeSubnet(); + } } - // network - const networkContractMetadata = JSON.stringify({ - version: 3, - type: "network", - name: network.name, - projectName: this.config.projectName, - }); + // Set networkContractMetadata based on node's zos version + let networkContractMetadata; + if (!isZOS4) { + networkContractMetadata = JSON.stringify({ + version: 3, + type: "network", + name: network.name, + projectName: this.config.projectName, + }); + } else { + networkContractMetadata = JSON.stringify({ + version: 4, + type: "network-light", + name: network.name, + projectName: this.config.projectName, + }); + } + const deploymentFactory = new DeploymentFactory(this.config); + let access_net_workload; let wgConfig = ""; let hasAccessNode = false; let accessNodes: Record = {}; - if (addAccess) { - accessNodes = await this.nodes.getAccessNodes(this.config.twinId); - for (const accessNode of Object.keys(accessNodes)) { - if (network.nodeExists(Number(accessNode))) { - hasAccessNode = true; - break; + if (!isZOS4 && network instanceof Network) { + if (addAccess) { + accessNodes = await this.nodes.getAccessNodes(this.config.twinId); + for (const accessNode of Object.keys(accessNodes)) { + if (network.nodeExists(Number(accessNode))) { + hasAccessNode = true; + break; + } } } - } - if ( - (!Object.keys(accessNodes).includes(nodeId.toString()) || nodeId !== accessNodeId) && - !hasAccessNode && - addAccess - ) { - // add node to any access node and deploy it - const filteredAccessNodes: number[] = []; - for (const accessNodeId of Object.keys(accessNodes)) { - if (accessNodes[accessNodeId]["ipv4"]) { - filteredAccessNodes.push(+accessNodeId); + if ( + (!Object.keys(accessNodes).includes(nodeId.toString()) || nodeId !== accessNodeId) && + !hasAccessNode && + addAccess && + !isZOS4 + ) { + const filteredAccessNodes: number[] = []; + for (const accessNodeId of Object.keys(accessNodes)) { + if (accessNodes[accessNodeId]["ipv4"]) { + filteredAccessNodes.push(+accessNodeId); + } } - } - let access_node_id = randomChoice(filteredAccessNodes); - if (accessNodeId) { - if (!filteredAccessNodes.includes(accessNodeId)) - throw new GridClientErrors.Nodes.AccessNodeError( - `Node ${accessNodeId} is not an access node or maybe it's down.`, - ); + let access_node_id = randomChoice(filteredAccessNodes); + if (accessNodeId) { + if (!filteredAccessNodes.includes(accessNodeId)) + throw new GridClientErrors.Nodes.AccessNodeError( + `Node ${accessNodeId} is not an access node or maybe it's down.`, + ); - access_node_id = accessNodeId; + access_node_id = accessNodeId; + } + access_net_workload = await network.addNode( + access_node_id, + mycelium, + description, + accessNodeSubnet, + myceliumNetworkSeeds, + ); + wgConfig = await network.addAccess(access_node_id, true); } - access_net_workload = await network.addNode( - access_node_id, - mycelium, - description, - accessNodeSubnet, - myceliumNetworkSeeds, - ); - wgConfig = await network.addAccess(access_node_id, true); } // If node exits on network check if mycelium needs to be added or not if (network.nodeExists(nodeId)) { @@ -262,7 +287,7 @@ class VMHL extends HighLevelBase { } const znet_workload = await network.addNode(nodeId, mycelium, description, userIPsubnet, myceliumNetworkSeeds); - if ((await network.exists()) && (znet_workload || access_net_workload)) { + if (network instanceof Network && (await network.exists()) && (znet_workload || access_net_workload)) { // update network for (const deployment of network.deployments) { const d = await deploymentFactory.fromObj(deployment); @@ -273,6 +298,7 @@ class VMHL extends HighLevelBase { ) { continue; } + workload.data = network.getUpdatedNetwork(workload["data"]); workload.version += 1; break; @@ -295,7 +321,8 @@ class VMHL extends HighLevelBase { } } else if (znet_workload) { // node not exist on the network - if (!access_net_workload && !hasAccessNode && addAccess) { + + if (!access_net_workload && !hasAccessNode && addAccess && network instanceof Network) { // this node is access node, so add access point on it wgConfig = await network.addAccess(nodeId, true); znet_workload["data"] = network.getUpdatedNetwork(znet_workload.data); @@ -313,7 +340,7 @@ class VMHL extends HighLevelBase { ), ); } - if (access_net_workload) { + if (access_net_workload && !isZOS4) { // network is not exist, and the node provide is not an access node const accessNodeId = access_net_workload.data["node_id"]; access_net_workload["data"] = network.getUpdatedNetwork(access_net_workload.data); @@ -332,7 +359,13 @@ class VMHL extends HighLevelBase { } // vm - const vm = new VMPrimitive(); + // Initalize vm based on node's zos version + let vm; + if (!isZOS4) { + vm = new VMPrimitive(); + } else { + vm = new VMLightPrimitive(); + } let machine_ip; if (ip !== "") { machine_ip = network.validateUserIP(nodeId, ip); @@ -412,6 +445,7 @@ class VMHL extends HighLevelBase { WorkloadTypes.zmachine, WorkloadTypes.qsfs, WorkloadTypes.zlogs, + WorkloadTypes.zmachinelight, ]); } } diff --git a/packages/grid_client/src/high_level/models.ts b/packages/grid_client/src/high_level/models.ts index 1b59b25510..b6405f078d 100644 --- a/packages/grid_client/src/high_level/models.ts +++ b/packages/grid_client/src/high_level/models.ts @@ -1,6 +1,7 @@ import { Contract } from "@threefold/tfchain_client"; import { Network } from "../primitives/network"; +import { ZNetworkLight } from "../primitives/networklight"; import { Deployment } from "../zos/deployment"; enum Operations { @@ -16,7 +17,7 @@ class TwinDeployment { public publicIps: number, public nodeId: number, public metadata: string, - public network: Network | null = null, + public network: Network | null | ZNetworkLight = null, public solutionProviderId: number | null = null, public returnNetworkContracts = false, ) {} diff --git a/packages/grid_client/src/high_level/twinDeploymentHandler.ts b/packages/grid_client/src/high_level/twinDeploymentHandler.ts index add5a8578d..e4e9b7ccef 100644 --- a/packages/grid_client/src/high_level/twinDeploymentHandler.ts +++ b/packages/grid_client/src/high_level/twinDeploymentHandler.ts @@ -8,7 +8,7 @@ import { GridClientConfig } from "../config"; import { formatErrorMessage } from "../helpers"; import { events } from "../helpers/events"; import { validateObject } from "../helpers/validator"; -import { DeploymentFactory, Nodes } from "../primitives/index"; +import { DeploymentFactory, Network, Nodes } from "../primitives/index"; import { Workload, WorkloadTypes } from "../zos/workload"; import { DeploymentResultContracts, Operations, TwinDeployment } from "./models"; class TwinDeploymentHandler { @@ -147,11 +147,13 @@ class TwinDeploymentHandler { for (const workload of twinDeployment.deployment.workloads) { if (workload.type !== WorkloadTypes.network) continue; const contract = contracts.created.filter(c => c.contractId === twinDeployment.deployment.contract_id); - twinDeployment.network.save(contract[0]); + if (twinDeployment.network instanceof Network) { + twinDeployment.network.save(contract[0]); + } } } // left just to delete the old keys - else if (twinDeployment.operation === Operations.delete) { + else if (twinDeployment.operation === Operations.delete && twinDeployment.network instanceof Network) { await twinDeployment.network.save(undefined, twinDeployment.deployment.contract_id); } } @@ -166,7 +168,7 @@ class TwinDeploymentHandler { continue; } const network_workloads = twinDeployment.deployment.workloads.filter( - workload => workload.type === WorkloadTypes.network, + workload => workload.type === WorkloadTypes.network || workload.type === WorkloadTypes.networklight, ); if (network_workloads.length > 0 || twinDeployment.publicIps > 0) { deployments.push(twinDeployment); @@ -539,7 +541,7 @@ class TwinDeploymentHandler { if (!twinDeployment.network) { break; } - if (workload.type === WorkloadTypes.network) { + if (workload.type === WorkloadTypes.network || workload.type === WorkloadTypes.networklight) { events.emit("logs", `Updating network workload with name: ${workload.name}`); twinDeployment.network.updateWorkload(twinDeployment.nodeId, workload); } diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index da2301a9cb..91cdf1c2b4 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -503,6 +503,7 @@ class BaseModule { const resultData = workload.result.data as ZmachineResult; return { version: workload.version, + type: workload.type, contractId: workload["contractId"], nodeId: workload["nodeId"], name: workload.name, diff --git a/packages/grid_client/src/modules/capacity.ts b/packages/grid_client/src/modules/capacity.ts index 98e0d7d57b..b3803edc29 100644 --- a/packages/grid_client/src/modules/capacity.ts +++ b/packages/grid_client/src/modules/capacity.ts @@ -159,6 +159,15 @@ class Capacity { return nodes; } + @expose + @validateInput + getFeaturesFromFilters(options?: FilterOptions): string[] { + let features: string[] = []; + + features = this.nodes.getFeaturesFromFilters(options); + return features; + } + /** * Retrieves a list of farms based on the provided options. * diff --git a/packages/grid_client/src/modules/k8s.ts b/packages/grid_client/src/modules/k8s.ts index 76575825e3..3eb1704a15 100644 --- a/packages/grid_client/src/modules/k8s.ts +++ b/packages/grid_client/src/modules/k8s.ts @@ -9,7 +9,9 @@ import { expose } from "../helpers/expose"; import { validateInput } from "../helpers/validator"; import { KubernetesHL } from "../high_level/kubernetes"; import { DeploymentResultContracts, TwinDeployment } from "../high_level/models"; +import { Nodes } from "../primitives"; import { Network } from "../primitives/network"; +import { ZNetworkLight } from "../primitives/networklight"; import { Deployment } from "../zos"; import { Workload, WorkloadTypes } from "../zos/workload"; import { BaseModule } from "./base"; @@ -28,6 +30,7 @@ class K8sModule extends BaseModule { WorkloadTypes.zlogs, ]; // TODO: remove deprecated kubernetes: KubernetesHL; + capacity: Nodes; /** * Class representing a Kubernetes Module. @@ -36,10 +39,12 @@ class K8sModule extends BaseModule { * This class provides methods for managing Kubernetes deployments, including creating, updating, listing, and deleting deployments. * @class K8sModule * @param {GridClientConfig} config - The configuration object for initializing the client. + * @param {Nodes} capacity - Used to get node features */ constructor(public config: GridClientConfig) { super(config); this.kubernetes = new KubernetesHL(config); + this.capacity = new Nodes(this.config.graphqlURL, this.config.proxyURL, this.config.rmbClient); } /** @@ -126,17 +131,11 @@ class K8sModule extends BaseModule { * @returns {Promise<[TwinDeployment[], Network, string]>} A tuple containing the created deployments, network configuration, and Wireguard configuration. */ async _createDeployment(options: K8SModel, masterIps: string[] = []): Promise<[TwinDeployment[], Network, string]> { - const network = new Network(options.network.name, options.network.ip_range, this.config); - await network.load(); + let network; + let contractMetadata; let deployments: TwinDeployment[] = []; let wireguardConfig = ""; - const contractMetadata = JSON.stringify({ - version: 3, - type: "kubernetes", - name: options.name, - projectName: this.config.projectName || `kubernetes/${options.name}`, - }); const masters_names: string[] = []; const workers_names: string[] = []; for (const master of options.masters) { @@ -144,6 +143,29 @@ class K8sModule extends BaseModule { throw new ValidationError(`Another master with the same name ${master.name} already exists.`); masters_names.push(master.name); + if (network) break; + // Sets the network and contractMetadata based on the node's zos version + const nodeTwinId = await this.capacity.getNodeTwinId(master.node_id); + const features = await this.rmb.request([nodeTwinId], "zos.system.node_features_get", "", 20, 3); + if (features.some(item => item.includes("zmachine-light") || item.includes("network-light"))) { + network = new ZNetworkLight(options.network.name, options.network.ip_range, this.config); + await network.load(); + contractMetadata = JSON.stringify({ + version: 4, + type: "kubernetes", + name: options.name, + projectName: this.config.projectName || `kubernetes/${options.name}`, + }); + } else { + network = new Network(options.network.name, options.network.ip_range, this.config); + await network.load(); + contractMetadata = JSON.stringify({ + version: 3, + type: "kubernetes", + name: options.name, + projectName: this.config.projectName || `kubernetes/${options.name}`, + }); + } const [twinDeployments, wgConfig] = await this.kubernetes.add_master( master.name, master.node_id, @@ -197,6 +219,28 @@ class K8sModule extends BaseModule { throw new ValidationError(`Another worker with the same name ${worker.name} already exists.`); workers_names.push(worker.name); + if (network) break; + const nodeTwinId = await this.capacity.getNodeTwinId(worker.node_id); + const features = await this.rmb.request([nodeTwinId], "zos.system.node_features_get", "", 20, 3); + if (features.some(item => item.includes("zmachine-light") || item.includes("network-light"))) { + network = new ZNetworkLight(options.network.name, options.network.ip_range, this.config); + await network.load(); + contractMetadata = JSON.stringify({ + version: 4, + type: "kubernetes", + name: options.name, + projectName: this.config.projectName || `kubernetes/${options.name}`, + }); + } else { + network = new Network(options.network.name, options.network.ip_range, this.config); + await network.load(); + contractMetadata = JSON.stringify({ + version: 3, + type: "kubernetes", + name: options.name, + projectName: this.config.projectName || `kubernetes/${options.name}`, + }); + } const [twinDeployments] = await this.kubernetes.add_worker( worker.name, worker.node_id, diff --git a/packages/grid_client/src/modules/machines.ts b/packages/grid_client/src/modules/machines.ts index 30f837ce1e..99c04dd18d 100644 --- a/packages/grid_client/src/modules/machines.ts +++ b/packages/grid_client/src/modules/machines.ts @@ -9,6 +9,8 @@ import { validateInput } from "../helpers/validator"; import { VMHL } from "../high_level/machine"; import { DeploymentResultContracts, TwinDeployment } from "../high_level/models"; import { Network } from "../primitives/network"; +import { ZNetworkLight } from "../primitives/networklight"; +import { Nodes } from "../primitives/nodes"; import { Deployment } from "../zos"; import { WorkloadTypes } from "../zos/workload"; import { BaseModule } from "./base"; @@ -25,8 +27,10 @@ class MachinesModule extends BaseModule { WorkloadTypes.ip, WorkloadTypes.ipv4, WorkloadTypes.zlogs, + WorkloadTypes.zmachinelight, ]; // TODO: remove deprecated vm: VMHL; + capacity: Nodes; /** * The MachinesModule class is responsible for managing virtual machine deployments. * It extends the BaseModule class and provides methods to deploy, list, get, update, add, and delete machines. @@ -38,6 +42,7 @@ class MachinesModule extends BaseModule { constructor(public config: GridClientConfig) { super(config); this.vm = new VMHL(config); + this.capacity = new Nodes(this.config.graphqlURL, this.config.proxyURL, this.config.rmbClient); } /** @@ -47,17 +52,35 @@ class MachinesModule extends BaseModule { * @returns {Promise<[TwinDeployment[], Network, string]>} - A promise that resolves to an array of twin deployments, the network, and the WireGuard configuration string. */ async _createDeployment(options: MachinesModel): Promise<[TwinDeployment[], Network, string]> { - const network = new Network(options.network.name, options.network.ip_range, this.config); - await network.load(); - + let network; + let contractMetadata; + for (const machine of options.machines) { + if (network) break; + // Sets the network and contractMetadata based on the node's zos version + const nodeTwinId = await this.capacity.getNodeTwinId(machine.node_id); + const features = await this.rmb.request([nodeTwinId], "zos.system.node_features_get", "", 20, 3); + if (features.some(item => item.includes("zmachine-light") || item.includes("network-light"))) { + network = new ZNetworkLight(options.network.name, options.network.ip_range, this.config); + await network.load(); + contractMetadata = JSON.stringify({ + version: 4, + type: "vm", + name: options.name, + projectName: this.config.projectName || `vm/${options.name}`, + }); + } else { + network = new Network(options.network.name, options.network.ip_range, this.config); + await network.load(); + contractMetadata = JSON.stringify({ + version: 3, + type: "vm", + name: options.name, + projectName: this.config.projectName || `vm/${options.name}`, + }); + } + } let twinDeployments: TwinDeployment[] = []; let wireguardConfig = ""; - const contractMetadata = JSON.stringify({ - version: 3, - type: "vm", - name: options.name, - projectName: this.config.projectName || `vm/${options.name}`, - }); const machines_names: string[] = []; @@ -65,7 +88,6 @@ class MachinesModule extends BaseModule { if (machines_names.includes(machine.name)) throw new ValidationError(`Another machine with the same name ${machine.name} already exists.`); machines_names.push(machine.name); - const [TDeployments, wgConfig] = await this.vm.create( machine.name, machine.node_id, @@ -180,7 +202,10 @@ class MachinesModule extends BaseModule { */ async getObj(deploymentName: string): Promise { const deployments = await this._get(deploymentName); - const workloads = await this._getWorkloadsByTypes(deploymentName, deployments, [WorkloadTypes.zmachine]); + const workloads = await this._getWorkloadsByTypes(deploymentName, deployments, [ + WorkloadTypes.zmachine, + WorkloadTypes.zmachinelight, + ]); const promises = workloads.map( async workload => await this._getZmachineData(deploymentName, deployments, workload), ); diff --git a/packages/grid_client/src/modules/models.ts b/packages/grid_client/src/modules/models.ts index 4d4f3fd9a7..e02eabc583 100644 --- a/packages/grid_client/src/modules/models.ts +++ b/packages/grid_client/src/modules/models.ts @@ -21,7 +21,7 @@ import { ValidateNested, } from "class-validator"; -import { IsAlphanumericExpectUnderscore } from "../helpers"; +import { Features, IsAlphanumericExpectUnderscore } from "../helpers"; import { Deployment } from "../zos/deployment"; import { ZdbModes } from "../zos/zdb"; import { blockchainType } from "./blockchainInterface"; @@ -142,6 +142,7 @@ class MachineModel { @Expose() @IsInt() @IsOptional() solutionProviderId?: number; @Expose() @IsString() @IsOptional() zlogsOutput?: string; @Expose() @IsString({ each: true }) @IsOptional() gpus?: string[]; + @Expose() @IsString({ each: true }) @IsOptional() features?: string[]; } class MachinesModel { @@ -635,6 +636,10 @@ class FilterOptions { @Expose() @IsOptional() @Transform(({ value }) => NodeStatus[value]) @IsEnum(NodeStatus) status?: NodeStatus; @Expose() @IsOptional() @IsString() region?: string; @Expose() @IsOptional() @IsBoolean() healthy?: boolean; + @Expose() @IsOptional() @IsBoolean() planetary?: boolean; + @Expose() @IsOptional() @IsBoolean() mycelium?: boolean; + @Expose() @IsOptional() @IsBoolean() wireguard?: boolean; + @Expose() @IsOptional() @IsString() features?: Features[]; } enum CertificationType { diff --git a/packages/grid_client/src/primitives/networklight.ts b/packages/grid_client/src/primitives/networklight.ts new file mode 100644 index 0000000000..19fa7bb526 --- /dev/null +++ b/packages/grid_client/src/primitives/networklight.ts @@ -0,0 +1,416 @@ +import { Contract } from "@threefold/tfchain_client"; +import { ValidationError } from "@threefold/types"; +import { Buffer } from "buffer"; +import { plainToInstance } from "class-transformer"; +import { Addr } from "netaddr"; +import { default as PrivateIp } from "private-ip"; + +import { RMB } from "../clients/rmb/client"; +import { TFClient } from "../clients/tf-grid/client"; +import { GqlNodeContract } from "../clients/tf-grid/contracts"; +import { GridClientConfig } from "../config"; +import { events } from "../helpers/events"; +import { formatErrorMessage, generateRandomHexSeed } from "../helpers/utils"; +import { validateHexSeed } from "../helpers/validator"; +import { MyceliumNetworkModel } from "../modules"; +import { BackendStorage } from "../storage/backend"; +import { NetworkLight } from "../zos"; +import { Deployment } from "../zos/deployment"; +import { Workload, WorkloadTypes } from "../zos/workload"; +import { DeploymentFactory } from "./deployment"; +import { Nodes } from "./nodes"; + +class Node { + node_id: number; + contract_id: number; + reserved_ips: string[] = []; +} +interface NetworkMetadata { + version: number; +} +class ZNetworkLight { + node: Node; + capacity: Nodes; + NodeIds: number[]; + deployments: Deployment[] = []; + network: NetworkLight; + contracts: Required[]; + reservedSubnets: string[] = []; + backendStorage: BackendStorage; + static newContracts: GqlNodeContract[] = []; + static deletedContracts: number[] = []; + rmb: RMB; + tfClient: TFClient; + + constructor(public name: string, public ipRange: string, public config: GridClientConfig) { + if (Addr(ipRange).prefix !== 16) { + this.ipRange = Addr(ipRange).mask(16); + this.ipRange = this.ipRange.toString(); + } + if (!this.isPrivateIP(ipRange)) { + throw new ValidationError("Network ip_range should be a private range."); + } + this.backendStorage = new BackendStorage( + config.backendStorageType, + config.substrateURL, + config.mnemonic, + config.storeSecret, + config.keypairType, + config.backendStorage, + config.seed, + ); + this.rmb = new RMB(config.rmbClient); + this.capacity = new Nodes(this.config.graphqlURL, this.config.proxyURL, this.config.rmbClient); + this.tfClient = config.tfclient; + } + + private getUpdatedMetadata(nodeId: number, metadata: string): string { + if (this.node.node_id === nodeId) { + const parsedMetadata: NetworkMetadata = JSON.parse(metadata || "{}"); + parsedMetadata.version = 4; + return JSON.stringify(parsedMetadata); + } + + return metadata; + } + + updateWorkload(nodeId: number, workload: Workload): Workload { + workload.data = this.getUpdatedNetwork(workload.data); + workload.metadata = this.getUpdatedMetadata(nodeId, workload.metadata); + return workload; + } + getUpdatedNetwork(znet_light): NetworkLight { + if (this.network?.subnet === znet_light.subnet) { + return this.network; + } + + return znet_light; + } + + async addNode( + nodeId: number, + mycelium: boolean, + description = "", + subnet = "", + myceliumSeeds: MyceliumNetworkModel[] = [], + ): Promise { + if (this.nodeExists(nodeId)) { + return; + } + events.emit("logs", `Adding node ${nodeId} to network ${this.name}`); + let znet_light = new NetworkLight(); + if (!subnet) { + znet_light.subnet = this.getFreeSubnet(); + } else { + znet_light.subnet = subnet; + } + znet_light["ip_range"] = this.ipRange; + znet_light["node_id"] = nodeId; + + if (mycelium) { + const myceliumNetworkSeed = myceliumSeeds.find(item => item.nodeId === nodeId); + let seed = generateRandomHexSeed(32); + if (myceliumNetworkSeed?.seed) { + seed = myceliumNetworkSeed.seed; + validateHexSeed(seed, 32); + } + + znet_light.mycelium = { + hex_key: seed, + peers: [], + }; + } + + this.network = znet_light; + this.updateNetworkDeployments(); + znet_light = this.getUpdatedNetwork(znet_light); + + const znet_light_workload = new Workload(); + znet_light_workload.version = 0; + znet_light_workload.name = this.name; + znet_light_workload.type = WorkloadTypes.networklight; + znet_light_workload.data = znet_light; + znet_light_workload.metadata = ""; + znet_light_workload.description = description; + + this.node = new Node(); + this.node.node_id = nodeId; + + return znet_light_workload; + } + + _fromObj(net: NetworkLight): NetworkLight { + const znet_light = plainToInstance(NetworkLight, net); + return znet_light; + } + deleteReservedIp(node_id: number, ip: string): string { + if (this.nodeExists(node_id) && this.node) { + this.node.reserved_ips = this.node?.reserved_ips?.filter(item => item !== ip); + } + return ip; + } + getNodeReservedIps(node_id: number): string[] { + if (this.nodeExists(node_id)) { + return this.node ? this.node?.reserved_ips : []; + } + return []; + } + + getFreeIP(node_id: number, subnet = ""): string | undefined { + let ip; + if (this.network["node_id"] !== node_id && subnet) { + ip = Addr(subnet).mask(32).increment().increment(); + } else if (this.network["node_id"] === node_id) { + ip = Addr(this.getNodeSubnet(node_id)).mask(32).increment().increment(); + } else { + throw new ValidationError("node_id or subnet must be specified."); + } + if (ip) { + ip = ip.toString().split("/")[0]; + if (this.node.node_id === node_id) { + this.node.reserved_ips.push(ip); + return ip; + } + throw new ValidationError(`node_id is not in the network. Please add it first.`); + } + } + ValidateFreeSubnet(subnet): string { + const reservedSubnets = this.getReservedSubnets(); + if (!reservedSubnets.includes(subnet)) { + this.reservedSubnets.push(subnet); + return subnet; + } else { + throw new ValidationError(`subnet ${subnet} is not free.`); + } + } + + updateNetworkDeployments(): void { + for (const deployment of this.deployments) { + const workloads = deployment["workloads"]; + for (const workload of workloads) { + if (workload["type"] !== WorkloadTypes.networklight) { + continue; + } + if (this.network.subnet === workload["data"]["subnet"]) { + workload["data"] = this.network; + break; + } + } + deployment["workloads"] = workloads; + } + } + async checkMycelium(nodeId: number, mycelium: boolean, myceliumSeeds: MyceliumNetworkModel[] = []) { + if (!mycelium) return; + const myceliumNetworkSeed = myceliumSeeds.find(item => item.nodeId == nodeId); + if (this.network && this.network.mycelium && this.network.mycelium?.hex_key) { + if (myceliumSeeds && myceliumSeeds.length > 0 && myceliumNetworkSeed?.seed !== this.network.mycelium.hex_key) { + throw new ValidationError(`Another mycelium seed is used for this network ${this.name} on this ${nodeId}`); + } + } else { + // If network has no mycelium and user wanna update it and add mycelium. + let seed = generateRandomHexSeed(32); + console.log("this.network in here", this.network); + if (this.network) { + if (myceliumNetworkSeed?.seed) { + validateHexSeed(myceliumNetworkSeed.seed, 32); + seed = myceliumNetworkSeed.seed; + } + + this.network.mycelium = { + hex_key: seed, + peers: [], + }; + this.getUpdatedNetwork(this.network); + this.updateNetworkDeployments(); + + const deploymentFactory = new DeploymentFactory(this.config); + const filteredDeployments = this.deployments.filter(deployment => deployment["node_id"] === nodeId); + for (const deployment of filteredDeployments) { + const d = await deploymentFactory.fromObj(deployment); + for (const workload of d["workloads"]) { + workload.data["mycelium"]["hex_key"] = seed; + workload.data = this.getUpdatedNetwork(workload["data"]); + workload.version += 1; + } + return d; + } + } + } + } + nodeExists(node_id: number): boolean { + if (this.NodeIds && this.NodeIds.includes(node_id)) { + return true; + } + return false; + } + + validateUserIP(node_id: number, ip_address = "") { + const nodeSubnet = this.getNodeSubnet(node_id); + const ip = Addr(ip_address); + + if (!Addr(nodeSubnet).contains(ip)) { + throw new ValidationError(`Selected ip is not available in node subnet, node subnet: ${nodeSubnet}`); + } + if (this.node.node_id === node_id) { + this.node.reserved_ips.push(ip_address); + return ip_address; + } + } + + getNodeSubnet(node_id: number): string | undefined { + if (this.network["node_id"] === node_id) { + return this.network.subnet; + } + } + + getFreeSubnet(): string { + console.log("this.ipRange", this.ipRange.toString()); + const subnet = Addr(this.ipRange).mask(24).nextSibling().nextSibling(); + return subnet.toString(); + } + + getReservedSubnets(): string[] { + const subnet = this.getNodeSubnet(this.node.node_id); + if (subnet && !this.reservedSubnets.includes(subnet)) { + this.reservedSubnets.push(subnet); + } + + return this.reservedSubnets; + } + + private async getMyNetworkContracts(fetch = false) { + if (fetch || !this.contracts) { + let contracts = await this.tfClient.contracts.listMyNodeContracts({ + graphqlURL: this.config.graphqlURL, + type: "network-light", + }); + const alreadyFetchedContracts: GqlNodeContract[] = []; + for (const contract of ZNetworkLight.newContracts) { + if (contract.parsedDeploymentData!.type !== "network-light") continue; + const c = contracts.filter(c => +c.contractID === +contract.contractID); + if (c.length > 0) { + alreadyFetchedContracts.push(contract); + continue; + } + contracts.push(contract); + } + + for (const contract of alreadyFetchedContracts) { + const index = ZNetworkLight.newContracts.indexOf(contract); + if (index > -1) ZNetworkLight.newContracts.splice(index, 1); + } + + contracts = contracts.filter(c => !ZNetworkLight.deletedContracts.includes(+c.contractID)); + + const parsedContracts: Required[] = []; + + for (const contract of contracts) { + const parsedDeploymentData = JSON.parse(contract.deploymentData); + parsedContracts.push({ ...contract, parsedDeploymentData }); + } + + this.contracts = parsedContracts; + } + + return this.contracts; + } + async load(): Promise { + if (!(await this.exists())) { + return; + } + this.NodeIds = []; + await this.loadNetworkFromContracts(); + } + + private async getReservedIps(nodeId: number): Promise { + const node_twin_id = await this.capacity.getNodeTwinId(nodeId); + const payload = JSON.stringify({ network_name: this.name }); + let reservedIps: string[]; + try { + reservedIps = await this.rmb.request([node_twin_id], "zos.network.list_private_ips", payload); + } catch (e) { + (e as Error).message = formatErrorMessage(`Failed to list reserved ips from node ${nodeId}`, e); + throw e; + } + return reservedIps; + } + + private async loadNetworkFromContracts() { + const contracts = await this.getDeploymentContracts(this.name); + for (const contract of contracts) { + const node_twin_id = await this.capacity.getNodeTwinId(contract.nodeID); + const payload = JSON.stringify({ contract_id: +contract.contractID }); + let res: Deployment; + try { + res = await this.rmb.request([node_twin_id], "zos.deployment.get", payload); + } catch (e) { + (e as Error).message = formatErrorMessage(`Failed to load network deployment ${contract.contractID}`, e); + throw e; + } + res["node_id"] = contract.nodeID; + for (const workload of res.workloads) { + const data = workload.data as NetworkLight; + if (workload.type !== WorkloadTypes.networklight || workload.name !== this.name) { + continue; + } + if (workload.result.state === "deleted") { + continue; + } + const znet_light = this._fromObj(data); + znet_light["node_id"] = contract.nodeID; + const reservedIps = await this.getReservedIps(contract.nodeID); + + this.node = { + contract_id: +contract.contractID, + node_id: contract.nodeID, + reserved_ips: reservedIps, + }; + this.network = znet_light; + this.network["node_id"] = contract.nodeID; + this.deployments.push(res); + } + } + } + async deleteNode(node_id: number): Promise { + if (!(await this.exists())) { + return 0; + } + events.emit("logs", `Deleting node ${node_id} from network ${this.name}`); + let contract_id = 0; + const node = new Node(); + if (this.node.node_id !== node_id) { + this.node = node; + } else { + contract_id = this.node.contract_id; + } + this.reservedSubnets = this.reservedSubnets.filter(subnet => subnet === this.network.subnet); + + return contract_id; + } + async getDeploymentContracts(name: string) { + const contracts = await this.getMyNetworkContracts(true); + for (const contract of contracts) { + this.NodeIds.push(contract.nodeID); + } + return contracts.filter(c => c.parsedDeploymentData.name === name); + } + + private getContractsName(contracts: Required[]): string[] { + return Array.from(new Set(contracts.map(c => c.parsedDeploymentData.name))); + } + + private async listAllNetworks() { + const contracts = await this.getMyNetworkContracts(true); + return this.getContractsName(contracts); + } + + private async exists() { + return (await this.listAllNetworks()).includes(this.name); + } + + isPrivateIP(ip: string): boolean { + return PrivateIp(ip.split("/")[0]); + } +} + +export { ZNetworkLight, Node }; diff --git a/packages/grid_client/src/primitives/nodes.ts b/packages/grid_client/src/primitives/nodes.ts index 3c22b5eeb4..d174aa9a30 100644 --- a/packages/grid_client/src/primitives/nodes.ts +++ b/packages/grid_client/src/primitives/nodes.ts @@ -8,10 +8,11 @@ import urlJoin from "url-join"; import { RMB } from "../clients"; import { Graphql } from "../clients/graphql/client"; -import { formatErrorMessage } from "../helpers"; +import { Features, formatErrorMessage } from "../helpers"; import { send, sendWithFullResponse } from "../helpers/requests"; import { convertObjectToQueryString } from "../helpers/utils"; -import { FarmFilterOptions, FilterOptions, NodeStatus } from "../modules/models"; +import { FarmFilterOptions, FilterOptions, MachineModel, NodeStatus } from "../modules/models"; +import { WorkloadTypes } from "../zos"; interface FarmInfo { name: string; @@ -62,6 +63,7 @@ interface NodeInfo { rentable: boolean; rented: boolean; price_usd: number; + features: Features[]; } interface PublicConfig { domain: string; @@ -333,9 +335,44 @@ class Nodes { }); } + /** + * Sets the features of the node according to filter options. + * + * @param options - An object containing filter options to set the node's features. + * @returns A string array representing the node's features. + */ + getFeaturesFromFilters(options: FilterOptions = {}): Features[] { + const featuresSet: Set = new Set(options.features || []); + + if (options.publicIPs) { + featuresSet.add(Features.ipv4); + featuresSet.add(Features.ip); + } + if (options.hasIPv6) { + featuresSet.add(Features.ip); + } + if (options.wireguard) { + featuresSet.add(Features.wireguard); + } + if (options.mycelium) { + featuresSet.add(Features.mycelium); + } + if (options.planetary) { + featuresSet.add(Features.yggdrasil); + } + + if (options.gateway) { + featuresSet.add(Features.gatewayfqdnproxy); + featuresSet.add(Features.gatewaynameproxy); + } + + return Array.from(featuresSet); + } + async filterNodes(options: FilterOptions = {}, url = ""): Promise { let nodes: NodeInfo[] = []; url = url || this.proxyURL; + options.features = this.getFeaturesFromFilters(options); const query = this.getNodeUrlQuery(options); nodes = await send("get", urlJoin(url, `/nodes?${query}`), "", {}); if (nodes.length) { @@ -423,6 +460,7 @@ class Nodes { healthy: options.healthy, sort_by: SortBy.FreeCRU, sort_order: SortOrder.Desc, + features: options.features, }; if (options.gateway) { diff --git a/packages/grid_client/src/primitives/vm.ts b/packages/grid_client/src/primitives/vm.ts index 880a7d030a..0d99f761bd 100644 --- a/packages/grid_client/src/primitives/vm.ts +++ b/packages/grid_client/src/primitives/vm.ts @@ -1,20 +1,94 @@ import { ComputeCapacity } from "../zos/computecapacity"; import { Workload, WorkloadTypes } from "../zos/workload"; import { Mount, Zmachine, ZmachineNetwork, ZNetworkInterface } from "../zos/zmachine"; +import { MachineInterface, ZmachineLight, ZmachineLightNetwork } from "../zos/zmachine_light"; -class VMPrimitive { +abstract class VMBase { + // Method to create compute capacity to be overidden _createComputeCapacity(cpu: number, memory: number): ComputeCapacity { const compute_capacity = new ComputeCapacity(); compute_capacity.cpu = cpu; - compute_capacity.memory = memory * 1024 ** 2; + compute_capacity.memory = memory * 1024 ** 2; // Convert memory to bytes return compute_capacity; } + + // Method to create network interface to be overidden + _createNetworkInterface(networkName: string, ip: string): ZNetworkInterface | MachineInterface { + throw new Error("Method '_createNetworkInterface' must be implemented in a subclass"); + } + + // Method to create machine network to be overidden + _createMachineNetwork( + networkName: string, + ip: string, + planetary: boolean, + mycelium: boolean, + myceliumSeed: string, + public_ip = "", + ): ZmachineNetwork | ZmachineLightNetwork { + throw new Error("Method '_createMachineNetwork' must be implemented in a subclass"); + } + + // Method for creating a workload based on wokload type + _createWorkload( + name: string, + flist: string, + cpu: number, + memory: number, + rootfs_size: number, + disks: Mount[], + networkName: string, + ip: string, + planetary: boolean, + mycelium: boolean, + myceliumSeed: string, + public_ip: string, + entrypoint: string, + env: Record, + metadata: string, + description: string, + version: number, + corex: boolean, + gpus: string[], + vm_data: Zmachine | ZmachineLight, + workloadType: WorkloadTypes, + ): Workload { + const compute_capacity = this._createComputeCapacity(cpu, memory); + const vm_network = this._createMachineNetwork(networkName, ip, planetary, mycelium, myceliumSeed, public_ip); + + const vm_workload = new Workload(); + vm_workload.version = version || 0; + vm_workload.name = name; + vm_workload.type = workloadType; + vm_workload.data = vm_data; + vm_workload.metadata = metadata; + vm_workload.description = description; + + vm_data.flist = flist; + vm_data.network = vm_network; + vm_data.size = rootfs_size * 1024 ** 3; // Convert rootfs size to bytes + vm_data.mounts = disks; + vm_data.entrypoint = entrypoint; + vm_data.compute_capacity = compute_capacity; + vm_data.env = env; + vm_data.corex = corex; + vm_data.gpu = gpus; + + return vm_workload; + } +} + +// VMPrimitive class, extending VMBasePrimitive +class VMPrimitive extends VMBase { + // Override _createNetworkInterface method _createNetworkInterface(networkName: string, ip: string): ZNetworkInterface { const znetwork_interface = new ZNetworkInterface(); znetwork_interface.network = networkName; znetwork_interface.ip = ip; return znetwork_interface; } + + // Override _createMachineNetwork method _createMachineNetwork( networkName: string, ip: string, @@ -36,6 +110,8 @@ class VMPrimitive { } return zmachine_network; } + + // Create method to create VM workload create( name: string, flist: string, @@ -58,25 +134,110 @@ class VMPrimitive { gpus: string[] = [], ): Workload { const zmachine = new Zmachine(); - zmachine.flist = flist; - zmachine.network = this._createMachineNetwork(networkName, ip, planetary, mycelium, myceliumSeed, public_ip); - zmachine.size = rootfs_size * 1024 ** 3; - zmachine.mounts = disks; - zmachine.entrypoint = entrypoint; - zmachine.compute_capacity = this._createComputeCapacity(cpu, memory); - zmachine.env = env; - zmachine.corex = corex; - zmachine.gpu = gpus; - - const zmachine_workload = new Workload(); - zmachine_workload.version = version || 0; - zmachine_workload.name = name; - zmachine_workload.type = WorkloadTypes.zmachine; - zmachine_workload.data = zmachine; - zmachine_workload.metadata = metadata; - zmachine_workload.description = description; - return zmachine_workload; + return this._createWorkload( + name, + flist, + cpu, + memory, + rootfs_size, + disks, + networkName, + ip, + planetary, + mycelium, + myceliumSeed, + public_ip, + entrypoint, + env, + metadata, + description, + version, + corex, + gpus, + zmachine, + WorkloadTypes.zmachine, + ); + } +} + +// VMLightPrimitive class, extending VMBasePrimitive +class VMLightPrimitive extends VMBase { + // Override _createNetworkInterface + _createNetworkInterface(networkName: string, ip: string): MachineInterface { + const zlightnetwork_interface = new MachineInterface(); + zlightnetwork_interface.network = networkName; + zlightnetwork_interface.ip = ip; + return zlightnetwork_interface; + } + + // Override _createMachineNetwork method + _createMachineNetwork( + networkName: string, + ip: string, + planetary: boolean, + mycelium: boolean, + myceliumSeed: string, + public_ip = "", + ): ZmachineLightNetwork { + const zmachine_lightnetwork = new ZmachineLightNetwork(); + zmachine_lightnetwork.interfaces = [this._createNetworkInterface(networkName, ip)]; + + if (mycelium) { + zmachine_lightnetwork.mycelium = { + hex_seed: myceliumSeed, + network: networkName, + }; + } + return zmachine_lightnetwork; + } + + // Create method to create VM light workload + create( + name: string, + flist: string, + cpu: number, + memory: number, + rootfs_size: number, + disks: Mount[], + networkName: string, + ip: string, + planetary: boolean, + mycelium: boolean, + myceliumSeed: string, + public_ip: string, + entrypoint: string, + env: Record, + metadata = "", + description = "", + version = 0, + corex = false, + gpus: string[] = [], + ): Workload { + const zmachine_light = new ZmachineLight(); + return this._createWorkload( + name, + flist, + cpu, + memory, + rootfs_size, + disks, + networkName, + ip, + planetary, + mycelium, + myceliumSeed, + public_ip, + entrypoint, + env, + metadata, + description, + version, + corex, + gpus, + zmachine_light, + WorkloadTypes.zmachinelight, + ); } } -export { VMPrimitive }; +export { VMPrimitive, VMLightPrimitive }; diff --git a/packages/grid_client/src/zos/index.ts b/packages/grid_client/src/zos/index.ts index b0126274a0..de94ac40f9 100644 --- a/packages/grid_client/src/zos/index.ts +++ b/packages/grid_client/src/zos/index.ts @@ -11,3 +11,5 @@ export * from "./znet"; export * from "./gateway"; export * from "./qsfs"; export * from "./zlogs"; +export * from "./zmachine_light"; +export * from "./network_light"; diff --git a/packages/grid_client/src/zos/network_light.ts b/packages/grid_client/src/zos/network_light.ts new file mode 100644 index 0000000000..7385b263c7 --- /dev/null +++ b/packages/grid_client/src/zos/network_light.ts @@ -0,0 +1,31 @@ +import { Expose, Type } from "class-transformer"; +import { IsNotEmpty, IsOptional, IsString, ValidateNested } from "class-validator"; + +import { ValidateMembers } from "../helpers"; +import { WorkloadData } from "./workload_base"; + +class Mycelium { + @Expose() @IsString() @IsNotEmpty() hex_key: string; + @Expose() @IsOptional() @IsString({ each: true }) peers?: string[]; +} + +@ValidateMembers() +class NetworkLight extends WorkloadData { + @Expose() @IsString() @IsNotEmpty() subnet: string; + @Expose() @IsOptional() @Type(() => Mycelium) @ValidateNested() mycelium?: Mycelium; + + challenge(): string { + let out = ""; + out += this.subnet; + + out += this.mycelium?.hex_key || ""; + if (this.mycelium?.peers) { + for (let i = 0; i < this.mycelium?.peers?.length; i++) { + out += this.mycelium?.peers[i] || ""; + } + } + return out; + } +} + +export { NetworkLight }; diff --git a/packages/grid_client/src/zos/workload.ts b/packages/grid_client/src/zos/workload.ts index 99f32a7f10..fb6d21855a 100644 --- a/packages/grid_client/src/zos/workload.ts +++ b/packages/grid_client/src/zos/workload.ts @@ -4,6 +4,7 @@ import { IsDefined, IsEnum, IsInt, IsNotEmpty, IsString, Min, ValidateNested } f import { ValidateMembers } from "../helpers"; import { GatewayFQDNProxy, GatewayNameProxy, GatewayResult } from "./gateway"; import { PublicIPv4, PublicIPv4Result } from "./ipv4"; // TODO: remove deprecated +import { NetworkLight } from "./network_light"; import { PublicIP, PublicIPResult } from "./public_ip"; import { QuantumSafeFS, QuantumSafeFSResult } from "./qsfs"; import { Volume, VolumeResult } from "./volume"; @@ -11,6 +12,7 @@ import { WorkloadData, WorkloadDataResult } from "./workload_base"; import { Zdb, ZdbResult } from "./zdb"; import { Zlogs, ZlogsResult } from "./zlogs"; import { Zmachine, ZmachineResult } from "./zmachine"; +import { ZmachineLight, ZmachineLightResult } from "./zmachine_light"; import { Zmount, ZmountResult } from "./zmount"; import { Znet } from "./znet"; @@ -32,6 +34,8 @@ enum WorkloadTypes { gatewaynameproxy = "gateway-name-proxy", qsfs = "qsfs", zlogs = "zlogs", + networklight = "network-light", + zmachinelight = "zmachine-light", } class DeploymentResult { @@ -54,6 +58,8 @@ class DeploymentResult { { value: WorkloadDataResult, name: WorkloadTypes.gatewaynameproxy }, { value: QuantumSafeFSResult, name: WorkloadTypes.qsfs }, { value: ZlogsResult, name: WorkloadTypes.zlogs }, + { value: WorkloadDataResult, name: WorkloadTypes.networklight }, + { value: ZmachineLightResult, name: WorkloadTypes.zmachinelight }, ], }, }) @@ -67,7 +73,8 @@ class DeploymentResult { | QuantumSafeFSResult | WorkloadDataResult | GatewayResult - | ZlogsResult; + | ZlogsResult + | ZmachineLightResult; } @ValidateMembers() @@ -95,6 +102,8 @@ class Workload { { value: GatewayNameProxy, name: WorkloadTypes.gatewaynameproxy }, { value: QuantumSafeFS, name: WorkloadTypes.qsfs }, { value: Zlogs, name: WorkloadTypes.zlogs }, + { value: NetworkLight, name: WorkloadTypes.networklight }, + { value: ZmachineLight, name: WorkloadTypes.zmachinelight }, ], }, }) @@ -110,7 +119,9 @@ class Workload { | GatewayFQDNProxy | GatewayNameProxy | QuantumSafeFS - | Zlogs; + | Zlogs + | NetworkLight + | ZmachineLight; @Expose() @IsString() @IsDefined() metadata: string; @Expose() @IsString() @IsDefined() description: string; diff --git a/packages/grid_client/src/zos/zmachine_light.ts b/packages/grid_client/src/zos/zmachine_light.ts new file mode 100644 index 0000000000..ac32760343 --- /dev/null +++ b/packages/grid_client/src/zos/zmachine_light.ts @@ -0,0 +1,84 @@ +import { Expose, Transform, Type } from "class-transformer"; +import { + IsBoolean, + IsDefined, + IsInt, + IsIP, + IsNotEmpty, + IsOptional, + IsString, + IsUrl, + Max, + Min, + ValidateNested, +} from "class-validator"; + +import { ValidateMembers } from "../helpers"; +import { ComputeCapacity } from "./computecapacity"; +import { WorkloadData, WorkloadDataResult } from "./workload_base"; +import { Mount, MyceliumIP } from "./zmachine"; + +class MachineInterface { + @Expose() @IsString() @IsNotEmpty() network: string; + @Expose() @IsIP() @IsNotEmpty() ip: string; +} + +class ZmachineLightNetwork { + //what is ValidateNested + @Expose() @Type(() => MachineInterface) @ValidateNested({ each: true }) interfaces: MachineInterface[]; + @Expose() @Type(() => MyceliumIP) @ValidateNested() mycelium: MyceliumIP; + + challenge(): string { + let out = ""; + for (let i = 0; i < this.interfaces.length; i++) { + out += this.interfaces[i].network; + out += this.interfaces[i].ip; + } + out += this.mycelium?.network || ""; + out += this.mycelium?.hex_seed || ""; + return out; + } +} +@ValidateMembers() +class ZmachineLight extends WorkloadData { + @Expose() @IsString() @IsNotEmpty() @IsUrl() flist: string; + @Expose() @Type(() => ZmachineLightNetwork) @ValidateNested() network: ZmachineLightNetwork; + @Expose() @IsInt() @Min(0) @Max(10 * 1024 ** 4) size: number; // in bytes + @Expose() @Type(() => ComputeCapacity) @ValidateNested() compute_capacity: ComputeCapacity; + @Expose() @Type(() => Mount) @ValidateNested({ each: true }) mounts: Mount[]; + @Expose() @IsString() @IsDefined() entrypoint: string; + @Expose() env: Record; + @Expose() @Transform(({ value }) => (value ? true : false)) @IsBoolean() corex: boolean; + @Expose() @IsString({ each: true }) @IsOptional() gpu?: string[]; + + challenge(): string { + let out = ""; + out += this.flist; + out += this.network.challenge(); + out += this.size || "0"; + out += this.compute_capacity.challenge(); + for (let i = 0; i < this.mounts.length; i++) { + out += this.mounts[i].challenge(); + } + out += this.entrypoint; + for (const key of Object.keys(this.env).sort()) { + out += key; + out += "="; + out += this.env[key]; + if (this.gpu) { + for (const g of this.gpu) { + out += g; + } + } + } + return out; + } +} + +class ZmachineLightResult extends WorkloadDataResult { + @Expose() id: string; + @Expose() ip: string; + @Expose() mycelium_ip: string; +} + +export { ZmachineLight, ZmachineLightNetwork, MachineInterface, ZmachineLightResult }; diff --git a/packages/grid_client/tests/modules/network_light.test.ts b/packages/grid_client/tests/modules/network_light.test.ts new file mode 100644 index 0000000000..c143a7c77f --- /dev/null +++ b/packages/grid_client/tests/modules/network_light.test.ts @@ -0,0 +1,61 @@ +import { plainToClass } from "class-transformer"; + +import { NetworkLight } from "../../src"; +let networkLight: NetworkLight; + +beforeEach(() => { + networkLight = new NetworkLight(); + networkLight.subnet = "192.168.0.0/16"; + networkLight.mycelium = { + hex_key: "abc123", + peers: ["peer1", "peer2"], + }; +}); + +describe("NetworkLight Class Tests", () => { + it("should create a valid NetworkLight instance", () => { + expect(networkLight).toBeInstanceOf(NetworkLight); + }); + + it("should correctly serialize and deserialize a NetworkLight instance", () => { + const serialized = JSON.stringify(networkLight); + const deserialized = plainToClass(NetworkLight, JSON.parse(serialized)); + + expect(deserialized).toBeInstanceOf(NetworkLight); + expect(deserialized.challenge()).toBe(networkLight.challenge()); + }); + + it("should correctly compute the challenge string", () => { + const expectedChallenge = + networkLight.subnet + networkLight.mycelium.hex_key + networkLight.mycelium.peers?.join(""); + + expect(networkLight.challenge()).toBe(expectedChallenge); + }); + + it("should correctly handle missing peers", () => { + networkLight.mycelium.peers = undefined; + + const expectedChallenge = networkLight.subnet + networkLight.mycelium.hex_key; + expect(networkLight.challenge()).toBe(expectedChallenge); + }); + + it("should throw an error if subnet is empty", () => { + const setEmptySubnet = () => (networkLight.subnet = ""); + + expect(setEmptySubnet).toThrow(); + }); + + it("should handle an empty mycelium object correctly", () => { + networkLight.mycelium = undefined as any; + + const expectedChallenge = networkLight.subnet; + expect(networkLight.challenge()).toBe(expectedChallenge); + }); + + it("should correctly handle peers being empty", () => { + networkLight.mycelium.peers = []; + + const expectedChallenge = networkLight.subnet + networkLight.mycelium.hex_key; + expect(networkLight.challenge()).toBe(expectedChallenge); + }); +}); diff --git a/packages/grid_client/tests/modules/zmachine_light.test.ts b/packages/grid_client/tests/modules/zmachine_light.test.ts new file mode 100644 index 0000000000..f3f864c562 --- /dev/null +++ b/packages/grid_client/tests/modules/zmachine_light.test.ts @@ -0,0 +1,158 @@ +import { plainToClass } from "class-transformer"; + +import { ComputeCapacity, MachineInterface, Mount, ZmachineLight, ZmachineLightNetwork } from "../../src"; + +let zmachineLight = new ZmachineLight(); +const computeCapacity = new ComputeCapacity(); +const network = new ZmachineLightNetwork(); +const disks = new Mount(); + +beforeEach(() => { + computeCapacity.cpu = 1; + computeCapacity.memory = 256 * 1024 ** 2; + + network.interfaces = [ + { + network: "znetwork", + ip: "10.20.2.2", + }, + ]; + network.mycelium = { + network: "mycelium_net", + hex_seed: "abc123", + }; + + const rootfs_size = 2; + + disks.name = "zdisk"; + disks.mountpoint = "/mnt/data"; + + zmachineLight.flist = "https://hub.grid.tf/tf-official-vms/ubuntu-22.04.flist"; + zmachineLight.network = network; + zmachineLight.size = rootfs_size * 1024 ** 3; + zmachineLight.mounts = [disks]; + zmachineLight.entrypoint = "/sbin/zinit init"; + zmachineLight.compute_capacity = computeCapacity; + zmachineLight.env = { key: "value" }; + zmachineLight.corex = false; + zmachineLight.gpu = ["AMD", "NVIDIA"]; +}); + +describe("ZmachineLight Class Tests", () => { + it("should create a valid ZmachineLight instance", () => { + expect(zmachineLight).toBeInstanceOf(ZmachineLight); + }); + + it("should correctly serialize and deserialize a ZmachineLight instance", () => { + const serialized = JSON.stringify(zmachineLight); + const deserialized = plainToClass(ZmachineLight, JSON.parse(serialized)); + + expect(deserialized).toBeInstanceOf(ZmachineLight); + expect(deserialized.challenge()).toBe(zmachineLight.challenge()); + }); + + it("should correctly handle env vars", () => { + const challenge = zmachineLight.challenge(); + + expect(challenge).toContain("key=value"); + }); + + it("should correctly compute the challenge string", () => { + const expectedChallenge = + zmachineLight.flist + + network.challenge() + + zmachineLight.size + + computeCapacity.challenge() + + zmachineLight.mounts[0].challenge() + + zmachineLight.entrypoint + + JSON.stringify(zmachineLight.env) + .replace(/[{"}"]/g, "") + .replace(":", "=") + + zmachineLight.gpu?.toString().replace(",", ""); + + expect(zmachineLight.challenge()).toBe(expectedChallenge); + expect(zmachineLight.challenge()).toContain("key=value"); + }); + + it("should correctly handle the gpu array", () => { + expect(zmachineLight.gpu).toContain("NVIDIA"); + + zmachineLight.gpu = []; + + expect(zmachineLight.challenge()).not.toContain("NVIDIA"); + + zmachineLight.gpu = ["NVIDIA", "AMD"]; + + expect(zmachineLight.challenge()).toContain("NVIDIA"); + expect(zmachineLight.challenge()).toContain("AMD"); + }); + + it("should fail validation for entering invalid flist", () => { + const emptyFlist = () => (zmachineLight.flist = ""); + const invalidURL = () => (zmachineLight.flist = "www.invalid-url"); + + expect(emptyFlist).toThrow(); + expect(invalidURL).toThrow(); + }); + + it("should fail validation for entering invalid entrypoint", () => { + const invalidEntrypoint = () => (zmachineLight.entrypoint = undefined as any); + + expect(invalidEntrypoint).toThrow(); + }); + + it("should fail validation for entering invalid size", () => { + const maxSize = () => (zmachineLight.size = 10 * 1024 ** 5); + const decimalSize = () => (zmachineLight.size = 1.2); + const negativeSize = () => (zmachineLight.size = -1); + + expect(maxSize).toThrow(); + expect(decimalSize).toThrow(); + expect(negativeSize).toThrow(); + }); + + it("should throw error if network interfaces values are invalid", () => { + const invalidNetwork = new ZmachineLightNetwork(); + invalidNetwork.interfaces = [new MachineInterface()]; + zmachineLight.network.interfaces[0].network = ""; // invalid network + zmachineLight.network.interfaces[0].ip = ""; // invalid IP + + const result = () => { + zmachineLight.network = invalidNetwork; + }; + + expect(result).toThrow(); + }); + + it("should throw an error if mount name is empty", () => { + const invalidMount = new Mount(); + invalidMount.name = ""; + invalidMount.mountpoint = "/mnt/data"; + + const result = () => { + zmachineLight.mounts = [invalidMount]; + }; + + expect(result).toThrow(); + }); + + it("should fail if zmachineLight is parsed to an invalid object", () => { + const invalidZmachineLight = `{ + "flist": "", + "network": "invalid_network_object", + "size": "not_a_number", + "compute_capacity": {}, + "mounts": "not_an_array", + "entrypoint": 123, + "env": "not_an_object", + "corex": "not_a_boolean", + "gpu": [123, "valid_string", false] + }`; + + const result = () => { + zmachineLight = plainToClass(ZmachineLight, JSON.parse(invalidZmachineLight)); + }; + + expect(result).toThrow(); + }); +}); diff --git a/packages/playground/src/components/caprover_worker.vue b/packages/playground/src/components/caprover_worker.vue index c717e4abad..b76fbed740 100644 --- a/packages/playground/src/components/caprover_worker.vue +++ b/packages/playground/src/components/caprover_worker.vue @@ -51,6 +51,9 @@ solutionDisk: $props.modelValue.solution?.disk, memory: $props.modelValue.solution?.memory, rootFilesystemSize, + planetary: $props.modelValue.planetary, + mycelium: $props.modelValue.mycelium, + wireguard: $props.modelValue.wireguard, }" v-model="$props.modelValue.selectionDetails" /> diff --git a/packages/playground/src/components/k8s_worker.vue b/packages/playground/src/components/k8s_worker.vue index 437c772f06..a9cf2aea68 100644 --- a/packages/playground/src/components/k8s_worker.vue +++ b/packages/playground/src/components/k8s_worker.vue @@ -104,6 +104,9 @@ ssdDisks: [$props.modelValue.diskSize], memory: $props.modelValue.memory, rootFilesystemSize: $props.modelValue.rootFsSize, + planetary: $props.modelValue.planetary, + mycelium: $props.modelValue.mycelium, + wireguard: $props.modelValue.wireguard, }" v-model="$props.modelValue.selectionDetails" /> diff --git a/packages/playground/src/components/manage_gateway_dialog.vue b/packages/playground/src/components/manage_gateway_dialog.vue index a08939b94a..3337b87a01 100644 --- a/packages/playground/src/components/manage_gateway_dialog.vue +++ b/packages/playground/src/components/manage_gateway_dialog.vue @@ -9,7 +9,6 @@ > - Domains List Add new domain @@ -69,9 +68,7 @@ {{ item.name }} - +