Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Calculate the root file system when creating the deployment machine #3006

Merged
merged 2 commits into from
Jun 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/grid_client/src/helpers/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ export * from "./events";
export * from "./validator";
export * from "./expose";
export * from "./migration";
export * from "./root_fs";
39 changes: 39 additions & 0 deletions packages/grid_client/src/helpers/root_fs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { Decimal } from "decimal.js";

const GB = 1024;

interface RootFSOptions {
/** The machine CPU, should be in cores e.g. 5 cores*/
CPUCores: number;
/** The machine memory, should be in megabytes e.g. 1024 or 2048 MG*/
RAMInMegaBytes: number;
}

/**
* Calculate the root filesystem size (CU - Compute Units) based on provided options.
*
* This function calculates the compute units (CU) required based on the CPU cores and RAM in megabytes.
* If both CPU cores and RAM are provided in the `options` parameter, it calculates CU by multiplying
* the CPU cores with RAM and dividing by 8 * GB, then converting the result to an integer. If the
* calculated CU is zero, it returns 500 / GB; otherwise, it returns 2.
*
* @param {RootFSOptions} [options] - Optional configuration object.
* @param {number} [options.CPUCores] - The number of CPU cores.
* @param {number} [options.RAMInMegaBytes] - The RAM size in megabytes.
*
* @returns {number} - The calculated compute units (CU) based on the provided options.
*/
function calculateRootFileSystem(options?: RootFSOptions): number {
let cu = 0;

if (options && options.CPUCores && options.RAMInMegaBytes) {
cu = new Decimal(options.CPUCores)
.mul(options.RAMInMegaBytes)
.divToInt(8 * GB)
.toNumber();
}

return cu === 0 ? 500 / GB : 2;
}

export { calculateRootFileSystem, RootFSOptions };
7 changes: 7 additions & 0 deletions packages/grid_client/src/high_level/machine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { GridClientErrors, ValidationError } from "@threefold/types";
import { Addr } from "netaddr";

import { events } from "../helpers/events";
import { calculateRootFileSystem } from "../helpers/root_fs";
import { randomChoice, zeroPadding } from "../helpers/utils";
import { validateHexSeed } from "../helpers/validator";
import { DiskModel, MyceliumNetworkModel, QSFSDiskModel } from "../modules/models";
Expand Down Expand Up @@ -53,6 +54,12 @@ class VMHL extends HighLevelBase {
zlogsOutput?: string,
gpus: string[] = [],
): Promise<[TwinDeployment[], string]> {
if (!rootfs_size) {
rootfs_size = calculateRootFileSystem({
CPUCores: cpu,
RAMInMegaBytes: memory,
});
}
const deployments: TwinDeployment[] = [];
const workloads: Workload[] = [];
let totalDisksSize = rootfs_size;
Expand Down
15 changes: 15 additions & 0 deletions packages/grid_client/tests/unittests/root_fs.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { calculateRootFileSystem } from "../../src/helpers/root_fs";

describe("Calculate the rootFS size based on the machine specs", () => {
it("should return 2GB when the options are provided", () => {
const options = { CPUCores: 5, RAMInMegaBytes: 2048 };
const result = calculateRootFileSystem(options);
expect(result).toEqual(2);
});

it("should return 0.48828125 when CPU cores and RAM are zero", () => {
const options = { CPUCores: 0, RAMInMegaBytes: 0 };
const result = calculateRootFileSystem(options);
expect(result).toEqual(0.48828125);
});
});
7 changes: 5 additions & 2 deletions packages/playground/src/components/caprover_worker.vue
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,13 @@
</template>

<script lang="ts">
import { calculateRootFileSystem } from "@threefold/grid_client";
import { computed, type PropType } from "vue";

import { manual } from "@/utils/manual";

import Networks from "../components/networks.vue";
import type { CaproverWorker } from "../types";
import rootFs from "../utils/root_fs";
import { generateName } from "../utils/strings";
import SelectSolutionFlavor from "./select_solution_flavor.vue";

Expand All @@ -68,7 +68,10 @@ export default {
setup(props) {
const rootFilesystemSize = computed(() => {
const { cpu = 0, memory = 0 } = props.modelValue.solution || {};
return rootFs(cpu, memory);
return calculateRootFileSystem({
CPUCores: cpu,
RAMInMegaBytes: memory,
});
});

return { rootFilesystemSize, manual };
Expand Down
11 changes: 8 additions & 3 deletions packages/playground/src/components/k8s_worker.vue
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,12 @@
<TfSelectionDetails
:filters-validators="{
memory: { min: 1024 },
rootFilesystemSize: { min: rootFs($props.modelValue.cpu ?? 0, $props.modelValue.memory ?? 0) },
rootFilesystemSize: {
min: calculateRootFileSystem({
CPUCores: $props.modelValue.cpu ?? 0,
RAMInMegaBytes: $props.modelValue.memory ?? 0,
}),
},
}"
:filters="{
ipv4: $props.modelValue.ipv4,
Expand All @@ -103,13 +108,13 @@
</template>

<script lang="ts">
import { calculateRootFileSystem } from "@threefold/grid_client";
import type { PropType } from "vue";

import { manual } from "@/utils/manual";

import Networks from "../components/networks.vue";
import type { K8SWorker } from "../types";
import rootFs from "../utils/root_fs";
import { generateName } from "../utils/strings";
import RootFsSize from "./root_fs_size.vue";

Expand Down Expand Up @@ -140,7 +145,7 @@ export default {
},
},
setup() {
return { rootFs, manual };
return { calculateRootFileSystem, manual };
},
};
</script>
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ import { ref } from "vue";

import { useGrid } from "../stores";
import { addMachine, deleteMachine, loadVM } from "../utils/deploy_vm";
import rootFs from "../utils/root_fs";

const props = defineProps<{ master: any; data: any[]; projectName: string }>();
const emits = defineEmits<{ (event: "close"): void; (event: "update:caprover", data: any): void }>();
Expand Down Expand Up @@ -139,7 +138,10 @@ async function deploy(layout: any) {
{ key: "SWM_NODE_MODE", value: "worker" },
{ key: "PUBLIC_KEY", value: props.master.env.PUBLIC_KEY },
],
rootFilesystemSize: rootFs(worker.value.solution!.cpu, worker.value.solution!.memory),
rootFilesystemSize: calculateRootFileSystem({
CPUCores: worker.value.solution!.cpu,
RAMInMegaBytes: worker.value.solution!.memory,
}),
});

const [leader, ...workers] = vm;
Expand Down Expand Up @@ -181,7 +183,7 @@ async function onDelete(cb: (workers: any[]) => void) {
</script>

<script lang="ts">
import type { GridClient } from "@threefold/grid_client";
import { calculateRootFileSystem, type GridClient } from "@threefold/grid_client";

import CaproverWorker, { createWorker } from "../components/caprover_worker.vue";
import ListTable from "../components/list_table.vue";
Expand Down
8 changes: 4 additions & 4 deletions packages/playground/src/components/root_fs_size.vue
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,17 @@
</template>

<script lang="ts" setup>
import { calculateRootFileSystem } from "@threefold/grid_client";
import { ref, watch } from "vue";

import type { Validators } from "../types";
import rootFs from "../utils/root_fs";

const props = defineProps<{ cpu?: number; memory?: number; modelValue?: number }>();
const emits = defineEmits<{ (event: "update:model-value", value: number): void }>();

const input = ref();

const value = ref(rootFs(props.cpu ?? 0, props.memory ?? 0));
const value = ref(calculateRootFileSystem({ CPUCores: props.cpu ?? 0, RAMInMegaBytes: props.memory ?? 0 }));
watch(value, value => emits("update:model-value", value), { immediate: true });

const edit = ref(false);
Expand All @@ -47,14 +47,14 @@ watch(
await input.value.validate(value.value);

if (!edit) {
value.value = rootFs(cpu, memory);
value.value = calculateRootFileSystem({ CPUCores: cpu, RAMInMegaBytes: memory });
}
},
);

function dynamicValidateRootFs(validators: Validators) {
return (value: string) => {
const min = rootFs(props.cpu ?? 0, props.memory ?? 0);
const min = calculateRootFileSystem({ CPUCores: props.cpu ?? 0, RAMInMegaBytes: props.memory ?? 0 });
return validators.min(`SSD Storage min value is ${min}GB.`, min)(value);
};
}
Expand Down
16 changes: 0 additions & 16 deletions packages/playground/src/utils/root_fs.ts

This file was deleted.

7 changes: 4 additions & 3 deletions packages/playground/src/weblets/freeflow.vue
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@
</template>

<script lang="ts" setup>
import type { GridClient } from "@threefold/grid_client";
import { calculateRootFileSystem, type GridClient } from "@threefold/grid_client";
import { computed, onMounted, type Ref, ref } from "vue";

import { manual } from "@/utils/manual";
Expand All @@ -103,7 +103,9 @@ const disks = ref<Disk[]>([]);
const dedicated = ref(false);
const certified = ref(false);
const ipv4 = ref(false);
const rootFilesystemSize = computed(() => rootFs(solution.value?.cpu ?? 0, solution.value?.memory ?? 0));
const rootFilesystemSize = computed(() =>
calculateRootFileSystem({ CPUCores: solution.value?.cpu ?? 0, RAMInMegaBytes: solution.value?.memory ?? 0 }),
);
const selectionDetails = ref<SelectionDetails>();
const selectedSSHKeys = ref("");
const gridStore = useGrid();
Expand Down Expand Up @@ -214,7 +216,6 @@ import ManageSshDeployemnt from "../components/ssh_keys/ManageSshDeployemnt.vue"
import { deploymentListEnvironments } from "../constants";
import type { SelectionDetails } from "../types/nodeSelector";
import { updateGrid } from "../utils/grid";
import rootFs from "../utils/root_fs";

export default {
name: "TFFreeflow",
Expand Down
22 changes: 17 additions & 5 deletions packages/playground/src/weblets/tf_caprover.vue
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,18 @@
:disk="
workers.reduce(
(disk, worker) =>
disk + (worker.solution?.disk ?? 0) + rootFs(worker.solution?.cpu ?? 0, worker.solution?.memory ?? 0),
leader.solution?.disk ?? 0 + rootFs(leader.solution?.cpu ?? 0, leader.solution?.memory ?? 0),
disk +
(worker.solution?.disk ?? 0) +
calculateRootFileSystem({
CPUCores: worker.solution?.cpu ?? 0,
RAMInMegaBytes: worker.solution?.memory ?? 0,
}),
leader.solution?.disk ??
0 +
calculateRootFileSystem({
CPUCores: leader.solution?.cpu ?? 0,
RAMInMegaBytes: leader.solution?.memory ?? 0,
}),
)
"
:ipv4="true"
Expand Down Expand Up @@ -169,7 +179,10 @@ function normalizeCaproverWorker(worker: CW, envs: Env[]): Machine {
publicIpv4: true,
planetary: true,
mycelium: worker.mycelium || false,
rootFilesystemSize: rootFs(worker.solution!.cpu, worker.solution!.memory),
rootFilesystemSize: calculateRootFileSystem({
CPUCores: worker.solution!.cpu,
RAMInMegaBytes: worker.solution!.memory,
}),
disks: [
{
name: "data0",
Expand All @@ -190,7 +203,7 @@ function updateSSHkeyEnv(selectedKeys: string) {
</script>

<script lang="ts">
import type { GridClient } from "@threefold/grid_client";
import { calculateRootFileSystem, type GridClient } from "@threefold/grid_client";

import { createNetwork } from "@/utils/deploy_helpers";

Expand All @@ -199,7 +212,6 @@ import ExpandableLayout from "../components/expandable_layout.vue";
import ManageSshDeployemnt from "../components/ssh_keys/ManageSshDeployemnt.vue";
import { deploymentListEnvironments } from "../constants";
import { updateGrid } from "../utils/grid";
import rootFs from "../utils/root_fs";

export default {
name: "TfCaprover",
Expand Down
7 changes: 4 additions & 3 deletions packages/playground/src/weblets/tf_casperlabs.vue
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
</template>

<script lang="ts" setup>
import type { GridClient } from "@threefold/grid_client";
import { calculateRootFileSystem, type GridClient } from "@threefold/grid_client";
import { computed, type Ref, ref } from "vue";

import { manual } from "@/utils/manual";
Expand Down Expand Up @@ -99,7 +99,9 @@ const certified = ref(false);
const ipv4 = ref(false);
const mycelium = ref(true);
const planetary = ref(false);
const rootFilesystemSize = computed(() => rootFs(solution.value?.cpu ?? 0, solution.value?.memory ?? 0));
const rootFilesystemSize = computed(() =>
calculateRootFileSystem({ CPUCores: solution.value?.cpu ?? 0, RAMInMegaBytes: solution.value?.memory ?? 0 }),
);
const selectionDetails = ref<SelectionDetails>();
const selectedSSHKeys = ref("");
const gridStore = useGrid();
Expand Down Expand Up @@ -203,7 +205,6 @@ import ManageSshDeployemnt from "../components/ssh_keys/ManageSshDeployemnt.vue"
import { deploymentListEnvironments } from "../constants";
import type { SelectionDetails } from "../types/nodeSelector";
import { updateGrid } from "../utils/grid";
import rootFs from "../utils/root_fs";

export default {
name: "TFCasperlabs",
Expand Down
7 changes: 4 additions & 3 deletions packages/playground/src/weblets/tf_discourse.vue
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
</template>

<script lang="ts" setup>
import type { GridClient } from "@threefold/grid_client";
import { calculateRootFileSystem, type GridClient } from "@threefold/grid_client";
import { Buffer } from "buffer";
import TweetNACL from "tweetnacl";
import { computed, type Ref, ref, watch } from "vue";
Expand All @@ -113,7 +113,6 @@ import { ProjectName } from "../types";
import { deployVM } from "../utils/deploy_vm";
import { deployGatewayName, getSubdomain, rollbackDeployment } from "../utils/gateway";
import { normalizeError } from "../utils/helpers";
import rootFs from "../utils/root_fs";
import { generateName, generatePassword } from "../utils/strings";

const layout = useLayout();
Expand All @@ -128,7 +127,9 @@ const planetary = ref(true);
const smtp = ref(createSMTPServer());
const dedicated = ref(false);
const certified = ref(false);
const rootFilesystemSize = computed(() => rootFs(solution.value?.cpu ?? 0, solution.value?.memory ?? 0));
const rootFilesystemSize = computed(() =>
calculateRootFileSystem({ CPUCores: solution.value?.cpu ?? 0, RAMInMegaBytes: solution.value?.memory ?? 0 }),
);
const selectionDetails = ref<SelectionDetails>();
const flist: Flist = {
value: "https://hub.grid.tf/tf-official-apps/forum-docker-v3.1.2.flist",
Expand Down
7 changes: 4 additions & 3 deletions packages/playground/src/weblets/tf_funkwhale.vue
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@
</template>

<script lang="ts" setup>
import type { GridClient } from "@threefold/grid_client";
import { calculateRootFileSystem, type GridClient } from "@threefold/grid_client";
import { computed, type Ref, ref } from "vue";

import { manual } from "@/utils/manual";
Expand All @@ -144,7 +144,9 @@ const username = ref("admin");
const email = ref(profileManager.profile?.email || "");
const password = ref(generatePassword(12));
const solution = ref() as Ref<SolutionFlavor>;
const rootFilesystemSize = computed(() => rootFs(solution.value?.cpu ?? 0, solution.value?.memory ?? 0));
const rootFilesystemSize = computed(() =>
calculateRootFileSystem({ CPUCores: solution.value?.cpu ?? 0, RAMInMegaBytes: solution.value?.memory ?? 0 }),
);
const flist: Flist = {
value: "https://hub.grid.tf/tf-official-apps/funkwhale-dec21.flist",
entryPoint: "/init.sh",
Expand Down Expand Up @@ -264,7 +266,6 @@ import ManageSshDeployemnt from "../components/ssh_keys/ManageSshDeployemnt.vue"
import { deploymentListEnvironments } from "../constants";
import type { SelectionDetails } from "../types/nodeSelector";
import { updateGrid } from "../utils/grid";
import rootFs from "../utils/root_fs";

export default {
name: "TfFunkwhale",
Expand Down
Loading
Loading