From 0f485eda33f05754f06d9f58cce2ceefaf14f1b0 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Thu, 28 Sep 2023 13:45:56 +0300 Subject: [PATCH 1/5] Update listing path and add migration --- packages/grid_client/src/modules/base.ts | 30 ++++++++++++++++++++- packages/grid_client/src/storage/backend.ts | 6 ++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index c89f57faf3..29d4d85fcf 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -52,6 +52,10 @@ class BaseModule { ); } + newGetDeploymentPath(...paths: string[]): string { + return PATH.join(this.config.storePath, this.moduleName, this.projectName, ...paths); + } + getDeploymentPath(name: string): string { return PATH.join(this.config.storePath, this.projectName, this.moduleName, name); } @@ -105,8 +109,32 @@ class BaseModule { } } + async _migrateListKeys(): Promise { + const oldPath = this.getDeploymentPath(""); + const oldKeys = await this.backendStorage.list(oldPath); + const oldValues = await Promise.all( + oldKeys.map(key => { + return this.backendStorage.load(PATH.join(oldPath, key, "contracts.json")); + }), + ); + + for (let i = 0; i < oldKeys.length; i++) { + const key = oldKeys[i]; + const path = this.newGetDeploymentPath(key, key, "contracts.json"); + const value = oldValues[i]; + + await this.backendStorage.dump(path, value); + await this.backendStorage.remove(PATH.join(oldPath, key, "contracts.json")); + } + } + async _list(): Promise { - return await this.backendStorage.list(this.getDeploymentPath("")); + const list = await this.backendStorage.list(this.newGetDeploymentPath("")); + + if (list.length > 0) return list; + + await this._migrateListKeys(); + return this.backendStorage.list(this.newGetDeploymentPath("")); } async exists(name: string): Promise { diff --git a/packages/grid_client/src/storage/backend.ts b/packages/grid_client/src/storage/backend.ts index ab9abd09af..67a1b99c17 100644 --- a/packages/grid_client/src/storage/backend.ts +++ b/packages/grid_client/src/storage/backend.ts @@ -24,7 +24,7 @@ enum BackendStorageType { } class BackendStorage { - storage; + storage: BackendStorageInterface; constructor( public type: BackendStorageType = BackendStorageType.auto, substrateURL = "", @@ -76,8 +76,8 @@ class BackendStorage { return JSON.parse(data.toString()); } - async list(key: string) { - return await this.storage.list(key); + list(key: string): Promise { + return this.storage.list(key); } async remove(key: string) { From 1ec5d479200cd70dfd2727901a02aee9085a13a8 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Thu, 28 Sep 2023 16:40:27 +0300 Subject: [PATCH 2/5] Fix migrating --- packages/grid_client/src/modules/base.ts | 38 +++++++++++++----------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index 29d4d85fcf..ec1544026e 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -61,7 +61,7 @@ class BaseModule { } async getDeploymentContracts(name: string) { - const path = PATH.join(this.getDeploymentPath(name), "contracts.json"); + const path = this.newGetDeploymentPath(name, name, "contracts.json"); const contracts = await this.backendStorage.load(path); if (!contracts) { return []; @@ -70,7 +70,7 @@ class BaseModule { } async save(name: string, contracts: Record) { - const contractsPath = PATH.join(this.getDeploymentPath(name), "contracts.json"); + const contractsPath = PATH.join(this.newGetDeploymentPath(name, name), "contracts.json"); const wireguardPath = PATH.join(this.getDeploymentPath(name), `${name}.conf`); const oldContracts = await this.getDeploymentContracts(name); let StoreContracts = oldContracts; @@ -111,28 +111,32 @@ class BaseModule { async _migrateListKeys(): Promise { const oldPath = this.getDeploymentPath(""); + const oldKeys = await this.backendStorage.list(oldPath); - const oldValues = await Promise.all( - oldKeys.map(key => { - return this.backendStorage.load(PATH.join(oldPath, key, "contracts.json")); - }), + if (oldKeys.length === 0) { + return; + } + + const values = await Promise.all( + oldKeys.map(k => this.backendStorage.load(PATH.join(oldPath, k, "contracts.json"))), ); - for (let i = 0; i < oldKeys.length; i++) { - const key = oldKeys[i]; - const path = this.newGetDeploymentPath(key, key, "contracts.json"); - const value = oldValues[i]; + const ext1 = await Promise.all( + oldKeys.map((k, i) => this.backendStorage.dump(this.newGetDeploymentPath(k, k, "contracts.json"), values[i])), + ); - await this.backendStorage.dump(path, value); - await this.backendStorage.remove(PATH.join(oldPath, key, "contracts.json")); - } - } + const ext2 = await Promise.all( + oldKeys.map(k => this.backendStorage.remove(PATH.join(oldPath, k, "contracts.json"))), + ); - async _list(): Promise { - const list = await this.backendStorage.list(this.newGetDeploymentPath("")); + const setExtrinsics = ext1.flat(1); + const removeExtrinsics = ext2.flat(1); - if (list.length > 0) return list; + await this.tfClient.applyAllExtrinsics(setExtrinsics); + await this.tfClient.applyAllExtrinsics(removeExtrinsics); + } + async _list(): Promise { await this._migrateListKeys(); return this.backendStorage.list(this.newGetDeploymentPath("")); } From edd074c7125a677e24e46db36c00f0bd63e12621 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Wed, 4 Oct 2023 15:37:47 +0300 Subject: [PATCH 3/5] Fix migrating path --- packages/grid_client/src/modules/base.ts | 30 ++++++++++++++++-------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index 3a956c373c..2eab085ec0 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -116,19 +116,29 @@ class BaseModule { oldKeys.map(k => this.backendStorage.load(PATH.join(oldPath, k, "contracts.json"))), ); - const ext1 = await Promise.all( - oldKeys.map((k, i) => this.backendStorage.dump(this.newGetDeploymentPath(k, k, "contracts.json"), values[i])), - ); + let ext1: any[] = []; + let ext2: any[] = []; + + for (let i = 0; i < oldKeys.length; i++) { + const key = oldKeys[i]; + const value = values[i]; + const from = PATH.join(oldPath, key, "contracts.json"); + const to = this.newGetDeploymentPath(key, key, "contracts.json"); + + if (value) { + ext1.push(this.backendStorage.dump(to, value)); + ext2.push(this.backendStorage.remove(from)); + } + } - const ext2 = await Promise.all( - oldKeys.map(k => this.backendStorage.remove(PATH.join(oldPath, k, "contracts.json"))), - ); + ext1 = await Promise.all(ext1.flat(1)); + ext2 = await Promise.all(ext2.flat(1)); - const setExtrinsics = ext1.flat(1); - const removeExtrinsics = ext2.flat(1); + ext1 = ext1.filter(x => !!x); + ext2 = ext2.filter(x => !!x); - await this.tfClient.applyAllExtrinsics(setExtrinsics); - await this.tfClient.applyAllExtrinsics(removeExtrinsics); + await this.tfClient.applyAllExtrinsics(ext1); + await this.tfClient.applyAllExtrinsics(ext2); } async _list(): Promise { From 6c97d52213c762d3968e984f09bf02fe42113588 Mon Sep 17 00:00:00 2001 From: MohamedElmdary Date: Thu, 5 Oct 2023 12:18:53 +0300 Subject: [PATCH 4/5] Check storageType before apply exts and rename deploymentPath method --- packages/grid_client/src/modules/base.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index 2eab085ec0..5bdd9855d7 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -47,7 +47,7 @@ class BaseModule { this.tfClient = config.tfclient; } - newGetDeploymentPath(...paths: string[]): string { + getNewDeploymentPath(...paths: string[]): string { return PATH.join(this.config.storePath, this.moduleName, this.projectName, ...paths); } @@ -56,7 +56,7 @@ class BaseModule { } async getDeploymentContracts(name: string) { - const path = this.newGetDeploymentPath(name, name, "contracts.json"); + const path = this.getNewDeploymentPath(name, name, "contracts.json"); const contracts = await this.backendStorage.load(path); if (!contracts) { return []; @@ -65,7 +65,7 @@ class BaseModule { } async save(name: string, contracts: Record) { - const contractsPath = PATH.join(this.newGetDeploymentPath(name, name), "contracts.json"); + const contractsPath = PATH.join(this.getNewDeploymentPath(name, name), "contracts.json"); const wireguardPath = PATH.join(this.getDeploymentPath(name), `${name}.conf`); const oldContracts = await this.getDeploymentContracts(name); let StoreContracts = oldContracts; @@ -123,7 +123,7 @@ class BaseModule { const key = oldKeys[i]; const value = values[i]; const from = PATH.join(oldPath, key, "contracts.json"); - const to = this.newGetDeploymentPath(key, key, "contracts.json"); + const to = this.getNewDeploymentPath(key, key, "contracts.json"); if (value) { ext1.push(this.backendStorage.dump(to, value)); @@ -134,16 +134,18 @@ class BaseModule { ext1 = await Promise.all(ext1.flat(1)); ext2 = await Promise.all(ext2.flat(1)); - ext1 = ext1.filter(x => !!x); - ext2 = ext2.filter(x => !!x); + if (this.backendStorage.type === BackendStorageType.tfkvstore) { + ext1 = ext1.filter(x => !!x); + ext2 = ext2.filter(x => !!x); - await this.tfClient.applyAllExtrinsics(ext1); - await this.tfClient.applyAllExtrinsics(ext2); + await this.tfClient.applyAllExtrinsics(ext1); + await this.tfClient.applyAllExtrinsics(ext2); + } } async _list(): Promise { await this._migrateListKeys(); - return this.backendStorage.list(this.newGetDeploymentPath("")); + return this.backendStorage.list(this.getNewDeploymentPath("")); } async exists(name: string): Promise { From 89552dc52181e178ce7375a270e25c83b149c18e Mon Sep 17 00:00:00 2001 From: Mohamed Rabie <31689104+MohamedElmdary@users.noreply.github.com> Date: Sun, 22 Oct 2023 10:31:52 +0300 Subject: [PATCH 5/5] Implement manage gateways in playground (#1215) * Add dialog to deploy gateway * Fix gateways listing & deploying path * Add listing gateways * Add list & deleting gateways * Add dialog to micro vm * Update project name in all solutions * Update old contracts projectName * Fix projectName path (add missing '/') * Add missing 'no-data-text' * Fix listing issue * Merge ext1/ext2 to be applied together * Remove contractId check * Fix typo backends to backend * Add tls pass through option * Update tooltip * Remove unload (loaded with no contracts) deployments from listing * Use domainName component instead of new template * Add visit action in gateways table * Migrate gateways from old solutions * add prefix projectName + twinId + gatewayName to gateways deployed for fullvm & microvm * Remove prefix while listing * Fix building errors * Fix issue in deploying and listing k8s --- packages/grid_client/src/modules/base.ts | 90 ++++- .../playground/src/components/domain_name.vue | 41 ++- .../playground/src/components/list_table.vue | 2 + .../src/components/manage_gateway_dialog.vue | 307 ++++++++++++++++++ .../src/components/vm_deployment_table.vue | 6 + .../src/components/weblet_layout.vue | 2 +- packages/playground/src/utils/gateway.ts | 14 +- .../playground/src/utils/load_deployment.ts | 49 ++- packages/playground/src/weblets/freeflow.vue | 2 +- packages/playground/src/weblets/full_vm.vue | 2 +- packages/playground/src/weblets/micro_vm.vue | 2 +- .../playground/src/weblets/tf_algorand.vue | 2 +- .../playground/src/weblets/tf_caprover.vue | 2 +- .../playground/src/weblets/tf_casperlabs.vue | 2 +- .../src/weblets/tf_deployment_list.vue | 19 ++ .../playground/src/weblets/tf_discourse.vue | 2 +- .../playground/src/weblets/tf_funkwhale.vue | 2 +- .../playground/src/weblets/tf_kubernetes.vue | 2 +- .../playground/src/weblets/tf_mattermost.vue | 2 +- .../playground/src/weblets/tf_nextcloud.vue | 2 +- .../playground/src/weblets/tf_node_pilot.vue | 2 +- .../playground/src/weblets/tf_owncloud.vue | 2 +- .../playground/src/weblets/tf_peertube.vue | 2 +- .../playground/src/weblets/tf_presearch.vue | 2 +- .../playground/src/weblets/tf_subsquid.vue | 2 +- packages/playground/src/weblets/tf_taiga.vue | 2 +- packages/playground/src/weblets/tf_umbrel.vue | 2 +- .../playground/src/weblets/tf_wordpress.vue | 2 +- 28 files changed, 513 insertions(+), 55 deletions(-) create mode 100644 packages/playground/src/components/manage_gateway_dialog.vue diff --git a/packages/grid_client/src/modules/base.ts b/packages/grid_client/src/modules/base.ts index 5bdd9855d7..1fbd0c1b2a 100644 --- a/packages/grid_client/src/modules/base.ts +++ b/packages/grid_client/src/modules/base.ts @@ -56,7 +56,7 @@ class BaseModule { } async getDeploymentContracts(name: string) { - const path = this.getNewDeploymentPath(name, name, "contracts.json"); + const path = this.getNewDeploymentPath(name, "contracts.json"); const contracts = await this.backendStorage.load(path); if (!contracts) { return []; @@ -65,7 +65,7 @@ class BaseModule { } async save(name: string, contracts: Record) { - const contractsPath = PATH.join(this.getNewDeploymentPath(name, name), "contracts.json"); + const contractsPath = PATH.join(this.getNewDeploymentPath(name), "contracts.json"); const wireguardPath = PATH.join(this.getDeploymentPath(name), `${name}.conf`); const oldContracts = await this.getDeploymentContracts(name); let StoreContracts = oldContracts; @@ -113,17 +113,88 @@ class BaseModule { } const values = await Promise.all( - oldKeys.map(k => this.backendStorage.load(PATH.join(oldPath, k, "contracts.json"))), + oldKeys.map(k => + this.backendStorage.load(PATH.join(oldPath, k, "contracts.json")).catch(error => { + console.log(`Error while fetching contarct data PATH[${PATH.join(oldPath, k, "contracts.json")}]`, error); + return null; + }), + ), ); + const contracts = await Promise.all( + values.flat(1).map(value => { + if (value) return this.tfClient.contracts.get({ id: value.contract_id }); + return Promise.resolve(null); + }), + ); + + const updateContractsExts: Promise[] = []; + for (const contract of contracts) { + if (!contract) { + continue; + } + + const { nodeContract } = contract.contractType || {}; + if (!nodeContract) { + continue; + } + + const { deploymentData, deploymentHash: hash } = nodeContract; + const oldData = JSON.parse(deploymentData || "{}") as unknown as { + type: string; + name: string; + projectName: string; + }; + const { name, projectName } = oldData; + + let instanceName = name; + + if (this.moduleName === "gateways") { + const [, instance] = name.split(this.config.twinId.toString()); + if (instance) { + instanceName = instance; + } + } + + if (projectName?.endsWith(`/${instanceName}`)) { + continue; + } + + oldData.projectName = `${projectName}/${instanceName}`; + + updateContractsExts.push( + this.tfClient.contracts.updateNode({ + id: contract.contractId, + data: JSON.stringify(oldData), + hash, + }), + ); + } + + const _updateContractsExts = await Promise.all(updateContractsExts.flat(1)); + const __updateContractsExts = _updateContractsExts.flat(1).filter(Boolean); + await this.tfClient.applyAllExtrinsics(__updateContractsExts); + let ext1: any[] = []; let ext2: any[] = []; for (let i = 0; i < oldKeys.length; i++) { - const key = oldKeys[i]; + const oldKey = oldKeys[i]; + + let newKey = oldKey; + if (this.moduleName === "gateways") { + const [, key] = oldKey.split(this.config.twinId.toString()); + if (key) { + newKey = key; + } + } + const value = values[i]; - const from = PATH.join(oldPath, key, "contracts.json"); - const to = this.getNewDeploymentPath(key, key, "contracts.json"); + const from = PATH.join(oldPath, oldKey, "contracts.json"); + const to = this.getNewDeploymentPath( + ...(this.projectName.includes(newKey) ? [oldKey] : [newKey, oldKey]), + "contracts.json", + ); if (value) { ext1.push(this.backendStorage.dump(to, value)); @@ -135,11 +206,10 @@ class BaseModule { ext2 = await Promise.all(ext2.flat(1)); if (this.backendStorage.type === BackendStorageType.tfkvstore) { - ext1 = ext1.filter(x => !!x); - ext2 = ext2.filter(x => !!x); + ext1 = ext1.flat(1).filter(x => !!x); + ext2 = ext2.flat(1).filter(x => !!x); - await this.tfClient.applyAllExtrinsics(ext1); - await this.tfClient.applyAllExtrinsics(ext2); + await this.tfClient.applyAllExtrinsics(ext1.concat(ext2)); } } diff --git a/packages/playground/src/components/domain_name.vue b/packages/playground/src/components/domain_name.vue index 5d2cd2c12e..6c7a28b5c5 100644 --- a/packages/playground/src/components/domain_name.vue +++ b/packages/playground/src/components/domain_name.vue @@ -1,6 +1,6 @@