From 4e4692ee67c32653caa5b176fa2c6b3958d2d97b Mon Sep 17 00:00:00 2001 From: Enderson Maia Date: Mon, 6 May 2024 18:55:14 +0100 Subject: [PATCH] fixup! feat(cli): use cruntime to build snapshot --- apps/cli/package.json | 2 + apps/cli/src/commands/build.ts | 136 ++++++++++++--------------------- apps/cli/src/commands/shell.ts | 2 +- pnpm-lock.yaml | 47 ++++++++++++ 4 files changed, 100 insertions(+), 87 deletions(-) diff --git a/apps/cli/package.json b/apps/cli/package.json index af7e93c2..bac26f3e 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -37,6 +37,7 @@ "ora": "^8.0.1", "progress-stream": "^2.0", "semver": "^7.5", + "tar-stream": "^3.1.7", "tmp": "^0.2.3", "viem": "^2.9.29" }, @@ -52,6 +53,7 @@ "@types/progress-stream": "^2.0", "@types/prompts": "^2.4", "@types/semver": "^7.5.8", + "@types/tar-stream": "^3.1.3", "@types/tmp": "^0.2", "@wagmi/cli": "^2.1.4", "copyfiles": "^2", diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index ba2f2e86..258b720b 100644 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -2,8 +2,9 @@ import { Flags } from "@oclif/core"; import bytes from "bytes"; import { execa } from "execa"; import fs from "fs-extra"; -import { createTar } from "nanotar"; +import path from "path"; import semver from "semver"; +import tar from "tar-stream"; import tmp from "tmp"; import { BaseCommand } from "../baseCommand.js"; @@ -166,13 +167,7 @@ Update your application Dockerfile using one of the templates at https://github. outputFilePath: string, ): Promise { // create docker tarball from app image - const { stdout: appCid } = await execa("docker", [ - "image", - "save", - image, - "-o", - outputFilePath, - ]); + await execa("docker", ["image", "save", image, "-o", outputFilePath]); } // this wraps the call to the sdk image with a one-shot approach @@ -284,26 +279,13 @@ Update your application Dockerfile using one of the templates at https://github. private async createMachineSnapshotCommand( info: ImageInfo, ): Promise { - const { flags } = await this.parse(BuildApplication); - const driveType = flags["drive-type"] as DriveType; - const ramSize = info.ramSize; - const entrypoint = [ - "rollup-init", - "crun", - "run", - "--config", - "/run/cruntime/config/config.json", - "--bundle", - "/run/cruntime", - "app", - ].join(" "); + const entrypoint = ["rollup-init", "crun", "run", "app"].join(" "); const cwd = "--append-init=WORKDIR=/run/cruntime"; const flashDriveArgs: string[] = [ `--flash-drive=label:root,filename:/tmp/input0`, - `--flash-drive=label:config,filename:/tmp/input1,mount:/run/cruntime/config`, - `--flash-drive=label:dapp,filename:/tmp/input2,mount:/run/cruntime/rootfs`, + `--flash-drive=label:dapp,filename:/tmp/input1,mount:/run/cruntime`, ]; const result = [ @@ -314,25 +296,21 @@ Update your application Dockerfile using one of the templates at https://github. "--final-hash", "--store=/tmp/output", "--append-bootargs=no4lvl", + "--append-bootargs=rootfstype=squashfs", "--append-init=/bin/sh /etc/cartesi-init.d/cruntime-init", cwd, `--append-entrypoint=${entrypoint}`, ]; - if (driveType === "sqfs") - result.push("--append-bootargs=rootfstype=squashfs"); - return result; } + //TODO: embedd cruntime.sqfs into cartesi/sdk image + // move the packages/cruntime code to packages/sdk private async createCruntimeDrive( image: string, - imageInfo: ImageInfo, sdkImage: string, ): Promise { - const { flags } = await this.parse(BuildApplication); - const driveType = flags["drive-type"] as DriveType; - const cruntimeTarPath = this.getContextPath("cruntime.tar"); const cruntimeGnutarPath = this.getContextPath("cruntime.gnutar"); const cruntimeDrivePath = this.getContextPath("cruntime.drive"); @@ -349,10 +327,7 @@ Update your application Dockerfile using one of the templates at https://github. await this.sdkRun( sdkImage, - BuildApplication.createDriveCommand( - driveType, - bytes.parse(imageInfo.dataSize), - ), + BuildApplication.createDriveCommand("sqfs", 0), [cruntimeGnutarPath], cruntimeDrivePath, ); @@ -362,40 +337,7 @@ Update your application Dockerfile using one of the templates at https://github. } } - private async createOCIConfigeDrive( - imageInfo: ImageInfo, - sdkImage: string, - ): Promise { - const { flags } = await this.parse(BuildApplication); - const driveType = flags["drive-type"] as DriveType; - - const ociConfigTarPath = this.getContextPath("ociconfig.tar"); - const ociConfigDrivePath = this.getContextPath("ociconfig.drive"); - - try { - const configTar = createTar([ - { - name: "config.json", - data: JSON.stringify(createConfig(imageInfo)), - }, - ]); - fs.writeFileSync(ociConfigTarPath, configTar); - - await this.sdkRun( - sdkImage, - BuildApplication.createDriveCommand( - driveType, - bytes.parse(imageInfo.dataSize), - ), - [ociConfigTarPath], - ociConfigDrivePath, - ); - } finally { - await fs.remove(ociConfigTarPath); - } - } - - private async createAppDrive( + private async createAppOCIBundle( image: string, imageInfo: ImageInfo, sdkImage: string, @@ -405,6 +347,7 @@ Update your application Dockerfile using one of the templates at https://github. const appTarPath = this.getContextPath("app.tar"); const appGnutarPath = this.getContextPath("app.gnutar"); + const appOCIBundlePath = this.getContextPath("app.ocibundle.tar"); const appDrivePath = this.getContextPath("app.drive"); try { @@ -419,6 +362,38 @@ Update your application Dockerfile using one of the templates at https://github. appGnutarPath, ); + // prepare OCI Bundle + const extract = tar.extract(); + const pack = tar.pack(); + const appGnuTarStream = fs.createReadStream(appGnutarPath); + const appOCIBundleStream = fs.createWriteStream(appOCIBundlePath); + const ociConfigJSON = JSON.stringify(createConfig(imageInfo)); + + // add config.json + pack.entry({ name: "config.json" }, ociConfigJSON, function (err) { + if (err) throw err; + console.log("ERROR config.json"); + pack.finalize(); + }); + + // add rootfs/ prefix + extract.on("entry", function (header, stream, callback) { + header.name = path.join("rootfs", header.name); + stream.pipe(pack.entry(header, callback)); + }); + + // save tarball for OCI Bundle + appGnuTarStream.pipe(extract); + pack.pipe(appOCIBundleStream); + + // appOCIBundleStream.on("close", function () { + // console.log(path + " has been written"); + // }); + + // extract.on("finish", function () { + // pack.finalize(); + // }); + // create drive await this.sdkRun( sdkImage, @@ -426,22 +401,21 @@ Update your application Dockerfile using one of the templates at https://github. driveType, bytes.parse(imageInfo.dataSize), ), - [appGnutarPath], + [appOCIBundlePath], appDrivePath, ); } finally { - await fs.remove(appGnutarPath); - await fs.remove(appTarPath); + //await fs.remove(appTarPath); + //await fs.remove(appGnutarPath); + //await fs.remove(appOCIBundlePath); } } public async run(): Promise { const { flags } = await this.parse(BuildApplication); - const tarPath = this.getContextPath("image.tar"); const snapshotPath = this.getContextPath("image"); const cruntimeDrivePath = this.getContextPath("cruntime.drive"); - const ociConfigDrivePath = this.getContextPath("ociconfig.drive"); const appDrivePath = this.getContextPath("app.drive"); // clean up temp files we create along the process @@ -460,28 +434,18 @@ Update your application Dockerfile using one of the templates at https://github. const sdkImage = `cartesi/sdk:${imageInfo.sdkVersion}`; try { - // create docker tarball for image specified - await this.createTarball(appImage, tarPath); - // create cruntime drive - await this.createCruntimeDrive( - CARTESI_CRUNTIME_IMAGE, - imageInfo, - sdkImage, - ); - - // create oci config drive - await this.createOCIConfigeDrive(imageInfo, sdkImage); + await this.createCruntimeDrive(CARTESI_CRUNTIME_IMAGE, sdkImage); // create app drive - await this.createAppDrive(appImage, imageInfo, sdkImage); + await this.createAppOCIBundle(appImage, imageInfo, sdkImage); // create machine snapshot if (!flags["skip-snapshot"]) { await this.sdkRun( sdkImage, await this.createMachineSnapshotCommand(imageInfo), - [cruntimeDrivePath, ociConfigDrivePath, appDrivePath], + [cruntimeDrivePath, appDrivePath], snapshotPath, ); await fs.chmod(snapshotPath, 0o755); diff --git a/apps/cli/src/commands/shell.ts b/apps/cli/src/commands/shell.ts index ea5060f4..e8d6d931 100644 --- a/apps/cli/src/commands/shell.ts +++ b/apps/cli/src/commands/shell.ts @@ -57,7 +57,7 @@ export default class Shell extends BaseCommand { args.push("-it"); } - await execa("docker", [...args, "--", "/bin/bash"], { + await execa("docker", [...args, "--", "/bin/sh"], { stdio: "inherit", }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a78a662..bcf365e8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,6 +80,9 @@ importers: semver: specifier: ^7.5 version: 7.6.0 + tar-stream: + specifier: ^3.1.7 + version: 3.1.7 tmp: specifier: ^0.2.3 version: 0.2.3 @@ -120,6 +123,9 @@ importers: '@types/semver': specifier: ^7.5.8 version: 7.5.8 + '@types/tar-stream': + specifier: ^3.1.3 + version: 3.1.3 '@types/tmp': specifier: ^0.2 version: 0.2.6 @@ -3999,6 +4005,12 @@ packages: resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==} dev: true + /@types/tar-stream@3.1.3: + resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==} + dependencies: + '@types/node': 20.12.7 + dev: true + /@types/through@0.0.33: resolution: {integrity: sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==} dependencies: @@ -4891,9 +4903,19 @@ packages: dequal: 2.0.3 dev: true + /b4a@1.6.6: + resolution: {integrity: sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==} + dev: false + /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + /bare-events@2.2.2: + resolution: {integrity: sha512-h7z00dWdG0PYOQEvChhOSWvOfkIKsdZGkWr083FgN/HyoQuebSew/cgirYqh9SCuy/hRvxc5Vy6Fw8xAmYHLkQ==} + requiresBuild: true + dev: false + optional: true + /base-x@3.0.9: resolution: {integrity: sha512-H7JU6iBHTal1gp56aKoaa//YUxEaAOUiydvrV/pILqIHXTtqxSkATOnDA2u+jZ/61sD+L/412+7kzXRtWukhpQ==} dependencies: @@ -6959,6 +6981,10 @@ packages: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: true + /fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + dev: false + /fast-glob@3.3.2: resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} engines: {node: '>=8.6.0'} @@ -9575,6 +9601,10 @@ packages: /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + /queue-tick@1.0.1: + resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} + dev: false + /quick-lru@4.0.1: resolution: {integrity: sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==} engines: {node: '>=8'} @@ -10294,6 +10324,15 @@ packages: mixme: 0.5.9 dev: true + /streamx@2.16.1: + resolution: {integrity: sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==} + dependencies: + fast-fifo: 1.3.2 + queue-tick: 1.0.1 + optionalDependencies: + bare-events: 2.2.2 + dev: false + /string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -10477,6 +10516,14 @@ packages: engines: {node: '>=6'} dev: true + /tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + dependencies: + b4a: 1.6.6 + fast-fifo: 1.3.2 + streamx: 2.16.1 + dev: false + /tar@6.2.1: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} engines: {node: '>=10'}