From d7d3aac8155882246668826425df7b840d71a0e9 Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Thu, 12 Dec 2024 15:04:20 +0100 Subject: [PATCH] feat(cmd/rofl): Add TDX container build support --- cmd/rofl/build/artifacts.go | 9 +- cmd/rofl/build/container.go | 114 +++++++++++++ cmd/rofl/build/tdx.go | 307 ++++++++++++++++++++++++------------ 3 files changed, 328 insertions(+), 102 deletions(-) create mode 100644 cmd/rofl/build/container.go diff --git a/cmd/rofl/build/artifacts.go b/cmd/rofl/build/artifacts.go index 54228f2d..b5b3ddd5 100644 --- a/cmd/rofl/build/artifacts.go +++ b/cmd/rofl/build/artifacts.go @@ -38,8 +38,10 @@ func maybeDownloadArtifact(kind, uri, knownHash string) string { cobra.CheckErr(fmt.Errorf("failed to parse %s artifact URL: %w", kind, err)) } - // In case the URI represents a local file, just return it. + // In case the URI represents a local file, check that it exists and return it. if url.Host == "" { + _, err = os.Stat(url.Path) + cobra.CheckErr(err) return url.Path } @@ -192,6 +194,11 @@ FILES: // copyFile copies the file at path src to a file at path dst using the given mode. func copyFile(src, dst string, mode os.FileMode) error { + err := os.MkdirAll(filepath.Dir(dst), 0o755) + if err != nil { + return fmt.Errorf("failed to create destination directory for '%s': %w", dst, err) + } + sf, err := os.Open(src) if err != nil { return fmt.Errorf("failed to open '%s': %w", src, err) diff --git a/cmd/rofl/build/container.go b/cmd/rofl/build/container.go new file mode 100644 index 00000000..c12f7877 --- /dev/null +++ b/cmd/rofl/build/container.go @@ -0,0 +1,114 @@ +package build + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + + "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle" + + "github.com/oasisprotocol/cli/cmd/common" + cliConfig "github.com/oasisprotocol/cli/config" +) + +const ( + artifactContainerRuntime = "rofl-container runtime" + artifactContainerCompose = "docker-compose.yaml" + + defaultContainerStage2TemplateURI = "https://github.com/oasisprotocol/oasis-boot/releases/download/v0.3.0/stage2-podman.tar.bz2" + + defaultContainerRuntimeURI = "https://github.com/oasisprotocol/oasis-sdk/releases/download/rofl-containers/v0.1.0/runtime" +) + +var ( + tdxContainerRuntimeURI string + tdxContainerRuntimeHash string + tdxContainerComposeURI string + tdxContainerComposeHash string + + tdxContainerCmd = &cobra.Command{ + Use: "container", + Short: "Build a container-based TDX ROFL application", + Args: cobra.NoArgs, + Run: func(_ *cobra.Command, _ []string) { + cfg := cliConfig.Global() + npa := common.GetNPASelection(cfg) + + if npa.ParaTime == nil { + cobra.CheckErr("no ParaTime selected") + } + + wantedArtifacts := tdxGetDefaultArtifacts() + wantedArtifacts = append(wantedArtifacts, + &artifact{ + kind: artifactContainerRuntime, + uri: tdxContainerRuntimeURI, + knownHash: tdxContainerRuntimeHash, + }, + &artifact{ + kind: artifactContainerCompose, + uri: tdxContainerComposeURI, + knownHash: tdxContainerComposeHash, + }, + ) + artifacts := tdxFetchArtifacts(wantedArtifacts) + + fmt.Println("Building a container-based TDX ROFL application...") + + detectBuildMode(npa) + + // Start creating the bundle early so we can fail before building anything. + bnd := &bundle.Bundle{ + Manifest: &bundle.Manifest{ + Name: "my-container-app", + ID: npa.ParaTime.Namespace(), + }, + } + var err error + bnd.Manifest.Version, err = version.FromString("0.0.0") + if err != nil { + cobra.CheckErr(fmt.Errorf("unsupported package version format: %w", err)) + } + + fmt.Printf("Name: %s\n", bnd.Manifest.Name) + fmt.Printf("Version: %s\n", bnd.Manifest.Version) + + // Use the pre-built container runtime. + initPath := artifacts[artifactContainerRuntime] + + stage2, err := tdxPrepareStage2(artifacts, initPath, map[string]string{ + artifacts[artifactContainerCompose]: "etc/oasis/containers/docker-compose.yaml", + }) + if err != nil { + cobra.CheckErr(err) + } + defer os.RemoveAll(stage2.tmpDir) + + fmt.Println("Creating ORC bundle...") + + // TODO: Get consensus trust root and add it as ROFL_CONSENSUS_TRUST_ROOT to cmdline. + // TODO: Get ROFL app id and add it as ROFL_APP_ID to cmdline. + + outFn, err := tdxBundleComponent(artifacts, bnd, stage2) + if err != nil { + _ = os.RemoveAll(stage2.tmpDir) + cobra.CheckErr(err) + } + + fmt.Printf("ROFL app built and bundle written to '%s'.\n", outFn) + }, + } +) + +func init() { + tdxContainerFlags := flag.NewFlagSet("", flag.ContinueOnError) + tdxContainerFlags.StringVar(&tdxContainerRuntimeURI, "runtime", defaultContainerRuntimeURI, "URL or path to runtime binary") + tdxContainerFlags.StringVar(&tdxContainerRuntimeHash, "runtime-hash", "", "optional SHA256 hash of runtime binary") + tdxContainerFlags.StringVar(&tdxContainerComposeURI, "compose", "docker-compose.yaml", "URL or path to docker-compose.yaml") + tdxContainerFlags.StringVar(&tdxContainerComposeHash, "compose-hash", "", "optional SHA256 hash of docker-compose.yaml") + + tdxContainerCmd.Flags().AddFlagSet(tdxContainerFlags) +} diff --git a/cmd/rofl/build/tdx.go b/cmd/rofl/build/tdx.go index 1c2aaf43..13917caf 100644 --- a/cmd/rofl/build/tdx.go +++ b/cmd/rofl/build/tdx.go @@ -35,6 +35,11 @@ var knownHashes = map[string]string{ defaultStage2TemplateURI: "8cbc67e4a05b01e6fc257a3ef378db50ec230bc4c7aacbfb9abf0f5b17dcb8fd", } +var ( + tdxFlags *flag.FlagSet + tdxStorageFlags *flag.FlagSet +) + var ( tdxFirmwareURI string tdxFirmwareHash string @@ -46,6 +51,9 @@ var ( tdxResourcesMemory uint64 tdxResourcesCPUCount uint8 + tdxTmpStorageMode string + tdxTmpStorageSize uint64 + tdxCmd = &cobra.Command{ Use: "tdx", Short: "Build a TDX-based ROFL application", @@ -58,24 +66,7 @@ var ( cobra.CheckErr("no ParaTime selected") } - // Obtain required artifacts. - artifacts := make(map[string]string) - for _, ar := range []struct { - kind string - uri string - knownHash string - }{ - {artifactFirmware, tdxFirmwareURI, tdxFirmwareHash}, - {artifactKernel, tdxKernelURI, tdxKernelHash}, - {artifactStage2, tdxStage2TemplateURI, tdxStage2TemplateHash}, - } { - // Automatically populate known hashes for known URIs. - if ar.knownHash == "" { - ar.knownHash = knownHashes[ar.uri] - } - - artifacts[ar.kind] = maybeDownloadArtifact(ar.kind, ar.uri, ar.knownHash) - } + artifacts := tdxFetchArtifacts(tdxGetDefaultArtifacts()) fmt.Println("Building a TDX-based Rust ROFL application...") @@ -120,105 +111,203 @@ var ( cobra.CheckErr(fmt.Errorf("failed to build runtime binary: %w", err)) } - // Create temporary directory and unpack stage 2 template into it. - fmt.Println("Preparing stage 2 root filesystem...") - tmpDir, err := os.MkdirTemp("", "oasis-build-stage2") + stage2, err := tdxPrepareStage2(artifacts, initPath, nil) if err != nil { - cobra.CheckErr(fmt.Errorf("failed to create temporary stage 2 build directory: %w", err)) + cobra.CheckErr(err) } - defer os.RemoveAll(tmpDir) // TODO: This doesn't work because of cobra.CheckErr + defer os.RemoveAll(stage2.tmpDir) - rootfsDir := filepath.Join(tmpDir, "rootfs") - if err = os.Mkdir(rootfsDir, 0o755); err != nil { - cobra.CheckErr(fmt.Errorf("failed to create temporary rootfs directory: %w", err)) - } + fmt.Println("Creating ORC bundle...") - // Unpack template into temporary directory. - fmt.Println("Unpacking template...") - if err = extractArchive(artifacts[artifactStage2], rootfsDir); err != nil { - cobra.CheckErr(fmt.Errorf("failed to extract stage 2 template: %w", err)) + outFn, err := tdxBundleComponent(artifacts, bnd, stage2) + if err != nil { + _ = os.RemoveAll(stage2.tmpDir) + cobra.CheckErr(err) } - // Add runtime as init. - fmt.Println("Adding runtime as init...") - err = copyFile(initPath, filepath.Join(rootfsDir, "init"), 0o755) - cobra.CheckErr(err) + fmt.Printf("ROFL app built and bundle written to '%s'.\n", outFn) + }, + } +) - // Create an ext4 filesystem. - fmt.Println("Creating ext4 filesystem...") - rootfsImage := filepath.Join(tmpDir, "rootfs.ext4") - rootfsSize, err := createExt4Fs(rootfsImage, rootfsDir) - if err != nil { - cobra.CheckErr(fmt.Errorf("failed to create rootfs image: %w", err)) - } +type artifact struct { + kind string + uri string + knownHash string +} - // Create dm-verity hash tree. - fmt.Println("Creating dm-verity hash tree...") - hashFile := filepath.Join(tmpDir, "rootfs.hash") - rootHash, err := createVerityHashTree(rootfsImage, hashFile) - if err != nil { - cobra.CheckErr(fmt.Errorf("failed to create verity hash tree: %w", err)) - } +// tdxGetDefaultArtifacts returns the list of default TDX artifacts. +func tdxGetDefaultArtifacts() []*artifact { + return []*artifact{ + {artifactFirmware, tdxFirmwareURI, tdxFirmwareHash}, + {artifactKernel, tdxKernelURI, tdxKernelHash}, + {artifactStage2, tdxStage2TemplateURI, tdxStage2TemplateHash}, + } +} - // Concatenate filesystem and hash tree into one image. - if err = concatFiles(rootfsImage, hashFile); err != nil { - cobra.CheckErr(fmt.Errorf("failed to concatenate rootfs and hash tree files: %w", err)) - } +// tdxFetchArtifacts obtains all of the required artifacts for a TDX image. +func tdxFetchArtifacts(artifacts []*artifact) map[string]string { + result := make(map[string]string) + for _, ar := range artifacts { + // Automatically populate known hashes for known URIs. + if ar.knownHash == "" { + ar.knownHash = knownHashes[ar.uri] + } - fmt.Println("Creating ORC bundle...") + result[ar.kind] = maybeDownloadArtifact(ar.kind, ar.uri, ar.knownHash) + } + return result +} - // Add the ROFL component. - firmwareName := "firmware.fd" - kernelName := "kernel.bin" - stage2Name := "stage2.img" - - comp := bundle.Component{ - Kind: component.ROFL, - Name: pkgMeta.Name, - TDX: &bundle.TDXMetadata{ - Firmware: firmwareName, - Kernel: kernelName, - Stage2Image: stage2Name, - ExtraKernelOptions: []string{ - "console=ttyS0", - fmt.Sprintf("oasis.stage2.roothash=%s", rootHash), - fmt.Sprintf("oasis.stage2.hash_offset=%d", rootfsSize), - }, - Resources: bundle.TDXResources{ - Memory: tdxResourcesMemory, - CPUCount: tdxResourcesCPUCount, - }, - }, - } - bnd.Manifest.Components = append(bnd.Manifest.Components, &comp) +type tdxStage2 struct { + tmpDir string + fn string + rootHash string + fsSize int64 +} - if err = bnd.Manifest.Validate(); err != nil { - cobra.CheckErr(fmt.Errorf("failed to validate manifest: %w", err)) - } +// tdxPrepareStage2 prepares the stage 2 rootfs. +func tdxPrepareStage2(artifacts map[string]string, initPath string, extraFiles map[string]string) (*tdxStage2, error) { + var ok bool - // Add all files. - fileMap := map[string]string{ - firmwareName: artifacts[artifactFirmware], - kernelName: artifacts[artifactKernel], - stage2Name: rootfsImage, - } - for dst, src := range fileMap { - _ = bnd.Add(dst, bundle.NewFileData(src)) - } + // Create temporary directory and unpack stage 2 template into it. + fmt.Println("Preparing stage 2 root filesystem...") + tmpDir, err := os.MkdirTemp("", "oasis-build-stage2") + if err != nil { + return nil, fmt.Errorf("failed to create temporary stage 2 build directory: %w", err) + } + defer func() { + // Ensure temporary directory is removed on errors. + if !ok { + _ = os.RemoveAll(tmpDir) + } + }() - // Write the bundle out. - outFn := fmt.Sprintf("%s.orc", bnd.Manifest.Name) - if outputFn != "" { - outFn = outputFn - } - if err = bnd.Write(outFn); err != nil { - cobra.CheckErr(fmt.Errorf("failed to write output bundle: %w", err)) - } + rootfsDir := filepath.Join(tmpDir, "rootfs") + if err = os.Mkdir(rootfsDir, 0o755); err != nil { + return nil, fmt.Errorf("failed to create temporary rootfs directory: %w", err) + } - fmt.Printf("ROFL app built and bundle written to '%s'.\n", outFn) + // Unpack template into temporary directory. + fmt.Println("Unpacking template...") + if err = extractArchive(artifacts[artifactStage2], rootfsDir); err != nil { + return nil, fmt.Errorf("failed to extract stage 2 template: %w", err) + } + + // Add runtime as init. + fmt.Println("Adding runtime as init...") + if err = copyFile(initPath, filepath.Join(rootfsDir, "init"), 0o755); err != nil { + return nil, err + } + + // Copy any extra files. + fmt.Println("Adding extra files...") + for src, dst := range extraFiles { + if err = copyFile(src, filepath.Join(rootfsDir, dst), 0o644); err != nil { + return nil, err + } + } + + // Create an ext4 filesystem. + fmt.Println("Creating ext4 filesystem...") + rootfsImage := filepath.Join(tmpDir, "rootfs.ext4") + rootfsSize, err := createExt4Fs(rootfsImage, rootfsDir) + if err != nil { + return nil, fmt.Errorf("failed to create rootfs image: %w", err) + } + + // Create dm-verity hash tree. + fmt.Println("Creating dm-verity hash tree...") + hashFile := filepath.Join(tmpDir, "rootfs.hash") + rootHash, err := createVerityHashTree(rootfsImage, hashFile) + if err != nil { + return nil, fmt.Errorf("failed to create verity hash tree: %w", err) + } + + // Concatenate filesystem and hash tree into one image. + if err = concatFiles(rootfsImage, hashFile); err != nil { + return nil, fmt.Errorf("failed to concatenate rootfs and hash tree files: %w", err) + } + + ok = true + + return &tdxStage2{ + tmpDir: tmpDir, + fn: rootfsImage, + rootHash: rootHash, + fsSize: rootfsSize, + }, nil +} + +// tdxBundleComponent adds the ROFL component to the given bundle. +func tdxBundleComponent(artifacts map[string]string, bnd *bundle.Bundle, stage2 *tdxStage2) (string, error) { + // Add the ROFL component. + firmwareName := "firmware.fd" + kernelName := "kernel.bin" + stage2Name := "stage2.img" + + comp := bundle.Component{ + Kind: component.ROFL, + Name: bnd.Manifest.Name, + TDX: &bundle.TDXMetadata{ + Firmware: firmwareName, + Kernel: kernelName, + Stage2Image: stage2Name, + ExtraKernelOptions: []string{ + "console=ttyS0", + fmt.Sprintf("oasis.stage2.roothash=%s", stage2.rootHash), + fmt.Sprintf("oasis.stage2.hash_offset=%d", stage2.fsSize), + }, + Resources: bundle.TDXResources{ + Memory: tdxResourcesMemory, + CPUCount: tdxResourcesCPUCount, + }, }, } -) + + switch tdxTmpStorageMode { + case "none": + case "ram": + comp.TDX.ExtraKernelOptions = append(comp.TDX.ExtraKernelOptions, + fmt.Sprintf("oasis.stage2.storage_mode=ram"), + fmt.Sprintf("oasis.stage2.storage_size=%d", tdxTmpStorageSize*1024*1024), + ) + case "disk": + comp.TDX.ExtraKernelOptions = append(comp.TDX.ExtraKernelOptions, + fmt.Sprintf("oasis.stage2.storage_mode=disk"), + fmt.Sprintf("oasis.stage2.storage_size=%d", tdxTmpStorageSize*1024*1024), + fmt.Sprintf("oasis.stage2.storage_offset=%d", 0), // TODO + ) + default: + return "", fmt.Errorf("unsupported ephemeral storage mode: %s", tdxTmpStorageMode) + } + + bnd.Manifest.Components = append(bnd.Manifest.Components, &comp) + + if err := bnd.Manifest.Validate(); err != nil { + return "", fmt.Errorf("failed to validate manifest: %w", err) + } + + // Add all files. + fileMap := map[string]string{ + firmwareName: artifacts[artifactFirmware], + kernelName: artifacts[artifactKernel], + stage2Name: stage2.fn, + } + for dst, src := range fileMap { + _ = bnd.Add(dst, bundle.NewFileData(src)) + } + + // Write the bundle out. + outFn := fmt.Sprintf("%s.orc", bnd.Manifest.Name) + if outputFn != "" { + outFn = outputFn + } + if err := bnd.Write(outFn); err != nil { + return "", fmt.Errorf("failed to write output bundle: %w", err) + } + return outFn, nil +} // tdxSetupBuildEnv sets up the TDX build environment. func tdxSetupBuildEnv() { @@ -248,7 +337,7 @@ func tdxSetupBuildEnv() { } func init() { - tdxFlags := flag.NewFlagSet("", flag.ContinueOnError) + tdxFlags = flag.NewFlagSet("", flag.ContinueOnError) tdxFlags.StringVar(&tdxFirmwareURI, "firmware", defaultFirmwareURI, "URL or path to firmware image") tdxFlags.StringVar(&tdxFirmwareHash, "firmware-hash", "", "optional SHA256 hash of firmware image") tdxFlags.StringVar(&tdxKernelURI, "kernel", defaultKernelURI, "URL or path to kernel image") @@ -259,6 +348,22 @@ func init() { tdxFlags.Uint64Var(&tdxResourcesMemory, "memory", 512, "required amount of VM memory in megabytes") tdxFlags.Uint8Var(&tdxResourcesCPUCount, "cpus", 1, "required number of vCPUs") + tdxFlags.StringVar(&tdxTmpStorageMode, "ephemeral-storage-mode", "none", "ephemeral storage mode") + tdxFlags.Uint64Var(&tdxTmpStorageSize, "ephemeral-storage-size", 64, "ephemeral storage size in megabytes") + tdxCmd.Flags().AddFlagSet(common.SelectorNPFlags) tdxCmd.Flags().AddFlagSet(tdxFlags) + + // XXX: We need to define the flags here due to init order (container gets called before tdx). + tdxContainerCmd.Flags().AddFlagSet(common.SelectorNPFlags) + tdxContainerCmd.Flags().AddFlagSet(tdxFlags) + + // Override some flag defaults. + flags := tdxContainerCmd.Flags() + flags.Lookup("template").DefValue = defaultContainerStage2TemplateURI + flags.Lookup("template").Value.Set(defaultContainerStage2TemplateURI) + flags.Lookup("ephemeral-storage-mode").DefValue = "ram" + flags.Lookup("ephemeral-storage-mode").Value.Set("ram") + + tdxCmd.AddCommand(tdxContainerCmd) }