diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index 78eb7885..080a7d07 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -2,10 +2,18 @@ package oci import ( "context" + "os" + "path" + "runtime" + "github.com/klauspost/pgzip" + "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/umoci" "github.com/opencontainers/umoci/oci/casext" + "github.com/opencontainers/umoci/oci/layer" "github.com/pkg/errors" + "stackerbuild.io/stacker/pkg/squashfs" ) func LookupManifest(oci casext.Engine, tag string) (ispec.Manifest, error) { @@ -76,3 +84,82 @@ func UpdateImageConfig(oci casext.Engine, name string, newConfig ispec.Image, ne return desc, nil } + +func unpackOne(ociDir string, bundlePath string, digest digest.Digest, isSquashfs bool) error { + if isSquashfs { + return squashfs.ExtractSingleSquash( + path.Join(ociDir, "blobs", "sha256", digest.Encoded()), + bundlePath, "overlay") + } + + oci, err := umoci.OpenLayout(ociDir) + if err != nil { + return err + } + defer oci.Close() + + compressed, err := oci.GetBlob(context.Background(), digest) + if err != nil { + return err + } + defer compressed.Close() + + uncompressed, err := pgzip.NewReader(compressed) + if err != nil { + return err + } + + return layer.UnpackLayer(bundlePath, uncompressed, nil) +} + +// Unpack an image with "tag" from "ociLayout" into paths returned by "pathfunc" +func Unpack(ociLayout, tag string, pathfunc func(digest.Digest) string) (int, error) { + oci, err := umoci.OpenLayout(ociLayout) + if err != nil { + return -1, err + } + defer oci.Close() + + manifest, err := LookupManifest(oci, tag) + if err != nil { + return -1, err + } + + pool := NewThreadPool(runtime.NumCPU()) + + for _, layer := range manifest.Layers { + digest := layer.Digest + contents := pathfunc(digest) + if squashfs.IsSquashfsMediaType(layer.MediaType) { + // don't really need to do this in parallel, but what + // the hell. + pool.Add(func(ctx context.Context) error { + return unpackOne(ociLayout, contents, digest, true) + }) + } else { + switch layer.MediaType { + case ispec.MediaTypeImageLayer: + fallthrough + case ispec.MediaTypeImageLayerGzip: + // don't extract things that have already been + // extracted + if _, err := os.Stat(contents); err == nil { + continue + } + + // TODO: when the umoci API grows support for uid + // shifting, we can use the fancier features of context + // cancelling in the thread pool... + pool.Add(func(ctx context.Context) error { + return unpackOne(ociLayout, contents, digest, false) + }) + default: + return -1, errors.Errorf("unknown media type %s", layer.MediaType) + } + } + } + + pool.DoneAddingJobs() + + return len(manifest.Layers), pool.Run() +} diff --git a/pkg/overlay/pool.go b/pkg/oci/pool.go similarity index 98% rename from pkg/overlay/pool.go rename to pkg/oci/pool.go index 24d9ff3d..0b67f231 100644 --- a/pkg/overlay/pool.go +++ b/pkg/oci/pool.go @@ -1,4 +1,4 @@ -package overlay +package oci import ( "context" diff --git a/pkg/overlay/pack.go b/pkg/overlay/pack.go index 2653b132..647e1d7f 100644 --- a/pkg/overlay/pack.go +++ b/pkg/overlay/pack.go @@ -8,11 +8,9 @@ import ( "os" "path" "path/filepath" - "runtime" "strings" "time" - "github.com/klauspost/pgzip" "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/umoci" @@ -44,57 +42,21 @@ func overlayPath(rootfs string, d digest.Digest, subdirs ...string) string { func (o *overlay) Unpack(tag, name string) error { cacheDir := path.Join(o.config.StackerDir, "layer-bases", "oci") - oci, err := umoci.OpenLayout(cacheDir) - if err != nil { - return err + + pathfunc := func(digest digest.Digest) string { + return overlayPath(o.config.RootFSDir, digest, "overlay") } - defer oci.Close() - manifest, err := stackeroci.LookupManifest(oci, tag) + _, err := stackeroci.Unpack(cacheDir, tag, pathfunc) if err != nil { return err } - pool := NewThreadPool(runtime.NumCPU()) - - for _, layer := range manifest.Layers { - digest := layer.Digest - contents := overlayPath(o.config.RootFSDir, digest, "overlay") - if squashfs.IsSquashfsMediaType(layer.MediaType) { - // don't really need to do this in parallel, but what - // the hell. - pool.Add(func(ctx context.Context) error { - return unpackOne(cacheDir, contents, digest, true) - }) - } else { - switch layer.MediaType { - case ispec.MediaTypeImageLayer: - fallthrough - case ispec.MediaTypeImageLayerGzip: - // don't extract things that have already been - // extracted - if _, err := os.Stat(contents); err == nil { - continue - } - - // TODO: when the umoci API grows support for uid - // shifting, we can use the fancier features of context - // cancelling in the thread pool... - pool.Add(func(ctx context.Context) error { - return unpackOne(cacheDir, contents, digest, false) - }) - default: - return errors.Errorf("unknown media type %s", layer.MediaType) - } - } - } - - pool.DoneAddingJobs() - - err = pool.Run() + oci, err := umoci.OpenLayout(cacheDir) if err != nil { return err } + defer oci.Close() err = o.Create(name) if err != nil { @@ -654,30 +616,3 @@ func repackOverlay(config types.StackerConfig, name string, layerTypes []types.L return ovl.write(config, name) } - -func unpackOne(ociDir string, bundlePath string, digest digest.Digest, isSquashfs bool) error { - if isSquashfs { - return squashfs.ExtractSingleSquash( - path.Join(ociDir, "blobs", "sha256", digest.Encoded()), - bundlePath, "overlay") - } - - oci, err := umoci.OpenLayout(ociDir) - if err != nil { - return err - } - defer oci.Close() - - compressed, err := oci.GetBlob(context.Background(), digest) - if err != nil { - return err - } - defer compressed.Close() - - uncompressed, err := pgzip.NewReader(compressed) - if err != nil { - return err - } - - return layer.UnpackLayer(bundlePath, uncompressed, nil) -} diff --git a/pkg/stacker/import.go b/pkg/stacker/import.go index 4bfe2b0a..4dffb504 100644 --- a/pkg/stacker/import.go +++ b/pkg/stacker/import.go @@ -13,6 +13,7 @@ import ( "github.com/vbatts/go-mtree" "stackerbuild.io/stacker/pkg/lib" "stackerbuild.io/stacker/pkg/log" + "stackerbuild.io/stacker/pkg/oci" "stackerbuild.io/stacker/pkg/types" ) @@ -281,22 +282,28 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st return "", err } - dname, err := os.MkdirTemp(c.RootFSDir, "import-rootfs-*") + tag, err := is.ParseTag() if err != nil { return "", err } - bsopt := BaseLayerOpts{ - Config: types.StackerConfig{RootFSDir: c.RootFSDir}, - Name: path.Base(dname), - Layer: types.Layer{From: is}, - Storage: storage, + pathfunc := func(digest digest.Digest) string { + _ = os.Remove(cache) + return cache } - if err := setupContainersImageRootfs(bsopt); err != nil { + + ociDir := path.Join(c.StackerDir, "layer-bases", "oci") + + n, err := oci.Unpack(ociDir, tag, pathfunc) + if err != nil { return "", err } - return dname, nil + if n > 1 { + return "", errors.Errorf("Currently supporting single-layer container image imports") + } + + return cache, nil } return "", errors.Errorf("unsupported url scheme %s", i) diff --git a/test/import.bats b/test/import.bats index 5b73cc32..dfbc49c7 100644 --- a/test/import.bats +++ b/test/import.bats @@ -403,6 +403,8 @@ cimg-import: import: - path: docker://alpine:edge dest: / + run: | + [ -d /var/lib/apk ] EOF stacker build }