diff --git a/pkg/bundle/create/create.go b/pkg/bundle/create/create.go index 5b3601351..8349bba85 100644 --- a/pkg/bundle/create/create.go +++ b/pkg/bundle/create/create.go @@ -232,7 +232,6 @@ func (o *Options) create(ctx context.Context, f createFunc) error { thisRun.Blobs = append(thisRun.Blobs, blobs...) // Add this run and metadata to top level metadata. meta.PastMirrors = append(meta.PastMirrors, thisRun) - meta.PastManifests = append(meta.PastManifests, manifests...) meta.PastBlobs = append(meta.PastBlobs, blobs...) // Update the metadata. @@ -309,7 +308,7 @@ func (o *Options) getFiles(meta v1alpha1.Metadata) ([]v1alpha1.Manifest, []v1alp defer os.Chdir(cwd) // Gather manifests we pulled - manifests, err := bundle.ReconcileManifests(meta, ".") + manifests, err := bundle.ReconcileManifests(".") if err != nil { return nil, nil, err diff --git a/pkg/bundle/files.go b/pkg/bundle/files.go index 66287008a..29cb4aeec 100644 --- a/pkg/bundle/files.go +++ b/pkg/bundle/files.go @@ -12,15 +12,7 @@ import ( // ReconcileManifest gather all manifests that were collected during a run // and checks against the current list -func ReconcileManifests(meta v1alpha1.Metadata, sourceDir string) (newManifest []v1alpha1.Manifest, err error) { - - foundFiles := make(map[string]struct{}, len(meta.PastManifests)) - for _, pf := range meta.PastManifests { - foundFiles[pf.Name] = struct{}{} - } - - // Ignore the current dir. - foundFiles["."] = struct{}{} +func ReconcileManifests(sourceDir string) (manifests []v1alpha1.Manifest, err error) { err = filepath.Walk("v2", func(fpath string, info os.FileInfo, err error) error { @@ -40,20 +32,12 @@ func ReconcileManifests(meta v1alpha1.Metadata, sourceDir string) (newManifest [ Name: fpath, } - if _, found := foundFiles[fpath]; !found { - - // Past files should only be image data, not tool metadata. - newManifest = append(newManifest, file) - foundFiles[fpath] = struct{}{} - - } else { - logrus.Debugf("Manifest %s exists in imageset, skipping...", fpath) - } + manifests = append(manifests, file) return nil }) - return newManifest, err + return manifests, err } // ReconcileBlobs gather all blobs that were collected during a run diff --git a/pkg/bundle/files_test.go b/pkg/bundle/files_test.go index c6486b6b7..c8fc3cf53 100644 --- a/pkg/bundle/files_test.go +++ b/pkg/bundle/files_test.go @@ -117,19 +117,18 @@ func Test_ReconcilingManifest(t *testing.T) { {Name: "v2"}, {Name: "v2/manifests"}, {Name: "v2/manifests/test1"}, + {Name: "v2/manifests/test2"}, }, }, want: []v1alpha1.Manifest{ + {Name: "v2"}, + {Name: "v2/manifests"}, + {Name: "v2/manifests/test1"}, {Name: "v2/manifests/test2"}, }, }, } for _, tt := range tests { - meta := v1alpha1.Metadata{ - MetadataSpec: v1alpha1.MetadataSpec{ - PastManifests: tt.fields.files, - }, - } tmpdir := t.TempDir() @@ -164,7 +163,7 @@ func Test_ReconcilingManifest(t *testing.T) { t.Fatal(err) } - actual, err := ReconcileManifests(meta, ".") + actual, err := ReconcileManifests(".") if err != nil { t.Fatal(err) diff --git a/pkg/bundle/publish/publish.go b/pkg/bundle/publish/publish.go index 1091da876..c6e95f275 100644 --- a/pkg/bundle/publish/publish.go +++ b/pkg/bundle/publish/publish.go @@ -7,7 +7,6 @@ import ( "io" "io/ioutil" "os" - "path" "path/filepath" "strings" "time" @@ -201,10 +200,14 @@ func (o *Options) Run(ctx context.Context, cmd *cobra.Command, f kcmdutil.Factor namedICSPMappings := map[string]map[reference.DockerImageReference]reference.DockerImageReference{} for _, imageName := range assocs.Keys() { + newImg, err := reference.Parse(imageName) + if err != nil { + return err + } + newImg.Registry = o.ToMirror + genericMappings := []imgmirror.Mapping{} releaseMapping := imgmirror.Mapping{} - // Map of remote layer digest to the set of paths they should be fetched to. - missingLayers := map[string][]string{} // Save original source and mirror destination mapping to generate ICSP for this image. imageRef, err := reference.Parse(imageName) @@ -229,6 +232,8 @@ func (o *Options) Run(ctx context.Context, cmd *cobra.Command, f kcmdutil.Factor for _, assoc := range values { + // Map of remote layer digest to the set of paths they should be fetched to. + missingLayers := map[string][]string{} manifestPath := filepath.Join("v2", assoc.Path, "manifests") // Ensure child manifests are all unpacked @@ -303,7 +308,7 @@ func (o *Options) Run(ctx context.Context, cmd *cobra.Command, f kcmdutil.Factor if len(missingLayers) != 0 { // Fetch all layers and mount them at the specified paths. - if err := o.fetchBlobs(ctx, incomingMeta, m, missingLayers); err != nil { + if err := o.fetchBlobs(ctx, incomingMeta, m, missingLayers, newImg); err != nil { return err } } @@ -340,6 +345,7 @@ func (o *Options) Run(ctx context.Context, cmd *cobra.Command, f kcmdutil.Factor errs = append(errs, err) } } + logrus.Infof("Pushed it %s", imageName) // If this is a release image mirror the full release if releaseMapping.Source.String() != "" { @@ -523,32 +529,7 @@ func copyBlobFile(src io.Reader, dstPath string) error { return nil } -func (o *Options) fetchBlobs(ctx context.Context, meta v1alpha1.Metadata, mapping imgmirror.Mapping, missingLayers map[string][]string) error { - catalogNamespaceNames := []string{} - dstRef := mapping.Destination.Ref - catalogNamespaceNames = append(catalogNamespaceNames, path.Join(dstRef.Namespace, dstRef.Name)) - - blobResources := map[string]string{} - for _, blob := range meta.PastBlobs { - resource := blob.NamespaceName - for _, nsName := range catalogNamespaceNames { - if nsName == resource { - // Blob is associated with the catalog image itself. - blobResources[blob.ID] = nsName - continue - } - suffix := strings.TrimPrefix(resource, nsName+"/") - if suffix == resource { - // Blob is not a child of the catalog image in nsName. - continue - } - // Blob may belong to multiple images. - if _, seenBlob := blobResources[blob.ID]; !seenBlob { - blobResources[blob.ID] = suffix - continue - } - } - } +func (o *Options) fetchBlobs(ctx context.Context, meta v1alpha1.Metadata, mapping imgmirror.Mapping, missingLayers map[string][]string, img reference.DockerImageReference) error { restctx, err := config.CreateContext(nil, false, o.SkipTLS) if err != nil { @@ -557,12 +538,7 @@ func (o *Options) fetchBlobs(ctx context.Context, meta v1alpha1.Metadata, mappin var errs []error for layerDigest, dstBlobPaths := range missingLayers { - resource, hasResource := blobResources[layerDigest] - if !hasResource { - errs = append(errs, fmt.Errorf("layer %s: no registry resource path found", layerDigest)) - continue - } - if err := o.fetchBlob(ctx, restctx, resource, layerDigest, dstBlobPaths); err != nil { + if err := o.fetchBlob(ctx, restctx, img, layerDigest, dstBlobPaths); err != nil { errs = append(errs, fmt.Errorf("layer %s: %v", layerDigest, err)) continue } @@ -573,13 +549,7 @@ func (o *Options) fetchBlobs(ctx context.Context, meta v1alpha1.Metadata, mappin // fetchBlob fetches a blob at //blobs/ // then copies it to each path in dstPaths. -func (o *Options) fetchBlob(ctx context.Context, restctx *registryclient.Context, resource, layerDigest string, dstPaths []string) error { - - refStr := path.Join(o.ToMirror, resource) - ref, err := reference.Parse(refStr) - if err != nil { - return fmt.Errorf("parse ref %s: %v", refStr, err) - } +func (o *Options) fetchBlob(ctx context.Context, restctx *registryclient.Context, ref reference.DockerImageReference, layerDigest string, dstPaths []string) error { logrus.Debugf("copying blob %s from %s", layerDigest, ref.Exact()) diff --git a/pkg/config/v1alpha1/metadata_types.go b/pkg/config/v1alpha1/metadata_types.go index 9fa770947..d52d09f88 100644 --- a/pkg/config/v1alpha1/metadata_types.go +++ b/pkg/config/v1alpha1/metadata_types.go @@ -29,8 +29,7 @@ type MetadataSpec struct { PastMirrors PastMirrors `json:"pastMirrors"` // PastFiles is a slice containing information for // all files created for an imageset - PastBlobs []Blob `json:"pastBlobs"` - PastManifests []Manifest `json:"pastManifests"` + PastBlobs []Blob `json:"pastBlobs"` } type PastMirror struct { @@ -60,9 +59,10 @@ type Blob struct { } type Manifest struct { - Name string `json:"name"` - Image string `json:"image"` - Namespace string `json:"namespace"` + Name string `json:"name"` + Image string `json:"image"` + // NamespaceName of image that owns this manifest. + NamespaceName string `json:"namespaceName"` } // OperatorMetadata holds an Operator's post-mirror metadata.