From 39e0fede405cb486e4701801ba67f2e5a9d21a2d Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Fri, 20 Oct 2023 18:17:22 -0400 Subject: [PATCH] pull carbide flavored hauler manifests from reg Signed-off-by: Adam Martin --- cmd/hauler/cli/store/info.go | 2 +- cmd/hauler/cli/store/sync.go | 256 ++++++++++++++++++++--------------- pkg/consts/consts.go | 4 + 3 files changed, 153 insertions(+), 109 deletions(-) diff --git a/cmd/hauler/cli/store/info.go b/cmd/hauler/cli/store/info.go index f4de62df..8f5b47cc 100644 --- a/cmd/hauler/cli/store/info.go +++ b/cmd/hauler/cli/store/info.go @@ -116,7 +116,7 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item ctype = "image" case consts.ChartConfigMediaType: ctype = "chart" - case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType: + case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType, consts.OCIArtifact: ctype = "file" default: ctype = "unknown" diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 7f90b345..c75b6a1e 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "strings" "github.com/spf13/cobra" "helm.sh/helm/v3/pkg/action" @@ -13,11 +14,11 @@ import ( "github.com/mitchellh/go-homedir" "github.com/rancherfederal/hauler/pkg/store" - "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" tchart "github.com/rancherfederal/hauler/pkg/collection/chart" "github.com/rancherfederal/hauler/pkg/collection/imagetxt" "github.com/rancherfederal/hauler/pkg/collection/k3s" + "github.com/rancherfederal/hauler/pkg/consts" "github.com/rancherfederal/hauler/pkg/content" "github.com/rancherfederal/hauler/pkg/cosign" "github.com/rancherfederal/hauler/pkg/log" @@ -27,6 +28,7 @@ type SyncOpts struct { *RootOpts ContentFiles []string Key string + Products []string } func (o *SyncOpts) AddFlags(cmd *cobra.Command) { @@ -34,6 +36,7 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files") f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for image signature verification") + f.StringSliceVar(&o.Products, "products", []string{}, "Used for RGS Carbide customers to supply a product and version and Hauler will retrieve the images. i.e. '--product rancher=v2.7.8'") } func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { @@ -45,157 +48,194 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { return err } + for _, product := range o.Products { + l.Infof("processing content file for product: '%s'", product) + parts := strings.Split(product, "=") + manifestLoc := fmt.Sprintf("%s/hauler/%s-manifest.yaml:%s", consts.CarbideRegistry, parts[0], parts[1]) + l.Infof("retrieving product manifest from: '%s'", manifestLoc) + img := v1alpha1.Image{ + Name: manifestLoc, + } + err := storeImage(ctx, s, img) + if err != nil { + return err + } + err = ExtractCmd(ctx, &ExtractOpts{RootOpts: o.RootOpts}, s, "hauler/rke2-manifest.yaml:v2.7.8") + if err != nil { + return err + } + filename := fmt.Sprintf("%s-manifest.yaml", parts[0]) + + fi, err := os.Open(filename) + if err != nil { + return err + } + err = processContent(ctx, fi, o, s) + if err != nil { + return err + } + } + for _, filename := range o.ContentFiles { l.Debugf("processing content file: '%s'", filename) fi, err := os.Open(filename) if err != nil { return err } + err = processContent(ctx, fi, o, s) + if err != nil { + return err + } + } + + return nil +} - reader := yaml.NewYAMLReader(bufio.NewReader(fi)) +func processContent(ctx context.Context, fi *os.File, o *SyncOpts, s *store.Layout) error { + l := log.FromContext(ctx) - var docs [][]byte - for { - raw, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return err - } + reader := yaml.NewYAMLReader(bufio.NewReader(fi)) - docs = append(docs, raw) + var docs [][]byte + for { + raw, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return err } - for _, doc := range docs { - obj, err := content.Load(doc) - if err != nil { - l.Debugf("skipping sync of unknown content") - continue - } + docs = append(docs, raw) + } - l.Infof("syncing [%s] to store", obj.GroupVersionKind().String()) + for _, doc := range docs { + obj, err := content.Load(doc) + if err != nil { + l.Debugf("skipping sync of unknown content") + continue + } - // TODO: Should type switch instead... - switch obj.GroupVersionKind().Kind { - case v1alpha1.FilesContentKind: - var cfg v1alpha1.Files - if err := yaml.Unmarshal(doc, &cfg); err != nil { - return err - } + l.Infof("syncing [%s] to store", obj.GroupVersionKind().String()) - for _, f := range cfg.Spec.Files { - err := storeFile(ctx, s, f) - if err != nil { - return err - } - } + // TODO: Should type switch instead... + switch obj.GroupVersionKind().Kind { + case v1alpha1.FilesContentKind: + var cfg v1alpha1.Files + if err := yaml.Unmarshal(doc, &cfg); err != nil { + return err + } - case v1alpha1.ImagesContentKind: - var cfg v1alpha1.Images - if err := yaml.Unmarshal(doc, &cfg); err != nil { + for _, f := range cfg.Spec.Files { + err := storeFile(ctx, s, f) + if err != nil { return err } + } + + case v1alpha1.ImagesContentKind: + var cfg v1alpha1.Images + if err := yaml.Unmarshal(doc, &cfg); err != nil { + return err + } + + for _, i := range cfg.Spec.Images { - for _, i := range cfg.Spec.Images { - - // Check if the user provided a key. - if o.Key != "" || i.Key != "" { - key := o.Key - if i.Key != "" { - key, err = homedir.Expand(i.Key) - } - l.Debugf("key for image [%s]", key) - - // verify signature using the provided key. - err := cosign.VerifySignature(ctx, s, key, i.Name) - if err != nil { - l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err) - continue - } - l.Infof("signature verified for image [%s]", i.Name) + // Check if the user provided a key. + if o.Key != "" || i.Key != "" { + key := o.Key + if i.Key != "" { + key, err = homedir.Expand(i.Key) } + l.Debugf("key for image [%s]", key) - err = storeImage(ctx, s, i) + // verify signature using the provided key. + err := cosign.VerifySignature(ctx, s, key, i.Name) if err != nil { - return err + l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err) + continue } + l.Infof("signature verified for image [%s]", i.Name) } - - case v1alpha1.ChartsContentKind: - var cfg v1alpha1.Charts - if err := yaml.Unmarshal(doc, &cfg); err != nil { + + err = storeImage(ctx, s, i) + if err != nil { return err } + } - for _, ch := range cfg.Spec.Charts { - // TODO: Provide a way to configure syncs - err := storeChart(ctx, s, ch, &action.ChartPathOptions{}) - if err != nil { - return err - } - } - - case v1alpha1.K3sCollectionKind: - var cfg v1alpha1.K3s - if err := yaml.Unmarshal(doc, &cfg); err != nil { - return err - } + case v1alpha1.ChartsContentKind: + var cfg v1alpha1.Charts + if err := yaml.Unmarshal(doc, &cfg); err != nil { + return err + } - k, err := k3s.NewK3s(cfg.Spec.Version) + for _, ch := range cfg.Spec.Charts { + // TODO: Provide a way to configure syncs + err := storeChart(ctx, s, ch, &action.ChartPathOptions{}) if err != nil { return err } + } - if _, err := s.AddOCICollection(ctx, k); err != nil { - return err - } + case v1alpha1.K3sCollectionKind: + var cfg v1alpha1.K3s + if err := yaml.Unmarshal(doc, &cfg); err != nil { + return err + } - case v1alpha1.ChartsCollectionKind: - var cfg v1alpha1.ThickCharts - if err := yaml.Unmarshal(doc, &cfg); err != nil { - return err - } + k, err := k3s.NewK3s(cfg.Spec.Version) + if err != nil { + return err + } - for _, cfg := range cfg.Spec.Charts { - tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{ - RepoURL: cfg.RepoURL, - Version: cfg.Version, - }) - if err != nil { - return err - } + if _, err := s.AddOCICollection(ctx, k); err != nil { + return err + } - if _, err := s.AddOCICollection(ctx, tc); err != nil { - return err - } + case v1alpha1.ChartsCollectionKind: + var cfg v1alpha1.ThickCharts + if err := yaml.Unmarshal(doc, &cfg); err != nil { + return err + } + + for _, cfg := range cfg.Spec.Charts { + tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{ + RepoURL: cfg.RepoURL, + Version: cfg.Version, + }) + if err != nil { + return err } - case v1alpha1.ImageTxtsContentKind: - var cfg v1alpha1.ImageTxts - if err := yaml.Unmarshal(doc, &cfg); err != nil { + if _, err := s.AddOCICollection(ctx, tc); err != nil { return err } + } - for _, cfgIt := range cfg.Spec.ImageTxts { - it, err := imagetxt.New(cfgIt.Ref, - imagetxt.WithIncludeSources(cfgIt.Sources.Include...), - imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...), - ) - if err != nil { - return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err) - } + case v1alpha1.ImageTxtsContentKind: + var cfg v1alpha1.ImageTxts + if err := yaml.Unmarshal(doc, &cfg); err != nil { + return err + } - if _, err := s.AddOCICollection(ctx, it); err != nil { - return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err) - } + for _, cfgIt := range cfg.Spec.ImageTxts { + it, err := imagetxt.New(cfgIt.Ref, + imagetxt.WithIncludeSources(cfgIt.Sources.Include...), + imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...), + ) + if err != nil { + return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err) } - default: - return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String()) + if _, err := s.AddOCICollection(ctx, it); err != nil { + return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err) + } } + + default: + return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String()) } } - return nil } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 0263408b..70db065c 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -10,6 +10,7 @@ const ( DockerForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" DockerUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar" OCILayer = "application/vnd.oci.image.layer.v1.tar+gzip" + OCIArtifact = "application/vnd.oci.empty.v1+json" // ChartConfigMediaType is the reserved media type for the Helm chart manifest config ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json" @@ -47,4 +48,7 @@ const ( KindAnnotationName = "kind" KindAnnotation = "dev.cosignproject.cosign/image" + + //CarbideRegistry = "rgcrprod.azurecr.us" + CarbideRegistry = "localhost:5001" )