Skip to content

Commit

Permalink
pull carbide flavored hauler manifests from reg
Browse files Browse the repository at this point in the history 
Signed-off-by: Adam Martin <[email protected]>
  • Loading branch information
@amartin120
amartin120 committed Oct 20, 2023
1 parent f2b0c44 commit 39e0fed
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 109 deletions.
2 changes: 1 addition & 1 deletion cmd/hauler/cli/store/info.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item
ctype = "image"
case consts.ChartConfigMediaType:
ctype = "chart"
case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType:
case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType, consts.OCIArtifact:
ctype = "file"
default:
ctype = "unknown"
Expand Down
256 changes: 148 additions & 108 deletions cmd/hauler/cli/store/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@ import (
"fmt"
"io"
"os"
"strings"

"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"k8s.io/apimachinery/pkg/util/yaml"
"github.com/mitchellh/go-homedir"

"github.com/rancherfederal/hauler/pkg/store"

"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
"github.com/rancherfederal/hauler/pkg/collection/imagetxt"
"github.com/rancherfederal/hauler/pkg/collection/k3s"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
Expand All @@ -27,13 +28,15 @@ type SyncOpts struct {
*RootOpts
ContentFiles []string
Key string
Products []string
}

func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()

f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for image signature verification")
f.StringSliceVar(&o.Products, "products", []string{}, "Used for RGS Carbide customers to supply a product and version and Hauler will retrieve the images. i.e. '--product rancher=v2.7.8'")
}

func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
Expand All @@ -45,157 +48,194 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
return err
}

for _, product := range o.Products {
l.Infof("processing content file for product: '%s'", product)
parts := strings.Split(product, "=")
manifestLoc := fmt.Sprintf("%s/hauler/%s-manifest.yaml:%s", consts.CarbideRegistry, parts[0], parts[1])
l.Infof("retrieving product manifest from: '%s'", manifestLoc)
img := v1alpha1.Image{
Name: manifestLoc,
}
err := storeImage(ctx, s, img)
if err != nil {
return err
}
err = ExtractCmd(ctx, &ExtractOpts{RootOpts: o.RootOpts}, s, "hauler/rke2-manifest.yaml:v2.7.8")
if err != nil {
return err
}
filename := fmt.Sprintf("%s-manifest.yaml", parts[0])

fi, err := os.Open(filename)
if err != nil {
return err
}
err = processContent(ctx, fi, o, s)
if err != nil {
return err
}
}

for _, filename := range o.ContentFiles {
l.Debugf("processing content file: '%s'", filename)
fi, err := os.Open(filename)
if err != nil {
return err
}
err = processContent(ctx, fi, o, s)
if err != nil {
return err
}
}

return nil
}

reader := yaml.NewYAMLReader(bufio.NewReader(fi))
func processContent(ctx context.Context, fi *os.File, o *SyncOpts, s *store.Layout) error {
l := log.FromContext(ctx)

var docs [][]byte
for {
raw, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
reader := yaml.NewYAMLReader(bufio.NewReader(fi))

docs = append(docs, raw)
var docs [][]byte
for {
raw, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}

for _, doc := range docs {
obj, err := content.Load(doc)
if err != nil {
l.Debugf("skipping sync of unknown content")
continue
}
docs = append(docs, raw)
}

l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())
for _, doc := range docs {
obj, err := content.Load(doc)
if err != nil {
l.Debugf("skipping sync of unknown content")
continue
}

// TODO: Should type switch instead...
switch obj.GroupVersionKind().Kind {
case v1alpha1.FilesContentKind:
var cfg v1alpha1.Files
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())

for _, f := range cfg.Spec.Files {
err := storeFile(ctx, s, f)
if err != nil {
return err
}
}
// TODO: Should type switch instead...
switch obj.GroupVersionKind().Kind {
case v1alpha1.FilesContentKind:
var cfg v1alpha1.Files
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

case v1alpha1.ImagesContentKind:
var cfg v1alpha1.Images
if err := yaml.Unmarshal(doc, &cfg); err != nil {
for _, f := range cfg.Spec.Files {
err := storeFile(ctx, s, f)
if err != nil {
return err
}
}

case v1alpha1.ImagesContentKind:
var cfg v1alpha1.Images
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

for _, i := range cfg.Spec.Images {

for _, i := range cfg.Spec.Images {

// Check if the user provided a key.
if o.Key != "" || i.Key != "" {
key := o.Key
if i.Key != "" {
key, err = homedir.Expand(i.Key)
}
l.Debugf("key for image [%s]", key)

// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name)
if err != nil {
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
}
l.Infof("signature verified for image [%s]", i.Name)
// Check if the user provided a key.
if o.Key != "" || i.Key != "" {
key := o.Key
if i.Key != "" {
key, err = homedir.Expand(i.Key)
}
l.Debugf("key for image [%s]", key)

err = storeImage(ctx, s, i)
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name)
if err != nil {
return err
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
}
l.Infof("signature verified for image [%s]", i.Name)
}

case v1alpha1.ChartsContentKind:
var cfg v1alpha1.Charts
if err := yaml.Unmarshal(doc, &cfg); err != nil {

err = storeImage(ctx, s, i)
if err != nil {
return err
}
}

for _, ch := range cfg.Spec.Charts {
// TODO: Provide a way to configure syncs
err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
if err != nil {
return err
}
}

case v1alpha1.K3sCollectionKind:
var cfg v1alpha1.K3s
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
case v1alpha1.ChartsContentKind:
var cfg v1alpha1.Charts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

k, err := k3s.NewK3s(cfg.Spec.Version)
for _, ch := range cfg.Spec.Charts {
// TODO: Provide a way to configure syncs
err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
if err != nil {
return err
}
}

if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}
case v1alpha1.K3sCollectionKind:
var cfg v1alpha1.K3s
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

case v1alpha1.ChartsCollectionKind:
var cfg v1alpha1.ThickCharts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
k, err := k3s.NewK3s(cfg.Spec.Version)
if err != nil {
return err
}

for _, cfg := range cfg.Spec.Charts {
tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
RepoURL: cfg.RepoURL,
Version: cfg.Version,
})
if err != nil {
return err
}
if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}

if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
case v1alpha1.ChartsCollectionKind:
var cfg v1alpha1.ThickCharts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

for _, cfg := range cfg.Spec.Charts {
tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
RepoURL: cfg.RepoURL,
Version: cfg.Version,
})
if err != nil {
return err
}

case v1alpha1.ImageTxtsContentKind:
var cfg v1alpha1.ImageTxts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
}

for _, cfgIt := range cfg.Spec.ImageTxts {
it, err := imagetxt.New(cfgIt.Ref,
imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
)
if err != nil {
return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
}
case v1alpha1.ImageTxtsContentKind:
var cfg v1alpha1.ImageTxts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

if _, err := s.AddOCICollection(ctx, it); err != nil {
return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
}
for _, cfgIt := range cfg.Spec.ImageTxts {
it, err := imagetxt.New(cfgIt.Ref,
imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
)
if err != nil {
return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
}

default:
return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
if _, err := s.AddOCICollection(ctx, it); err != nil {
return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
}
}

default:
return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
}
}

return nil
}
4 changes: 4 additions & 0 deletions pkg/consts/consts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const (
DockerForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
DockerUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar"
OCILayer = "application/vnd.oci.image.layer.v1.tar+gzip"
OCIArtifact = "application/vnd.oci.empty.v1+json"

// ChartConfigMediaType is the reserved media type for the Helm chart manifest config
ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
Expand Down Expand Up @@ -47,4 +48,7 @@ const (

KindAnnotationName = "kind"
KindAnnotation = "dev.cosignproject.cosign/image"

//CarbideRegistry = "rgcrprod.azurecr.us"
CarbideRegistry = "localhost:5001"
)

0 comments on commit 39e0fed

Please sign in to comment.