Skip to content

Commit

Permalink
Okd graph support (#117)
Browse files Browse the repository at this point in the history 
* commented out query params in cincinnati.go. that needs fixed too.

* almost. need to change the associations to output
correctly

* works. lots of regex

* that wasnt supposed to be there

* useless comments

* more unneeded logging

* save

* cherry-picked from jpower. thanks!

Co-authored-by: Jennifer Power <[email protected]>
  • Loading branch information
@jpower432
Alex Flom and jpower432 authored Oct 13, 2021
1 parent 9e61748 commit 6ce8472
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 36 deletions.
18 changes: 11 additions & 7 deletions pkg/bundle/cincinnati.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,12 @@ func (c Client) GetUpdates(ctx context.Context, uri *url.URL, arch string, chann
var current Update
// Prepare parametrized cincinnati query.
queryParams := uri.Query()
queryParams.Add("arch", arch)
queryParams.Add("channel", channel)
queryParams.Add("id", c.id.String())
queryParams.Add("version", version.String())
if channel != "okd" {
queryParams.Add("arch", arch)
queryParams.Add("channel", channel)
queryParams.Add("id", c.id.String())
queryParams.Add("version", version.String())
}
uri.RawQuery = queryParams.Encode()

// Download the update graph.
Expand Down Expand Up @@ -159,9 +161,11 @@ func (c Client) GetUpdates(ctx context.Context, uri *url.URL, arch string, chann
func (c Client) GetChannelLatest(ctx context.Context, uri *url.URL, arch string, channel string) (semver.Version, error) {
// Prepare parametrized cincinnati query.
queryParams := uri.Query()
queryParams.Add("arch", arch)
queryParams.Add("channel", channel)
queryParams.Add("id", c.id.String())
if channel != "okd" {
queryParams.Add("arch", arch)
queryParams.Add("channel", channel)
queryParams.Add("id", c.id.String())
}
uri.RawQuery = queryParams.Encode()

// Download the update graph.
Expand Down
75 changes: 47 additions & 28 deletions pkg/bundle/release.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ func NewReleaseOptions(ro cli.RootOptions, flags *pflag.FlagSet) *ReleaseOptions
}

const (
UpdateUrl string = "https://api.openshift.com/api/upgrades_info/v1/graph"
UpdateUrl string = "https://api.openshift.com/api/upgrades_info/v1/graph"
OkdUpdateURL string = "https://origin-release.ci.openshift.org/graph"
)

func getTLSConfig() (*tls.Config, error) {
Expand All @@ -75,8 +76,9 @@ func getTLSConfig() (*tls.Config, error) {
return config, nil
}

func newClient() (Client, *url.URL, error) {
upstream, err := url.Parse(UpdateUrl)
func newClient(u string) (Client, *url.URL, error) {

upstream, err := url.Parse(u)
if err != nil {
return Client{}, nil, err
}
Expand All @@ -96,9 +98,9 @@ func newClient() (Client, *url.URL, error) {
}

// Next calculate the upgrade path from the current version to the channel's latest
func calculateUpgradePath(ch v1alpha1.ReleaseChannel, v semver.Version, arch string) (Update, []Update, error) {
func calculateUpgradePath(ch v1alpha1.ReleaseChannel, v semver.Version, url, arch string) (Update, []Update, error) {

client, upstream, err := newClient()
client, upstream, err := newClient(url)
if err != nil {
return Update{}, nil, err
}
Expand All @@ -115,9 +117,9 @@ func calculateUpgradePath(ch v1alpha1.ReleaseChannel, v semver.Version, arch str
return upgrade, upgrades, nil
}

func GetLatestVersion(ch v1alpha1.ReleaseChannel, arch string) (Update, error) {
func GetLatestVersion(ch v1alpha1.ReleaseChannel, url, arch string) (Update, error) {

client, upstream, err := newClient()
client, upstream, err := newClient(url)
if err != nil {
return Update{}, err
}
Expand All @@ -138,7 +140,7 @@ func GetLatestVersion(ch v1alpha1.ReleaseChannel, arch string) (Update, error) {
return upgrade, err
}

func (o *ReleaseOptions) downloadMirror(secret []byte, toDir, from, arch string) (image.AssociationSet, error) {
func (o *ReleaseOptions) downloadMirror(secret []byte, toDir, from, arch, version string) (image.AssociationSet, error) {
opts := release.NewMirrorOptions(o.IOStreams)
opts.From = from
opts.ToDir = toDir
Expand All @@ -156,18 +158,20 @@ func (o *ReleaseOptions) downloadMirror(secret []byte, toDir, from, arch string)
opts.SecurityOptions.Insecure = o.SkipTLS
opts.SecurityOptions.SkipVerification = o.SkipVerification
opts.DryRun = o.DryRun

logrus.Debugln("Starting release download")
if err := opts.Run(); err != nil {
return nil, err
}

// Retrive the mapping information for release
mapping, images, err := o.getMapping(*opts, arch)
logrus.Debugln("starting mapping")
mapping, images, err := o.getMapping(*opts, arch, version)

if err != nil {
return nil, fmt.Errorf("error could retrieve mapping information: %v", err)
}

logrus.Debugln("starting image association")
assocs, err := image.AssociateImageLayers(toDir, mapping, images, image.TypeOCPRelease)
if err != nil {
return nil, err
Expand Down Expand Up @@ -195,19 +199,26 @@ func (o *ReleaseOptions) GetReleasesInitial(cfg v1alpha1.ImageSetConfiguration)

// For each channel in the config file
for _, ch := range cfg.Mirror.OCP.Channels {
// If okd is channel name, then use okd api

var url string
if ch.Name == "okd" {
url = OkdUpdateURL
} else {
url = UpdateUrl
}
for _, arch := range o.arch {

if len(ch.Versions) == 0 {

// If no version was specified from the channel, then get the latest release
latest, err := GetLatestVersion(ch, arch)
latest, err := GetLatestVersion(ch, url, arch)
if err != nil {
return nil, err
}
logrus.Infof("Image to download: %v", latest.Image)
// Download the release
assocs, err := o.downloadMirror([]byte(pullSecret), srcDir, latest.Image, arch)
assocs, err := o.downloadMirror([]byte(pullSecret), srcDir, latest.Image, arch, latest.Version.String())
if err != nil {
return nil, err
}
Expand All @@ -226,13 +237,13 @@ func (o *ReleaseOptions) GetReleasesInitial(cfg v1alpha1.ImageSetConfiguration)
}

// This dumps the available upgrades from the last downloaded version
requested, _, err := calculateUpgradePath(ch, ver, arch)
requested, _, err := calculateUpgradePath(ch, ver, url, arch)
if err != nil {
return nil, fmt.Errorf("failed to get upgrade graph: %v", err)
}

logrus.Infof("requested: %v", requested.Version)
assocs, err := o.downloadMirror([]byte(pullSecret), srcDir, requested.Image, arch)
assocs, err := o.downloadMirror([]byte(pullSecret), srcDir, requested.Image, arch, v)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -275,6 +286,13 @@ func (o *ReleaseOptions) GetReleasesDiff(_ v1alpha1.PastMirror, cfg v1alpha1.Ima
srcDir := filepath.Join(o.Dir, config.SourceDir)

for _, ch := range cfg.Mirror.OCP.Channels {
// If okd is channel name, then use okd api
var url string
if ch.Name == "okd" {
url = OkdUpdateURL
} else {
url = UpdateUrl
}
for _, arch := range o.arch {
// Check for specific version declarations for each specific version
for _, v := range ch.Versions {
Expand All @@ -287,13 +305,13 @@ func (o *ReleaseOptions) GetReleasesDiff(_ v1alpha1.PastMirror, cfg v1alpha1.Ima
}

// This dumps the available upgrades from the last downloaded version
requested, _, err := calculateUpgradePath(ch, ver, arch)
requested, _, err := calculateUpgradePath(ch, ver, url, arch)
if err != nil {
return nil, fmt.Errorf("failed to get upgrade graph: %v", err)
}

logrus.Infof("requested: %v", requested.Version)
assocs, err := o.downloadMirror([]byte(pullSecret), srcDir, requested.Image, arch)
assocs, err := o.downloadMirror([]byte(pullSecret), srcDir, requested.Image, arch, v)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -331,9 +349,7 @@ func (o *ReleaseOptions) GetReleasesDiff(_ v1alpha1.PastMirror, cfg v1alpha1.Ima
}

// getMapping will run release mirror with ToMirror set to true to get mapping information
// FIXME(jpower): the provided mapping does not provide a digest mapping so ICSP cannot
// be created
func (o *ReleaseOptions) getMapping(opts release.MirrorOptions, arch string) (mappings map[string]string, images []string, err error) {
func (o *ReleaseOptions) getMapping(opts release.MirrorOptions, arch, version string) (mappings map[string]string, images []string, err error) {

mappingPath := filepath.Join(o.Dir, "release-mapping.txt")
file, err := os.Create(mappingPath)
Expand Down Expand Up @@ -366,30 +382,33 @@ func (o *ReleaseOptions) getMapping(opts release.MirrorOptions, arch string) (ma
text := scanner.Text()
split := strings.Split(text, " ")
srcRef := split[0]

// Get release image name from mapping
// Only the top release need to be resolve because all other image key associated to the
// will be updated to this value
if strings.Contains(srcRef, "ocp-release") {
//
// afflom - Select on ocp-release OR origin
if strings.Contains(srcRef, "ocp-release") || strings.Contains(srcRef, "origin/release") {
if !image.IsImagePinned(srcRef) {
srcRef, err = pinImages(context.TODO(), srcRef, "", o.SkipTLS)
}
o.release = srcRef
}

// Generate name of target directory
var names []string
dstRef := opts.TargetFn(split[1]).Exact()

// TODO: arch is not provided when getting mapping from release does not included
// the arch in the directory. Need to investigate, but adding it here as a workaround
nameSplit := strings.Split(dstRef, "-")
names = []string{nameSplit[1], arch}
names = append(names, nameSplit[2:]...)
nameSplit := strings.Split(dstRef, version)
names := []string{version, arch}
image := strings.Trim(nameSplit[2], "-")

if image != "" {
names = append(names, image)
}

dstRef = strings.Join(names, "-")

if _, err := file.WriteString(srcRef + "=" + dstRef + "\n"); err != nil {
// Append mapping file
if _, err := file.WriteString(srcRef + "=file://openshift/release:" + dstRef + "\n"); err != nil {
return mappings, images, err
}

Expand Down
11 changes: 10 additions & 1 deletion pkg/image/association.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"

ctrsimgmanifest "github.com/containers/image/v5/manifest"
Expand Down Expand Up @@ -289,17 +290,24 @@ func AssociateImageLayers(rootDir string, imgMappings map[string]string, images
}

// TODO(estroz): maybe just use imgsource.ParseReference() here.
dirRef = strings.TrimPrefix(dirRef, "file://")
re := regexp.MustCompile(`.*file://`)

prefix := re.FindString(dirRef)
dirRef = strings.TrimPrefix(dirRef, prefix)

tagIdx := strings.LastIndex(dirRef, ":")
if tagIdx == -1 {
errs = append(errs, fmt.Errorf("image %q mapping %q has no tag or digest component", image, dirRef))
continue
}

idx := tagIdx
if idIdx := strings.LastIndex(dirRef, "@"); idIdx != -1 {
idx = idIdx
}

tagOrID := dirRef[idx+1:]

dirRef = dirRef[:idx]

imagePath := filepath.Join(localRoot, dirRef)
Expand All @@ -314,6 +322,7 @@ func AssociateImageLayers(rootDir string, imgMappings map[string]string, images
associations, err := associateImageLayers(image, localRoot, dirRef, tagOrID, typ, skipParse)
if err != nil {
errs = append(errs, err)
continue
}
for _, association := range associations {
bundleAssociations.Add(image, association)
Expand Down

0 comments on commit 6ce8472

Please sign in to comment.