Support diff airgap bundles

cmd/kots/cli/admin-console-push-images.go

@@ -10,8 +10,9 @@ import (
 	"github.com/replicatedhq/kots/pkg/docker/registry"
 	dockerregistry "github.com/replicatedhq/kots/pkg/docker/registry"
 	registrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types"
+	"github.com/replicatedhq/kots/pkg/image"
+	imagetypes "github.com/replicatedhq/kots/pkg/image/types"
 	"github.com/replicatedhq/kots/pkg/k8sutil"
-	"github.com/replicatedhq/kots/pkg/kotsadm"
 	kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types"
 	"github.com/replicatedhq/kots/pkg/logger"
 	"github.com/spf13/cobra"
@@ -53,7 +54,7 @@ func AdminPushImagesCmd() *cobra.Command {
 			}
 
 			if _, err := os.Stat(imageSource); err == nil {
-				err = kotsadm.PushImages(imageSource, *options)
+				err = image.PushImages(imageSource, *options)
 				if err != nil {
 					return errors.Wrap(err, "failed to push images")
 				}
@@ -75,7 +76,7 @@ func AdminPushImagesCmd() *cobra.Command {
 	return cmd
 }
 
-func genAndCheckPushOptions(endpoint string, namespace string, log *logger.CLILogger, v *viper.Viper) (*kotsadmtypes.PushImagesOptions, error) {
+func genAndCheckPushOptions(endpoint string, namespace string, log *logger.CLILogger, v *viper.Viper) (*imagetypes.PushImagesOptions, error) {
 	host, err := getHostFromEndpoint(endpoint)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed get host from endpoint")
@@ -114,7 +115,7 @@ func genAndCheckPushOptions(endpoint string, namespace string, log *logger.CLILo
 		log.FinishSpinner()
 	}
 
-	options := kotsadmtypes.PushImagesOptions{
+	options := imagetypes.PushImagesOptions{
 		KotsadmTag: v.GetString("kotsadm-tag"),
 		Registry: registrytypes.RegistryOptions{
 			Endpoint: endpoint,

cmd/kots/cli/install.go

@@ -24,6 +24,7 @@ import (
 	dockerregistry "github.com/replicatedhq/kots/pkg/docker/registry"
 	"github.com/replicatedhq/kots/pkg/handlers"
 	"github.com/replicatedhq/kots/pkg/identity"
+	"github.com/replicatedhq/kots/pkg/image"
 	"github.com/replicatedhq/kots/pkg/k8sutil"
 	k8sutiltypes "github.com/replicatedhq/kots/pkg/k8sutil/types"
 	"github.com/replicatedhq/kots/pkg/kotsadm"
@@ -332,7 +333,7 @@ func InstallCmd() *cobra.Command {
 				}
 				defer os.RemoveAll(airgapRootDir)
 
-				err = kotsadm.ExtractAppAirgapArchive(airgapArchive, airgapRootDir, v.GetBool("disable-image-push"), deployOptions.ProgressWriter)
+				err = image.ExtractAppAirgapArchive(airgapArchive, airgapRootDir, v.GetBool("disable-image-push"), deployOptions.ProgressWriter)
 				if err != nil {
 					return errors.Wrap(err, "failed to extract images")
 				}

pkg/apparchive/helm-v1beta2.go

@@ -3,7 +3,6 @@ package apparchive
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -13,9 +12,11 @@ import (
 	"github.com/pkg/errors"
 	"github.com/replicatedhq/kots/pkg/base"
 	"github.com/replicatedhq/kots/pkg/docker/registry"
+	dockerregistrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types"
 	"github.com/replicatedhq/kots/pkg/image"
 	"github.com/replicatedhq/kots/pkg/kotsutil"
 	"github.com/replicatedhq/kots/pkg/logger"
+	"github.com/replicatedhq/kots/pkg/midstream"
 	upstreamtypes "github.com/replicatedhq/kots/pkg/upstream/types"
 	"github.com/replicatedhq/kots/pkg/util"
 	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
@@ -168,7 +169,7 @@ func WriteV1Beta2HelmCharts(opts WriteV1Beta2HelmChartsOptions) error {
 	if len(checkedImages) > 0 {
 		opts.KotsKinds.Installation.Spec.KnownImages = append(opts.KotsKinds.Installation.Spec.KnownImages, checkedImages...)
 
-		if err := SaveInstallation(&opts.KotsKinds.Installation, opts.Upstream.GetUpstreamDir(opts.WriteUpstreamOptions)); err != nil {
+		if err := kotsutil.SaveInstallation(&opts.KotsKinds.Installation, opts.Upstream.GetUpstreamDir(opts.WriteUpstreamOptions)); err != nil {
 			return errors.Wrap(err, "failed to save installation")
 		}
 	}
@@ -278,7 +279,7 @@ func templateV1Beta2HelmChartWithValuesToDir(helmChart *kotsv1beta2.HelmChart, c
 	return nil
 }
 
-func processV1Beta2HelmChartImages(opts WriteV1Beta2HelmChartsOptions, helmChart *kotsv1beta2.HelmChart, chartDir string) (*image.RewriteImagesResult, error) {
+func processV1Beta2HelmChartImages(opts WriteV1Beta2HelmChartsOptions, helmChart *kotsv1beta2.HelmChart, chartDir string) (*base.RewriteImagesResult, error) {
 	// template the chart with the builder values to a temp dir and then process images
 	tmpDir, err := os.MkdirTemp("", fmt.Sprintf("kots-images-%s", helmChart.GetDirName()))
 	if err != nil {
@@ -297,7 +298,7 @@ func processV1Beta2HelmChartImages(opts WriteV1Beta2HelmChartsOptions, helmChart
 	}
 
 	builderValuesPath := path.Join(tmpDir, "builder-values.yaml")
-	if err := ioutil.WriteFile(builderValuesPath, builderValuesContent, 0644); err != nil {
+	if err := os.WriteFile(builderValuesPath, builderValuesContent, 0644); err != nil {
 		return nil, errors.Wrap(err, "failed to write builder values file")
 	}
 
@@ -316,7 +317,24 @@ func processV1Beta2HelmChartImages(opts WriteV1Beta2HelmChartsOptions, helmChart
 		dockerHubRegistryCreds, _ = registry.GetCredentialsForRegistryFromConfigJSON(dockerhubSecret.Data[".dockerconfigjson"], registry.DockerHubRegistryName)
 	}
 
-	result, err := image.RewriteBaseImages(opts.ProcessImageOptions, templatedOutputDir, opts.KotsKinds, opts.KotsKinds.License, dockerHubRegistryCreds, opts.RenderOptions.Log)
+	destRegistry := &dockerregistrytypes.RegistryOptions{
+		Endpoint:  opts.ProcessImageOptions.RegistrySettings.Hostname,
+		Namespace: opts.ProcessImageOptions.RegistrySettings.Namespace,
+		Username:  opts.ProcessImageOptions.RegistrySettings.Username,
+		Password:  opts.ProcessImageOptions.RegistrySettings.Password,
+	}
+
+	baseImages, err := image.FindImagesInDir(templatedOutputDir)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find base images")
+	}
+
+	kotsKindsImages, err := kotsutil.GetImagesFromKotsKinds(opts.KotsKinds, destRegistry)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get images from kots kinds")
+	}
+
+	result, err := midstream.RewriteBaseImages(opts.ProcessImageOptions, baseImages, kotsKindsImages, opts.KotsKinds, opts.KotsKinds.License, dockerHubRegistryCreds, opts.RenderOptions.Log)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to rewrite base images")
 	}

pkg/base/airgap.go

@@ -0,0 +1,95 @@
+package base
+
+import (
+	"io"
+
+	"github.com/pkg/errors"
+	registrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types"
+	"github.com/replicatedhq/kots/pkg/image"
+	imagetypes "github.com/replicatedhq/kots/pkg/image/types"
+	"github.com/replicatedhq/kots/pkg/imageutil"
+	"github.com/replicatedhq/kots/pkg/kotsutil"
+	"github.com/replicatedhq/kots/pkg/logger"
+	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
+	kustomizetypes "sigs.k8s.io/kustomize/api/types"
+)
+
+type ProcessAirgapImagesOptions struct {
+	BaseImages          []string
+	KotsKindsImages     []string
+	RootDir             string
+	AirgapRoot          string
+	AirgapBundle        string
+	CreateAppDir        bool
+	PushImages          bool
+	Log                 *logger.CLILogger
+	ReplicatedRegistry  registrytypes.RegistryOptions
+	ReportWriter        io.Writer
+	DestinationRegistry registrytypes.RegistryOptions
+	KotsKinds           *kotsutil.KotsKinds
+}
+
+type ProcessAirgapImagesResult struct {
+	KustomizeImages []kustomizetypes.Image
+	KnownImages     []kotsv1beta1.InstallationImage
+}
+
+func ProcessAirgapImages(opts ProcessAirgapImagesOptions) (*ProcessAirgapImagesResult, error) {
+	pushOpts := imagetypes.PushImagesOptions{
+		Registry:       opts.DestinationRegistry,
+		Log:            opts.Log,
+		ProgressWriter: opts.ReportWriter,
+		LogForUI:       true,
+	}
+
+	if opts.PushImages {
+		if opts.AirgapBundle != "" {
+			err := image.TagAndPushAppImagesFromBundle(opts.AirgapBundle, pushOpts)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to push images from bundle")
+			}
+		} else {
+			err := image.TagAndPushAppImagesFromPath(opts.AirgapRoot, pushOpts)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to push images from dir")
+			}
+		}
+	}
+
+	rewrittenImages := []kustomizetypes.Image{}
+	for _, image := range append(opts.BaseImages, opts.KotsKindsImages...) {
+		rewrittenImage, err := imageutil.RewriteDockerRegistryImage(opts.DestinationRegistry, image)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to rewrite image %s", image)
+		}
+		rewrittenImages = append(rewrittenImages, *rewrittenImage)
+	}
+
+	withAltNames := make([]kustomizetypes.Image, 0)
+	for _, i := range rewrittenImages {
+		altNames, err := imageutil.BuildImageAltNames(i)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to build image alt names")
+		}
+		withAltNames = append(withAltNames, altNames...)
+	}
+
+	result := &ProcessAirgapImagesResult{
+		KustomizeImages: withAltNames,
+		// This list is slightly different from the list we get from app specs because of alternative names,
+		// but it still works because after rewriting image names with private registry, the lists become the same.
+		KnownImages: installationImagesFromKustomizeImages(withAltNames),
+	}
+	return result, nil
+}
+
+func installationImagesFromKustomizeImages(images []kustomizetypes.Image) []kotsv1beta1.InstallationImage {
+	result := make([]kotsv1beta1.InstallationImage, 0)
+	for _, i := range images {
+		result = append(result, kotsv1beta1.InstallationImage{
+			Image:     imageutil.SrcImageFromKustomizeImage(i),
+			IsPrivate: true,
+		})
+	}
+	return result
+}

pkg/upstream/push_images_test.go → pkg/base/airgap_test.go

@@ -1,4 +1,4 @@
-package upstream
+package base
 
 import (
 	"reflect"
@@ -8,7 +8,7 @@ import (
 	kustomizetypes "sigs.k8s.io/kustomize/api/types"
 )
 
-func Test_makeInstallationImages(t *testing.T) {
+func Test_installationImagesFromKustomizeImages(t *testing.T) {
 	tests := []struct {
 		name   string
 		images []kustomizetypes.Image
@@ -49,8 +49,8 @@ func Test_makeInstallationImages(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := makeInstallationImages(tt.images); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("makeInstallationImages() = %v, want %v", got, tt.want)
+			if got := installationImagesFromKustomizeImages(tt.images); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("installationImagesFromKustomizeImages() = %v, want %v", got, tt.want)
 			}
 		})
 	}

pkg/base/base.go

@@ -8,7 +8,6 @@ import (
 	"strconv"
 
 	"github.com/pkg/errors"
-	"github.com/replicatedhq/kots/pkg/k8sdoc"
 	"github.com/replicatedhq/kots/pkg/kotsutil"
 	kotsscheme "github.com/replicatedhq/kotskinds/client/kotsclientset/scheme"
 	troubleshootscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
@@ -285,28 +284,6 @@ func (b Base) ListErrorFiles() []BaseFile {
 	return files
 }
 
-func FindObjectsWithImages(b *Base) []k8sdoc.K8sDoc {
-	var objects []k8sdoc.K8sDoc
-	for _, file := range b.Files {
-		parsed, err := k8sdoc.ParseYAML(file.Content)
-		if err != nil {
-			continue
-		}
-
-		images := parsed.ListImages()
-		if len(images) > 0 {
-			objects = append(objects, parsed)
-		}
-	}
-
-	for _, subBase := range b.Bases {
-		subObjects := FindObjectsWithImages(&subBase)
-		objects = append(objects, subObjects...)
-	}
-
-	return objects
-}
-
 func PrependBaseFilesPath(files []BaseFile, prefix string) []BaseFile {
 	if prefix == "" {
 		return files