diff --git a/go.mod b/go.mod index 8f421fca21..1c4a351de9 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe + github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82 github.com/replicatedhq/kurlkinds v1.3.6 github.com/replicatedhq/troubleshoot v0.72.1 github.com/replicatedhq/yaml/v3 v3.0.0-beta5-replicatedhq diff --git a/go.sum b/go.sum index 85f0a2a2de..a620b1ab72 100644 --- a/go.sum +++ b/go.sum @@ -1525,8 +1525,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe h1:3AJInd06UxzqHmgy8+24CPsT2tYSE0zToJZyuX9q+MA= -github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I= +github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82 h1:QniKgIpcXu4wBMM4xIXGz+lkAU+hSIXFuVM+vxkNk0Y= +github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I= github.com/replicatedhq/kurlkinds v1.3.6 h1:/dhS32cSSZR4yS4vA8EquBvz+VgJCyTqBO9Xw+6eI4M= github.com/replicatedhq/kurlkinds v1.3.6/go.mod h1:c5+hoAkuARgftB2Ft3RCyWRZZPhL0clHEaw7XoGDAg4= github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851 h1:eRlNDHxGfVkPCRXbA4BfQJvt5DHjFiTtWy3R/t4djyY= diff --git a/pkg/airgap/airgap.go b/pkg/airgap/airgap.go index 80dee5a4ff..083baa5217 100644 --- a/pkg/airgap/airgap.go +++ b/pkg/airgap/airgap.go @@ -227,6 +227,7 @@ func CreateAppFromAirgap(opts CreateAirgapAppOpts) (finalError error) { Password: opts.RegistryPassword, IsReadOnly: opts.RegistryIsReadOnly, }, + AppID: opts.PendingApp.ID, AppSlug: opts.PendingApp.Slug, AppSequence: 0, AppVersionLabel: instParams.AppVersionLabel, diff --git a/pkg/airgap/update.go b/pkg/airgap/update.go index f9e1b80e2a..945dd9cac7 100644 --- a/pkg/airgap/update.go +++ b/pkg/airgap/update.go @@ -173,6 +173,7 @@ func UpdateAppFromPath(a *apptypes.App, airgapRoot string, airgapBundlePath stri Silent: true, RewriteImages: true, RewriteImageOptions: registrySettings, + AppID: a.ID, AppSlug: a.Slug, AppSequence: appSequence, SkipCompatibilityCheck: skipCompatibilityCheck, diff --git a/pkg/k8sutil/kotsadm.go b/pkg/k8sutil/kotsadm.go index bc101e168b..a3b625bf50 100644 --- a/pkg/k8sutil/kotsadm.go +++ b/pkg/k8sutil/kotsadm.go @@ -9,6 +9,7 @@ import ( types "github.com/replicatedhq/kots/pkg/k8sutil/types" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" "github.com/replicatedhq/kots/pkg/util" + "github.com/segmentio/ksuid" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -81,11 +82,23 @@ func IsKotsadmClusterScoped(ctx context.Context, clientset kubernetes.Interface, return false } -func GetKotsadmIDConfigMap() (*corev1.ConfigMap, error) { - clientset, err := GetClientset() - if err != nil { - return nil, errors.Wrap(err, "failed to get clientset") +func GetKotsadmID(clientset kubernetes.Interface) string { + var clusterID string + configMap, err := GetKotsadmIDConfigMap(clientset) + // if configmap is not found, generate a new guid and create a new configmap, if configmap is found, use the existing guid, otherwise generate + if err != nil && !kuberneteserrors.IsNotFound(err) { + clusterID = ksuid.New().String() + } else if configMap != nil { + clusterID = configMap.Data["id"] + } else { + // configmap is missing for some reason, recreate with new guid, this will appear as a new instance in the report + clusterID = ksuid.New().String() + CreateKotsadmIDConfigMap(clientset, clusterID) } + return clusterID +} + +func GetKotsadmIDConfigMap(clientset kubernetes.Interface) (*corev1.ConfigMap, error) { namespace := util.PodNamespace existingConfigmap, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), KotsadmIDConfigMapName, metav1.GetOptions{}) if err != nil && !kuberneteserrors.IsNotFound(err) { @@ -96,12 +109,8 @@ func GetKotsadmIDConfigMap() (*corev1.ConfigMap, error) { return existingConfigmap, nil } -func CreateKotsadmIDConfigMap(kotsadmID string) error { +func CreateKotsadmIDConfigMap(clientset kubernetes.Interface, kotsadmID string) error { var err error = nil - clientset, err := GetClientset() - if err != nil { - return err - } configmap := corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -136,11 +145,7 @@ func IsKotsadmIDConfigMapPresent() (bool, error) { return true, nil } -func UpdateKotsadmIDConfigMap(kotsadmID string) error { - clientset, err := GetClientset() - if err != nil { - return errors.Wrap(err, "failed to get clientset") - } +func UpdateKotsadmIDConfigMap(clientset kubernetes.Interface, kotsadmID string) error { namespace := util.PodNamespace existingConfigMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), KotsadmIDConfigMapName, metav1.GetOptions{}) if err != nil && !kuberneteserrors.IsNotFound(err) { diff --git a/pkg/k8sutil/kotsadm_test.go b/pkg/k8sutil/kotsadm_test.go new file mode 100644 index 0000000000..455bb34fa4 --- /dev/null +++ b/pkg/k8sutil/kotsadm_test.go @@ -0,0 +1,62 @@ +package k8sutil + +import ( + "context" + "testing" + + "gopkg.in/go-playground/assert.v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +func TestGetKotsadmID(t *testing.T) { + + type args struct { + clientset kubernetes.Interface + } + tests := []struct { + name string + args args + want string + shouldCreateConfigMap bool + }{ + { + name: "configmap exists", + args: args{ + clientset: fake.NewSimpleClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: KotsadmIDConfigMapName}, + Data: map[string]string{"id": "cluster-id"}, + }), + }, + want: "cluster-id", + shouldCreateConfigMap: false, + }, + { + name: "configmap does not exist, should create", + args: args{ + clientset: fake.NewSimpleClientset(), + }, + want: "", + shouldCreateConfigMap: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetKotsadmID(tt.args.clientset) + if tt.want != "" { + assert.Equal(t, tt.want, got) + } else { + // a random uuid is generated + assert.NotEqual(t, "", got) + } + + if tt.shouldCreateConfigMap { + // should have created the configmap if it didn't exist + _, err := tt.args.clientset.CoreV1().ConfigMaps("").Get(context.TODO(), KotsadmIDConfigMapName, metav1.GetOptions{}) + assert.Equal(t, nil, err) + } + }) + } +} diff --git a/pkg/kotsadmupstream/upstream.go b/pkg/kotsadmupstream/upstream.go index 7d2f6eb4f4..1e2e77e5fd 100644 --- a/pkg/kotsadmupstream/upstream.go +++ b/pkg/kotsadmupstream/upstream.go @@ -222,6 +222,7 @@ func DownloadUpdate(appID string, update types.Update, skipPreflights bool, skip ExcludeAdminConsole: true, CreateAppDir: false, ReportWriter: pipeWriter, + AppID: a.ID, AppSlug: a.Slug, AppSequence: appSequence, IsGitOps: a.IsGitOps, diff --git a/pkg/kotsutil/yaml.go b/pkg/kotsutil/yaml.go index fb31c73d5f..d613deedd0 100644 --- a/pkg/kotsutil/yaml.go +++ b/pkg/kotsutil/yaml.go @@ -2,10 +2,13 @@ package kotsutil import ( "bytes" + "fmt" + "strings" "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/util" yaml "github.com/replicatedhq/yaml/v3" + goyaml "gopkg.in/yaml.v3" k8syaml "sigs.k8s.io/yaml" ) @@ -85,3 +88,159 @@ func removeNilFieldsFromMap(input map[string]interface{}) bool { return removedItems } + +func MergeYAMLNodes(targetNodes []*goyaml.Node, overrideNodes []*goyaml.Node) []*goyaml.Node { + // Since inputs are arrays and not maps, we need to: + // 1. Copy all keys in targetNodes, overriding the ones that match from overrideNodes + // 2. Add all keys from overrideNodes that don't exist in targetNodes + + if len(overrideNodes) == 0 { + return targetNodes + } + + if len(targetNodes) == 0 { + return overrideNodes + } + + // Special case where top level node is either a mapping node or an array + if len(targetNodes) == 1 && len(overrideNodes) == 1 { + if targetNodes[0].Kind == goyaml.MappingNode && overrideNodes[0].Kind == goyaml.MappingNode { + return []*goyaml.Node{ + { + Kind: goyaml.MappingNode, + Content: MergeYAMLNodes(targetNodes[0].Content, overrideNodes[0].Content), + }, + } + } + + if targetNodes[0].Value == overrideNodes[0].Value { + return overrideNodes + } + + return append(targetNodes, overrideNodes...) + } + + // 1. Copy all keys in targetNodes, overriding the ones that match from overrideNodes + newNodes := make([]*goyaml.Node, 0) + for i := 0; i < len(targetNodes)-1; i += 2 { + var additionalNode *goyaml.Node + for j := 0; j < len(overrideNodes)-1; j += 2 { + nodeNameI := targetNodes[i] + nodeValueI := targetNodes[i+1] + + nodeNameJ := overrideNodes[j] + nodeValueJ := overrideNodes[j+1] + + if nodeNameI.Value != nodeNameJ.Value { + continue + } + + additionalNode = &goyaml.Node{ + Kind: nodeValueJ.Kind, + Tag: nodeValueJ.Tag, + Line: nodeValueJ.Line, + Style: nodeValueJ.Style, + Anchor: nodeValueJ.Anchor, + Value: nodeValueJ.Value, + Alias: nodeValueJ.Alias, + HeadComment: nodeValueJ.HeadComment, + LineComment: nodeValueJ.LineComment, + FootComment: nodeValueJ.FootComment, + Column: nodeValueJ.Column, + } + + if nodeValueI.Kind == goyaml.MappingNode && nodeValueJ.Kind == goyaml.MappingNode { + additionalNode.Content = MergeYAMLNodes(nodeValueI.Content, nodeValueJ.Content) + } else { + additionalNode.Content = nodeValueJ.Content + } + + break + } + + if additionalNode != nil { + newNodes = append(newNodes, targetNodes[i], additionalNode) + } else { + newNodes = append(newNodes, targetNodes[i], targetNodes[i+1]) + } + } + + // 2. Add all keys from overrideNodes that don't exist in targetNodes + for j := 0; j < len(overrideNodes)-1; j += 2 { + isFound := false + for i := 0; i < len(newNodes)-1; i += 2 { + nodeNameI := newNodes[i] + nodeValueI := newNodes[i+1] + + additionalNodeName := overrideNodes[j] + additionalNodeValue := overrideNodes[j+1] + + if nodeNameI.Value != additionalNodeName.Value { + continue + } + + if nodeValueI.Kind == goyaml.MappingNode && additionalNodeValue.Kind == goyaml.MappingNode { + nodeValueI.Content = MergeYAMLNodes(nodeValueI.Content, additionalNodeValue.Content) + } + + isFound = true + break + } + + if !isFound { + newNodes = append(newNodes, overrideNodes[j], overrideNodes[j+1]) + } + } + + return newNodes +} + +func ContentToDocNode(doc *goyaml.Node, nodes []*goyaml.Node) *goyaml.Node { + if doc == nil { + return &goyaml.Node{ + Kind: goyaml.DocumentNode, + Content: nodes, + } + } + return &goyaml.Node{ + Kind: doc.Kind, + Tag: doc.Tag, + Line: doc.Line, + Style: doc.Style, + Anchor: doc.Anchor, + Value: doc.Value, + Alias: doc.Alias, + HeadComment: doc.HeadComment, + LineComment: doc.LineComment, + FootComment: doc.FootComment, + Column: doc.Column, + Content: nodes, + } +} + +func NodeToYAML(node *goyaml.Node) ([]byte, error) { + var renderedContents bytes.Buffer + yamlEncoder := goyaml.NewEncoder(&renderedContents) + yamlEncoder.SetIndent(2) // this may change indentations of the original values.yaml, but this matches out tests + err := yamlEncoder.Encode(node) + if err != nil { + return nil, errors.Wrap(err, "marshal") + } + + return renderedContents.Bytes(), nil +} + +// Handy functions for printing YAML nodes +func PrintNodes(nodes []*goyaml.Node, i int) { + for _, n := range nodes { + PrintNode(n, i) + } +} +func PrintNode(n *goyaml.Node, i int) { + if n == nil { + return + } + indent := strings.Repeat(" ", i*2) + fmt.Printf("%stag:%v, style:%v, kind:%v, value:%v\n", indent, n.Tag, n.Style, n.Kind, n.Value) + PrintNodes(n.Content, i+1) +} diff --git a/pkg/online/online.go b/pkg/online/online.go index a7435fb1a9..2d91e585e4 100644 --- a/pkg/online/online.go +++ b/pkg/online/online.go @@ -151,6 +151,7 @@ func CreateAppFromOnline(opts CreateOnlineAppOpts) (_ *kotsutil.KotsKinds, final ConfigFile: configFile, IdentityConfigFile: identityConfigFile, ReportWriter: pipeWriter, + AppID: opts.PendingApp.ID, AppSlug: opts.PendingApp.Slug, AppSequence: 0, AppVersionLabel: opts.PendingApp.VersionLabel, diff --git a/pkg/pull/pull.go b/pkg/pull/pull.go index c9df6ef167..24ef5f4436 100644 --- a/pkg/pull/pull.go +++ b/pkg/pull/pull.go @@ -58,6 +58,7 @@ type PullOptions struct { RewriteImageOptions registrytypes.RegistrySettings SkipHelmChartCheck bool ReportWriter io.Writer + AppID string AppSlug string AppSequence int64 AppVersionLabel string @@ -286,6 +287,9 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { IsOpenShift: k8sutil.IsOpenShift(clientset), IsGKEAutopilot: k8sutil.IsGKEAutopilot(clientset), IncludeMinio: pullOptions.IncludeMinio, + IsAirgap: pullOptions.AirgapRoot != "", + KotsadmID: k8sutil.GetKotsadmID(clientset), + AppID: pullOptions.AppID, } if err := upstream.WriteUpstream(u, writeUpstreamOptions); err != nil { log.FinishSpinnerWithError() diff --git a/pkg/reporting/app.go b/pkg/reporting/app.go index a7942bbdb7..dcb02e12c4 100644 --- a/pkg/reporting/app.go +++ b/pkg/reporting/app.go @@ -138,13 +138,18 @@ func initFromDownstream() error { return errors.Wrap(err, "failed to check configmap") } + clientset, err := k8sutil.GetClientset() + if err != nil { + return errors.Wrap(err, "failed to get clientset") + } + if isKotsadmIDGenerated && !cmpExists { kotsadmID := ksuid.New().String() - err = k8sutil.CreateKotsadmIDConfigMap(kotsadmID) + err = k8sutil.CreateKotsadmIDConfigMap(clientset, kotsadmID) } else if !isKotsadmIDGenerated && !cmpExists { - err = k8sutil.CreateKotsadmIDConfigMap(clusterID) + err = k8sutil.CreateKotsadmIDConfigMap(clientset, clusterID) } else if !isKotsadmIDGenerated && cmpExists { - err = k8sutil.UpdateKotsadmIDConfigMap(clusterID) + err = k8sutil.UpdateKotsadmIDConfigMap(clientset, clusterID) } else { // id exists and so as configmap, noop } @@ -181,16 +186,7 @@ func GetReportingInfo(appID string) *types.ReportingInfo { if util.IsHelmManaged() { r.ClusterID = clusterID } else { - configMap, err := k8sutil.GetKotsadmIDConfigMap() - if err != nil { - r.ClusterID = ksuid.New().String() - } else if configMap != nil { - r.ClusterID = configMap.Data["id"] - } else { - // configmap is missing for some reason, recreate with new guid, this will appear as a new instance in the report - r.ClusterID = ksuid.New().String() - k8sutil.CreateKotsadmIDConfigMap(r.ClusterID) - } + r.ClusterID = k8sutil.GetKotsadmID(clientset) di, err := getDownstreamInfo(appID) if err != nil { diff --git a/pkg/rewrite/rewrite.go b/pkg/rewrite/rewrite.go index cca1f3e5a5..3b1ba5e4ec 100644 --- a/pkg/rewrite/rewrite.go +++ b/pkg/rewrite/rewrite.go @@ -74,6 +74,7 @@ func Rewrite(rewriteOptions RewriteOptions) error { CurrentVersionIsRequired: rewriteOptions.Installation.Spec.IsRequired, CurrentReplicatedRegistryDomain: rewriteOptions.Installation.Spec.ReplicatedRegistryDomain, CurrentReplicatedProxyDomain: rewriteOptions.Installation.Spec.ReplicatedProxyDomain, + CurrentReplicatedChartNames: rewriteOptions.Installation.Spec.ReplicatedChartNames, EncryptionKey: rewriteOptions.Installation.Spec.EncryptionKey, License: rewriteOptions.License, AppSequence: rewriteOptions.AppSequence, @@ -103,6 +104,9 @@ func Rewrite(rewriteOptions RewriteOptions) error { PreserveInstallation: true, IsOpenShift: k8sutil.IsOpenShift(clientset), IsGKEAutopilot: k8sutil.IsGKEAutopilot(clientset), + IsAirgap: rewriteOptions.IsAirgap, + KotsadmID: k8sutil.GetKotsadmID(clientset), + AppID: rewriteOptions.AppID, } if err = upstream.WriteUpstream(u, writeUpstreamOptions); err != nil { log.FinishSpinnerWithError() diff --git a/pkg/upstream/fetch.go b/pkg/upstream/fetch.go index 3f325d4fe0..b7663280f8 100644 --- a/pkg/upstream/fetch.go +++ b/pkg/upstream/fetch.go @@ -49,6 +49,7 @@ func downloadUpstream(upstreamURI string, fetchOptions *types.FetchOptions) (*ty pickVersionIsRequired(fetchOptions), pickReplicatedRegistryDomain(fetchOptions), pickReplicatedProxyDomain(fetchOptions), + pickReplicatedChartNames(fetchOptions), fetchOptions.AppSlug, fetchOptions.AppSequence, fetchOptions.Airgap != nil, @@ -110,3 +111,10 @@ func pickCursor(fetchOptions *types.FetchOptions) replicatedapp.ReplicatedCursor Cursor: fetchOptions.CurrentCursor, } } + +func pickReplicatedChartNames(fetchOptions *types.FetchOptions) []string { + if fetchOptions.Airgap != nil { + return fetchOptions.Airgap.Spec.ReplicatedChartNames + } + return fetchOptions.CurrentReplicatedChartNames +} diff --git a/pkg/upstream/helm.go b/pkg/upstream/helm.go new file mode 100644 index 0000000000..1f0232399b --- /dev/null +++ b/pkg/upstream/helm.go @@ -0,0 +1,461 @@ +package upstream + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/replicatedhq/kots/pkg/buildversion" + "github.com/replicatedhq/kots/pkg/kotsutil" + "github.com/replicatedhq/kots/pkg/upstream/types" + "gopkg.in/yaml.v3" +) + +// configureChart will configure the chart archive (values.yaml), +// repackage it, and return the updated content of the chart +func configureChart(chartContent []byte, u *types.Upstream, options types.WriteOptions) ([]byte, error) { + replicatedChartName, isSubchart, err := findReplicatedChart(bytes.NewReader(chartContent), u.ReplicatedChartNames) + if err != nil { + return nil, errors.Wrap(err, "find replicated chart") + } + if replicatedChartName == "" { + return chartContent, nil + } + + chartValues, pathInArchive, extractedArchiveRoot, err := findTopLevelChartValues(bytes.NewReader(chartContent)) + if err != nil { + return nil, errors.Wrap(err, "find top level chart values") + } + defer os.RemoveAll(extractedArchiveRoot) + + updatedValues, err := configureChartValues(chartValues, replicatedChartName, isSubchart, u, options) + if err != nil { + return nil, errors.Wrap(err, "configure values yaml") + } + + if err := os.WriteFile(filepath.Join(extractedArchiveRoot, pathInArchive), updatedValues, 0644); err != nil { + return nil, errors.Wrap(err, "write configured values.yaml") + } + + updatedArchive, err := packageChartArchive(extractedArchiveRoot) + if err != nil { + return nil, errors.Wrap(err, "package chart archive") + } + defer os.RemoveAll(updatedArchive) + + updatedContents, err := os.ReadFile(updatedArchive) + if err != nil { + return nil, errors.Wrap(err, "read updated archive") + } + + return updatedContents, nil +} + +// findReplicatedChart will look for the replicated chart in the archive +// and return the name of the replicated chart and whether it is the parent chart or a subchart +func findReplicatedChart(chartArchive io.Reader, replicatedChartNames []string) (string, bool, error) { + gzReader, err := gzip.NewReader(chartArchive) + if err != nil { + return "", false, errors.Wrap(err, "failed to create gzip reader") + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return "", false, errors.Wrap(err, "failed to read header from tar") + } + + switch header.Typeflag { + case tar.TypeReg: + if filepath.Base(header.Name) != "Chart.yaml" { + continue + } + + // we only care about the root Chart.yaml file or the Chart.yaml file of direct subcharts (not subsubcharts) + parts := strings.Split(header.Name, string(os.PathSeparator)) // e.g. replicated/Chart.yaml or nginx/charts/replicated/Chart.yaml + if len(parts) != 2 && len(parts) != 4 { + continue + } + + content, err := io.ReadAll(tarReader) + if err != nil { + return "", false, errors.Wrapf(err, "failed to read file %s", header.Name) + } + + chartInfo := struct { + ChartName string `json:"name" yaml:"name"` + }{} + if err := yaml.Unmarshal(content, &chartInfo); err != nil { + return "", false, errors.Wrapf(err, "failed to unmarshal %s", header.Name) + } + + for _, replicatedChartName := range replicatedChartNames { + if chartInfo.ChartName == replicatedChartName { + return replicatedChartName, len(parts) == 4, nil + } + } + } + } + + return "", false, nil +} + +func findTopLevelChartValues(r io.Reader) (valuesYaml []byte, pathInArchive string, workspace string, finalErr error) { + workspace, err := os.MkdirTemp("", "extracted-chart-") + if err != nil { + finalErr = errors.Wrap(err, "failed to create temp directory") + return + } + + defer func() { + if finalErr != nil { + os.RemoveAll(workspace) + workspace = "" + } + }() + + gzReader, err := gzip.NewReader(r) + if err != nil { + finalErr = errors.Wrap(err, "failed to create gzip reader") + return + } + + tarReader := tar.NewReader(gzReader) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + finalErr = errors.Wrap(err, "failed to read header from tar") + return + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.Mkdir(filepath.Join(workspace, header.Name), fs.FileMode(header.Mode)); err != nil { + finalErr = errors.Wrap(err, "failed to create directory from archive") + return + } + case tar.TypeReg: + content, err := io.ReadAll(tarReader) + if err != nil { + finalErr = errors.Wrap(err, "failed to read file") + return + } + + if filepath.Base(header.Name) == "values.yaml" { + // only get the values.yaml from the top level chart + p := filepath.Dir(header.Name) + if !strings.Contains(p, string(os.PathSeparator)) { + pathInArchive = header.Name + valuesYaml = content + } + } + + dir := filepath.Dir(filepath.Join(workspace, header.Name)) + if err := os.MkdirAll(dir, 0700); err != nil { + finalErr = errors.Wrap(err, "failed to create directory from filename") + return + } + + outFile, err := os.Create(filepath.Join(workspace, header.Name)) + if err != nil { + finalErr = errors.Wrap(err, "failed to create file") + return + } + defer outFile.Close() + if err := os.WriteFile(outFile.Name(), content, header.FileInfo().Mode()); err != nil { + finalErr = errors.Wrap(err, "failed to write file") + return + } + } + } + + return +} + +func configureChartValues(valuesYAML []byte, replicatedChartName string, isSubchart bool, u *types.Upstream, options types.WriteOptions) ([]byte, error) { + // unmarshal to insert the replicated values + var valuesNode yaml.Node + if err := yaml.Unmarshal([]byte(valuesYAML), &valuesNode); err != nil { + return nil, errors.Wrap(err, "unmarshal values") + } + + if len(valuesNode.Content) == 0 { + return nil, errors.New("no content") + } + + if replicatedChartName != "" { + err := addReplicatedValues(valuesNode.Content[0], replicatedChartName, isSubchart, u, options) + if err != nil { + return nil, errors.Wrap(err, "add replicated values") + } + } + + if err := addGlobalReplicatedValues(valuesNode.Content[0], u, options); err != nil { + return nil, errors.Wrap(err, "add global replicated values") + } + + updatedValues, err := kotsutil.NodeToYAML(&valuesNode) + if err != nil { + return nil, errors.Wrap(err, "node to yaml") + } + + return updatedValues, nil +} + +func addReplicatedValues(doc *yaml.Node, replicatedChartName string, isSubchart bool, u *types.Upstream, options types.WriteOptions) error { + replicatedValues, err := buildReplicatedValues(u, options) + if err != nil { + return errors.Wrap(err, "build replicated values") + } + + targetNode := doc + hasReplicatedValues := false + v := replicatedValues + + // if replicated sdk is included as a subchart, + // we make sure to add the values under the subchart name + // as helm expects the field name to match the subchart name + if isSubchart { + for i, n := range doc.Content { + if n.Value == replicatedChartName { // check if field already exists + targetNode = doc.Content[i+1] + hasReplicatedValues = true + break + } + } + if !hasReplicatedValues { + v = map[string]interface{}{ + replicatedChartName: replicatedValues, + } + } + } + + additionalYAML, err := yaml.Marshal(v) + if err != nil { + return errors.Wrap(err, "marshal additional values") + } + + var additionalNode yaml.Node + if err := yaml.Unmarshal([]byte(additionalYAML), &additionalNode); err != nil { + return errors.Wrap(err, "unmarshal additional values") + } + + if !hasReplicatedValues && isSubchart { + targetNode.Content = append(targetNode.Content, additionalNode.Content[0].Content...) + } else { + targetNode.Content = kotsutil.MergeYAMLNodes(targetNode.Content, additionalNode.Content[0].Content) + } + + return nil +} + +func buildReplicatedValues(u *types.Upstream, options types.WriteOptions) (map[string]interface{}, error) { + replicatedValues := map[string]interface{}{ + "replicatedID": options.KotsadmID, + "appID": options.AppID, + "userAgent": buildversion.GetUserAgent(), + "airgap": options.IsAirgap, + } + + // only add the license if this is an airgap install + // because the airgap builder doesn't have the license context + if u.License != nil && options.IsAirgap { + replicatedValues["license"] = string(MustMarshalLicense(u.License)) + } + + return replicatedValues, nil +} + +func addGlobalReplicatedValues(doc *yaml.Node, u *types.Upstream, options types.WriteOptions) error { + globalReplicatedValues, err := buildGlobalReplicatedValues(u, options) + if err != nil { + return errors.Wrap(err, "build global replicated values") + } + if len(globalReplicatedValues) == 0 { + return nil + } + + targetNode := doc + hasGlobal := false + for i, n := range doc.Content { + if n.Value == "global" { + targetNode = doc.Content[i+1] + hasGlobal = true + break + } + } + + hasGlobalReplicated := false + if hasGlobal { + for i, n := range targetNode.Content { + if n.Value == "replicated" { + targetNode = targetNode.Content[i+1] + hasGlobalReplicated = true + break + } + } + } + + v := globalReplicatedValues + if !hasGlobalReplicated { + v = map[string]interface{}{ + "replicated": v, + } + if !hasGlobal { + v = map[string]interface{}{ + "global": v, + } + } + } + + additionalYAML, err := yaml.Marshal(v) + if err != nil { + return errors.Wrap(err, "marshal additional values") + } + + var additionalNode yaml.Node + if err := yaml.Unmarshal([]byte(additionalYAML), &additionalNode); err != nil { + return errors.Wrap(err, "unmarshal additional values") + } + + if hasGlobalReplicated || hasGlobal { + targetNode.Content = kotsutil.MergeYAMLNodes(targetNode.Content, additionalNode.Content[0].Content) + } else { + targetNode.Content = append(targetNode.Content, additionalNode.Content[0].Content...) + } + + return nil +} + +func buildGlobalReplicatedValues(u *types.Upstream, options types.WriteOptions) (map[string]interface{}, error) { + globalReplicatedValues := map[string]interface{}{} + + // only add license related info if this is an airgap install + // because the airgap builder doesn't have the license context + if u.License != nil && options.IsAirgap { + globalReplicatedValues["channelName"] = u.License.Spec.ChannelName + globalReplicatedValues["customerName"] = u.License.Spec.CustomerName + globalReplicatedValues["customerEmail"] = u.License.Spec.CustomerEmail + globalReplicatedValues["licenseID"] = u.License.Spec.LicenseID + globalReplicatedValues["licenseType"] = u.License.Spec.LicenseType + + // we marshal and then unmarshal entitlements into an interface to evaluate entitlement values + // and end up with a single value instead of (intVal, boolVal, strVal, and type) + marshalledEntitlements, err := json.Marshal(u.License.Spec.Entitlements) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal entitlements") + } + + var licenseFields map[string]interface{} + if err := json.Unmarshal(marshalledEntitlements, &licenseFields); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal entitlements") + } + + // add the field name if missing + for k, v := range licenseFields { + if name, ok := v.(map[string]interface{})["name"]; !ok || name == "" { + licenseFields[k].(map[string]interface{})["name"] = k + } + } + + globalReplicatedValues["licenseFields"] = licenseFields + + // add docker config json + auth := fmt.Sprintf("%s:%s", u.License.Spec.LicenseID, u.License.Spec.LicenseID) + encodedAuth := base64.StdEncoding.EncodeToString([]byte(auth)) + dockercfg := map[string]interface{}{ + "auths": map[string]interface{}{ + u.ReplicatedProxyDomain: map[string]string{ + "auth": encodedAuth, + }, + u.ReplicatedRegistryDomain: map[string]string{ + "auth": encodedAuth, + }, + }, + } + + b, err := json.Marshal(dockercfg) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal dockercfg") + } + + globalReplicatedValues["dockerconfigjson"] = base64.StdEncoding.EncodeToString(b) + } + + return globalReplicatedValues, nil +} + +func packageChartArchive(extractedArchiveRoot string) (string, error) { + configuredChartArchive, err := os.CreateTemp("", "configured-chart-") + if err != nil { + return "", errors.Wrap(err, "create temp file") + } + + gzipWriter := gzip.NewWriter(configuredChartArchive) + defer gzipWriter.Close() + + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + err = filepath.Walk(extractedArchiveRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + file, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "open file '%s'", path) + } + defer file.Close() + + rel, err := filepath.Rel(extractedArchiveRoot, path) + if err != nil { + return errors.New(fmt.Sprintf("Could not get relative path for file '%s', got error '%s'", path, err.Error())) + } + + header := &tar.Header{ + Name: rel, + Size: info.Size(), + Mode: int64(info.Mode()), + ModTime: info.ModTime(), + } + + err = tarWriter.WriteHeader(header) + if err != nil { + return errors.New(fmt.Sprintf("Could not write header for file '%s', got error '%s'", path, err.Error())) + } + + _, err = io.Copy(tarWriter, file) + if err != nil { + return errors.New(fmt.Sprintf("Could not copy the file '%s' data to the tarball, got error '%s'", path, err.Error())) + } + + return nil + }) + if err != nil { + return "", errors.Wrap(err, "walk file tree") + } + + return configuredChartArchive.Name(), nil +} diff --git a/pkg/upstream/helm_test.go b/pkg/upstream/helm_test.go new file mode 100644 index 0000000000..a30d15ccd8 --- /dev/null +++ b/pkg/upstream/helm_test.go @@ -0,0 +1,1293 @@ +package upstream + +import ( + "fmt" + "testing" + + "github.com/pmezard/go-difflib/difflib" + "github.com/replicatedhq/kots/pkg/upstream/types" + "github.com/replicatedhq/kots/pkg/util" + kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Test_configureChart(t *testing.T) { + testReplicatedChartNames := []string{ + "replicated", + "replicated-sdk", + } + + type Test struct { + name string + isAirgap bool + chartContent map[string]string + want map[string]string + wantErr bool + } + + tests := []Test{ + { + name: "online - a standalone non-replicated chart", + isAirgap: false, + chartContent: map[string]string{ + "non-replicated/Chart.yaml": `apiVersion: v1 +name: not-replicated +version: 1.0.0 +description: Not a Replicated Chart +`, + "non-replicated/values.yaml": `# this values.yaml file should not change + +# do not change global values +global: + some: value + +# use this value to configure the chart +some: value +`, + }, + want: map[string]string{ + "non-replicated/Chart.yaml": `apiVersion: v1 +name: not-replicated +version: 1.0.0 +description: Not a Replicated Chart +`, + "non-replicated/values.yaml": `# this values.yaml file should not change + +# do not change global values +global: + some: value + +# use this value to configure the chart +some: value +`, + }, + wantErr: false, + }, + { + name: "airgap - a standalone non-replicated chart", + isAirgap: true, + chartContent: map[string]string{ + "non-replicated/Chart.yaml": `apiVersion: v1 +name: not-replicated +version: 1.0.0 +description: Not a Replicated Chart +`, + "non-replicated/values.yaml": `# this values.yaml file should not change + +# do not change global values +global: + some: value + +# use this value to configure the chart +some: value +`, + }, + want: map[string]string{ + "non-replicated/Chart.yaml": `apiVersion: v1 +name: not-replicated +version: 1.0.0 +description: Not a Replicated Chart +`, + "non-replicated/values.yaml": `# this values.yaml file should not change + +# do not change global values +global: + some: value + +# use this value to configure the chart +some: value +`, + }, + wantErr: false, + }, + { + name: "online - an nginx chart with the 'common' subchart only", + isAirgap: false, + chartContent: map[string]string{ + "nginx/Chart.yaml": `apiVersion: v2 +name: nginx +version: 12.0.1 +description: An NGINX Chart +`, + "nginx/values.yaml": `## @section Global parameters +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass + +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker registry secret names as an array +## +global: + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## + imagePullSecrets: [] + +## @section Common parameters + +## @param nameOverride String to partially override nginx.fullname template (will maintain the release name) +## +nameOverride: "" +`, + "nginx/charts/common/Chart.yaml": `apiVersion: v2 +name: common +version: 1.13.1 +description: A Common Chart +`, + "nginx/charts/common/values.yaml": `# do not change this file + +# do not change global values +global: + some: value + +# keep this comment +another: value +`, + }, + want: map[string]string{ + "nginx/Chart.yaml": `apiVersion: v2 +name: nginx +version: 12.0.1 +description: An NGINX Chart +`, + "nginx/values.yaml": `## @section Global parameters +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass + +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker registry secret names as an array +## +global: + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## + imagePullSecrets: [] + +## @section Common parameters + +## @param nameOverride String to partially override nginx.fullname template (will maintain the release name) +## +nameOverride: "" +`, + "nginx/charts/common/Chart.yaml": `apiVersion: v2 +name: common +version: 1.13.1 +description: A Common Chart +`, + "nginx/charts/common/values.yaml": `# do not change this file + +# do not change global values +global: + some: value + +# keep this comment +another: value +`, + }, + wantErr: false, + }, + { + name: "airgap - an nginx chart with the 'common' subchart only", + isAirgap: true, + chartContent: map[string]string{ + "nginx/Chart.yaml": `apiVersion: v2 +name: nginx +version: 12.0.1 +description: An NGINX Chart +`, + "nginx/values.yaml": `## @section Global parameters +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass + +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker registry secret names as an array +## +global: + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## + imagePullSecrets: [] + +## @section Common parameters + +## @param nameOverride String to partially override nginx.fullname template (will maintain the release name) +## +nameOverride: "" +`, + "nginx/charts/common/Chart.yaml": `apiVersion: v2 +name: common +version: 1.13.1 +description: A Common Chart +`, + "nginx/charts/common/values.yaml": `# do not change this file + +# do not change global values +global: + some: value + +# keep this comment +another: value +`, + }, + want: map[string]string{ + "nginx/Chart.yaml": `apiVersion: v2 +name: nginx +version: 12.0.1 +description: An NGINX Chart +`, + "nginx/values.yaml": `## @section Global parameters +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass + +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker registry secret names as an array +## +global: + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## + imagePullSecrets: [] + +## @section Common parameters + +## @param nameOverride String to partially override nginx.fullname template (will maintain the release name) +## +nameOverride: "" +`, + "nginx/charts/common/Chart.yaml": `apiVersion: v2 +name: common +version: 1.13.1 +description: A Common Chart +`, + "nginx/charts/common/values.yaml": `# do not change this file + +# do not change global values +global: + some: value + +# keep this comment +another: value +`, + }, + wantErr: false, + }, + } + + // Generate dynamic tests using the supported replicated chart names + for _, chartName := range testReplicatedChartNames { + tests = append(tests, Test{ + name: "online - a standalone replicated chart", + isAirgap: false, + chartContent: map[string]string{ + "replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "replicated/values.yaml": `# preserve this comment + +license: online-license +appName: online-app-name +channelID: online-channel-id +channelName: online-channel-name +channelSequence: 2 +releaseCreatedAt: "2023-10-02T00:00:00Z" +releaseNotes: override my release notes +releaseSequence: 1 +statusInformers: + - deployment/replicated + - service/replicated +versionLabel: 1.0.0 +# and this comment + +global: + replicated: + licenseID: online-license-id + channelName: online-channel-name + customerName: Online Customer Name + customerEmail: online-customer@example.com + licenseType: dev + dockerconfigjson: bm90LWEtZG9ja2VyLWNvbmZpZy1qc29uCg== + licenseFields: + expires_at: + name: expires_at + title: Expiration + description: License Expiration + value: "" + valueType: String + signature: + v1: nwZmD/sMFzKKxkd7JaAcKU/2uBE5m23w7+8xqLMXjUturMVCF5cF66EVMAibb2nHOqytie+N35GYSwIeTd16PKwbFBDd12c2E5M9COWwjVRcVTz4OnNWmHv9PEqZIbXhvfCLlyJ/aY3zV9Pno1VLFcYxGMrBugncEo4ecHkEbaVp3VLS4wn8EykAC1byvYBshzEXppYYd3c6a9cNw50Z6inI/IaKVxIForuz+Yn5uRAsjRyCY2auBCMeHMhY+CQ+4Vl5WtGjuJuE1g7t8AVZqt2JDBgDuxZAZX/JGncfzUaaDl87athMTtBKnFkTnCl34UXPkhsgM0LC4YoUiyKYjQ== +some: value +# and this comment as well +`, + }, + want: map[string]string{ + "replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "replicated/values.yaml": `# preserve this comment + +license: online-license +appName: online-app-name +channelID: online-channel-id +channelName: online-channel-name +channelSequence: 2 +releaseCreatedAt: "2023-10-02T00:00:00Z" +releaseNotes: override my release notes +releaseSequence: 1 +statusInformers: + - deployment/replicated + - service/replicated +versionLabel: 1.0.0 +# and this comment + +global: + replicated: + licenseID: online-license-id + channelName: online-channel-name + customerName: Online Customer Name + customerEmail: online-customer@example.com + licenseType: dev + dockerconfigjson: bm90LWEtZG9ja2VyLWNvbmZpZy1qc29uCg== + licenseFields: + expires_at: + name: expires_at + title: Expiration + description: License Expiration + value: "" + valueType: String + signature: + v1: nwZmD/sMFzKKxkd7JaAcKU/2uBE5m23w7+8xqLMXjUturMVCF5cF66EVMAibb2nHOqytie+N35GYSwIeTd16PKwbFBDd12c2E5M9COWwjVRcVTz4OnNWmHv9PEqZIbXhvfCLlyJ/aY3zV9Pno1VLFcYxGMrBugncEo4ecHkEbaVp3VLS4wn8EykAC1byvYBshzEXppYYd3c6a9cNw50Z6inI/IaKVxIForuz+Yn5uRAsjRyCY2auBCMeHMhY+CQ+4Vl5WtGjuJuE1g7t8AVZqt2JDBgDuxZAZX/JGncfzUaaDl87athMTtBKnFkTnCl34UXPkhsgM0LC4YoUiyKYjQ== +some: value +# and this comment as well + +airgap: false +appID: app-id +replicatedID: kotsadm-id +userAgent: KOTS/v0.0.0-unknown +`, + }, + wantErr: false, + }) + + tests = append(tests, Test{ + name: "airgap - a standalone replicated chart", + isAirgap: true, + chartContent: map[string]string{ + "replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "replicated/values.yaml": `# preserve this comment + +license: "" +appName: app-name +channelID: channel-id +channelName: channel-name +channelSequence: 2 +releaseCreatedAt: "2023-10-02T00:00:00Z" +releaseNotes: override my release notes +releaseSequence: 1 +statusInformers: + - deployment/replicated + - service/replicated +versionLabel: 1.0.0 +# and this comment + +some: value +# and this comment as well +`, + }, + want: map[string]string{ + "replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "replicated/values.yaml": `# preserve this comment + +license: | + apiVersion: kots.io/v1beta1 + kind: License + metadata: + creationTimestamp: null + name: kots-license + spec: + appSlug: app-slug + channelName: channel-name + customerEmail: customer@example.com + customerName: Customer Name + endpoint: https://replicated.app + entitlements: + license-field: + description: This is a license field + signature: {} + title: License Field + value: license-field-value + valueType: string + licenseID: license-id + licenseType: dev + signature: "" + status: {} +appName: app-name +channelID: channel-id +channelName: channel-name +channelSequence: 2 +releaseCreatedAt: "2023-10-02T00:00:00Z" +releaseNotes: override my release notes +releaseSequence: 1 +statusInformers: + - deployment/replicated + - service/replicated +versionLabel: 1.0.0 +# and this comment + +some: value +# and this comment as well + +airgap: true +appID: app-id +replicatedID: kotsadm-id +userAgent: KOTS/v0.0.0-unknown +global: + replicated: + channelName: channel-name + customerEmail: customer@example.com + customerName: Customer Name + dockerconfigjson: eyJhdXRocyI6eyJjdXN0b20ucHJveHkuY29tIjp7ImF1dGgiOiJiR2xqWlc1elpTMXBaRHBzYVdObGJuTmxMV2xrIn0sImN1c3RvbS5yZWdpc3RyeS5jb20iOnsiYXV0aCI6ImJHbGpaVzV6WlMxcFpEcHNhV05sYm5ObExXbGsifX19 + licenseFields: + license-field: + description: This is a license field + name: license-field + signature: {} + title: License Field + value: license-field-value + valueType: string + licenseID: license-id + licenseType: dev +`, + }, + wantErr: false, + }) + + tests = append(tests, Test{ + name: "online - a guestbook chart with the replicated subchart", + isAirgap: false, + chartContent: map[string]string{ + "guestbook/Chart.yaml": `apiVersion: v2 +name: guestbook +version: 1.16.0 +description: A Guestbook Chart +`, + "guestbook/values.yaml": fmt.Sprintf(`affinity: {} + +# use this value to override the chart name +fullnameOverride: "" + +# use this value to set the image pull policy +image: + pullPolicy: IfNotPresent +%s: + license: online-license + appName: online-app-name + channelID: online-channel-id + channelName: online-channel-name + channelSequence: 2 + releaseCreatedAt: "2023-10-02T00:00:00Z" + releaseNotes: override my release notes + releaseSequence: 1 + statusInformers: + - deployment/replicated + - service/replicated + versionLabel: 1.0.0 +global: + replicated: + licenseID: online-license-id + channelName: online-channel-name + customerName: Online Customer Name + customerEmail: online-customer@example.com + licenseType: dev + dockerconfigjson: bm90LWEtZG9ja2VyLWNvbmZpZy1qc29uCg== + licenseFields: + expires_at: + name: expires_at + title: Expiration + description: License Expiration + value: "" + valueType: String + signature: + v1: nwZmD/sMFzKKxkd7JaAcKU/2uBE5m23w7+8xqLMXjUturMVCF5cF66EVMAibb2nHOqytie+N35GYSwIeTd16PKwbFBDd12c2E5M9COWwjVRcVTz4OnNWmHv9PEqZIbXhvfCLlyJ/aY3zV9Pno1VLFcYxGMrBugncEo4ecHkEbaVp3VLS4wn8EykAC1byvYBshzEXppYYd3c6a9cNw50Z6inI/IaKVxIForuz+Yn5uRAsjRyCY2auBCMeHMhY+CQ+4Vl5WtGjuJuE1g7t8AVZqt2JDBgDuxZAZX/JGncfzUaaDl87athMTtBKnFkTnCl34UXPkhsgM0LC4YoUiyKYjQ== +`, chartName), + "guestbook/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "guestbook/charts/replicated/values.yaml": `# preserve this comment + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + want: map[string]string{ + "guestbook/Chart.yaml": `apiVersion: v2 +name: guestbook +version: 1.16.0 +description: A Guestbook Chart +`, + "guestbook/values.yaml": fmt.Sprintf(`affinity: {} +# use this value to override the chart name +fullnameOverride: "" +# use this value to set the image pull policy +image: + pullPolicy: IfNotPresent +%s: + license: online-license + appName: online-app-name + channelID: online-channel-id + channelName: online-channel-name + channelSequence: 2 + releaseCreatedAt: "2023-10-02T00:00:00Z" + releaseNotes: override my release notes + releaseSequence: 1 + statusInformers: + - deployment/replicated + - service/replicated + versionLabel: 1.0.0 + airgap: false + appID: app-id + replicatedID: kotsadm-id + userAgent: KOTS/v0.0.0-unknown +global: + replicated: + licenseID: online-license-id + channelName: online-channel-name + customerName: Online Customer Name + customerEmail: online-customer@example.com + licenseType: dev + dockerconfigjson: bm90LWEtZG9ja2VyLWNvbmZpZy1qc29uCg== + licenseFields: + expires_at: + name: expires_at + title: Expiration + description: License Expiration + value: "" + valueType: String + signature: + v1: nwZmD/sMFzKKxkd7JaAcKU/2uBE5m23w7+8xqLMXjUturMVCF5cF66EVMAibb2nHOqytie+N35GYSwIeTd16PKwbFBDd12c2E5M9COWwjVRcVTz4OnNWmHv9PEqZIbXhvfCLlyJ/aY3zV9Pno1VLFcYxGMrBugncEo4ecHkEbaVp3VLS4wn8EykAC1byvYBshzEXppYYd3c6a9cNw50Z6inI/IaKVxIForuz+Yn5uRAsjRyCY2auBCMeHMhY+CQ+4Vl5WtGjuJuE1g7t8AVZqt2JDBgDuxZAZX/JGncfzUaaDl87athMTtBKnFkTnCl34UXPkhsgM0LC4YoUiyKYjQ== +`, chartName), + "guestbook/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "guestbook/charts/replicated/values.yaml": `# preserve this comment + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + wantErr: false, + }) + + tests = append(tests, Test{ + name: "airgap - a guestbook chart with the replicated subchart", + isAirgap: true, + chartContent: map[string]string{ + "guestbook/Chart.yaml": `apiVersion: v2 +name: guestbook +version: 1.16.0 +description: A Guestbook Chart +`, + "guestbook/values.yaml": fmt.Sprintf(`affinity: {} + +# use this value to override the chart name +fullnameOverride: "" + +# use this value to set the image pull policy +image: + pullPolicy: IfNotPresent +%s: + appName: app-name + channelID: channel-id + channelName: channel-name + channelSequence: 2 + releaseCreatedAt: "2023-10-02T00:00:00Z" + releaseNotes: override my release notes + releaseSequence: 1 + statusInformers: + - deployment/replicated + - service/replicated + versionLabel: 1.0.0 +`, chartName), + "guestbook/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "guestbook/charts/replicated/values.yaml": `# preserve this comment + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + want: map[string]string{ + "guestbook/Chart.yaml": `apiVersion: v2 +name: guestbook +version: 1.16.0 +description: A Guestbook Chart +`, + "guestbook/values.yaml": fmt.Sprintf(`affinity: {} +# use this value to override the chart name +fullnameOverride: "" +# use this value to set the image pull policy +image: + pullPolicy: IfNotPresent +%s: + appName: app-name + channelID: channel-id + channelName: channel-name + channelSequence: 2 + releaseCreatedAt: "2023-10-02T00:00:00Z" + releaseNotes: override my release notes + releaseSequence: 1 + statusInformers: + - deployment/replicated + - service/replicated + versionLabel: 1.0.0 + airgap: true + appID: app-id + license: | + apiVersion: kots.io/v1beta1 + kind: License + metadata: + creationTimestamp: null + name: kots-license + spec: + appSlug: app-slug + channelName: channel-name + customerEmail: customer@example.com + customerName: Customer Name + endpoint: https://replicated.app + entitlements: + license-field: + description: This is a license field + signature: {} + title: License Field + value: license-field-value + valueType: string + licenseID: license-id + licenseType: dev + signature: "" + status: {} + replicatedID: kotsadm-id + userAgent: KOTS/v0.0.0-unknown +global: + replicated: + channelName: channel-name + customerEmail: customer@example.com + customerName: Customer Name + dockerconfigjson: eyJhdXRocyI6eyJjdXN0b20ucHJveHkuY29tIjp7ImF1dGgiOiJiR2xqWlc1elpTMXBaRHBzYVdObGJuTmxMV2xrIn0sImN1c3RvbS5yZWdpc3RyeS5jb20iOnsiYXV0aCI6ImJHbGpaVzV6WlMxcFpEcHNhV05sYm5ObExXbGsifX19 + licenseFields: + license-field: + description: This is a license field + name: license-field + signature: {} + title: License Field + value: license-field-value + valueType: string + licenseID: license-id + licenseType: dev +`, chartName), + "guestbook/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "guestbook/charts/replicated/values.yaml": `# preserve this comment + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + wantErr: false, + }) + + tests = append(tests, Test{ + name: "online - a redis chart with the replicated subchart and predefined replicated and global values", + isAirgap: false, + chartContent: map[string]string{ + "redis/Chart.yaml": `apiVersion: v1 +name: redis +version: 5.0.7 +description: A Redis Chart +`, + "redis/values.yaml": fmt.Sprintf(`## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + # imageRegistry: myRegistryName + # imagePullSecrets: + # - myRegistryKeySecretName + # storageClass: myStorageClass + redis: {} + replicated: + some: value + licenseID: online-license-id + channelName: online-channel-name + customerName: Online Customer Name + customerEmail: online-customer@example.com + licenseType: dev + dockerconfigjson: bm90LWEtZG9ja2VyLWNvbmZpZy1qc29uCg== + licenseFields: + expires_at: + name: expires_at + title: Expiration + description: License Expiration + value: "" + valueType: String + signature: + v1: nwZmD/sMFzKKxkd7JaAcKU/2uBE5m23w7+8xqLMXjUturMVCF5cF66EVMAibb2nHOqytie+N35GYSwIeTd16PKwbFBDd12c2E5M9COWwjVRcVTz4OnNWmHv9PEqZIbXhvfCLlyJ/aY3zV9Pno1VLFcYxGMrBugncEo4ecHkEbaVp3VLS4wn8EykAC1byvYBshzEXppYYd3c6a9cNw50Z6inI/IaKVxIForuz+Yn5uRAsjRyCY2auBCMeHMhY+CQ+4Vl5WtGjuJuE1g7t8AVZqt2JDBgDuxZAZX/JGncfzUaaDl87athMTtBKnFkTnCl34UXPkhsgM0LC4YoUiyKYjQ== + +# values related to the replicated subchart +%s: + some: value + license: online-license + appName: online-app-name + channelID: online-channel-id + channelName: online-channel-name + channelSequence: 2 + releaseCreatedAt: "2023-10-02T00:00:00Z" + releaseNotes: override my release notes + releaseSequence: 1 + statusInformers: + - deployment/replicated + - service/replicated + versionLabel: 1.0.0 +`, chartName), + "redis/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "redis/charts/replicated/values.yaml": `# preserve this comment + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + want: map[string]string{ + "redis/Chart.yaml": `apiVersion: v1 +name: redis +version: 5.0.7 +description: A Redis Chart +`, + "redis/values.yaml": fmt.Sprintf(`## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + # imageRegistry: myRegistryName + # imagePullSecrets: + # - myRegistryKeySecretName + # storageClass: myStorageClass + redis: {} + replicated: + some: value + licenseID: online-license-id + channelName: online-channel-name + customerName: Online Customer Name + customerEmail: online-customer@example.com + licenseType: dev + dockerconfigjson: bm90LWEtZG9ja2VyLWNvbmZpZy1qc29uCg== + licenseFields: + expires_at: + name: expires_at + title: Expiration + description: License Expiration + value: "" + valueType: String + signature: + v1: nwZmD/sMFzKKxkd7JaAcKU/2uBE5m23w7+8xqLMXjUturMVCF5cF66EVMAibb2nHOqytie+N35GYSwIeTd16PKwbFBDd12c2E5M9COWwjVRcVTz4OnNWmHv9PEqZIbXhvfCLlyJ/aY3zV9Pno1VLFcYxGMrBugncEo4ecHkEbaVp3VLS4wn8EykAC1byvYBshzEXppYYd3c6a9cNw50Z6inI/IaKVxIForuz+Yn5uRAsjRyCY2auBCMeHMhY+CQ+4Vl5WtGjuJuE1g7t8AVZqt2JDBgDuxZAZX/JGncfzUaaDl87athMTtBKnFkTnCl34UXPkhsgM0LC4YoUiyKYjQ== +# values related to the replicated subchart +%s: + some: value + license: online-license + appName: online-app-name + channelID: online-channel-id + channelName: online-channel-name + channelSequence: 2 + releaseCreatedAt: "2023-10-02T00:00:00Z" + releaseNotes: override my release notes + releaseSequence: 1 + statusInformers: + - deployment/replicated + - service/replicated + versionLabel: 1.0.0 + airgap: false + appID: app-id + replicatedID: kotsadm-id + userAgent: KOTS/v0.0.0-unknown +`, chartName), + "redis/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "redis/charts/replicated/values.yaml": `# preserve this comment + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + wantErr: false, + }) + + tests = append(tests, Test{ + name: "airgap - a redis chart with the replicated subchart and predefined replicated and global values", + isAirgap: true, + chartContent: map[string]string{ + "redis/Chart.yaml": `apiVersion: v1 +name: redis +version: 5.0.7 +description: A Redis Chart +`, + "redis/values.yaml": fmt.Sprintf(`## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + # imageRegistry: myRegistryName + # imagePullSecrets: + # - myRegistryKeySecretName + # storageClass: myStorageClass + redis: {} + replicated: + some: value + +# values related to the replicated subchart +%s: + some: value + appName: app-name + channelID: channel-id + channelName: channel-name + channelSequence: 2 + releaseCreatedAt: "2023-10-02T00:00:00Z" + releaseNotes: override my release notes + releaseSequence: 1 + statusInformers: + - deployment/replicated + - service/replicated + versionLabel: 1.0.0 +`, chartName), + "redis/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "redis/charts/replicated/values.yaml": `# preserve this comment + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + want: map[string]string{ + "redis/Chart.yaml": `apiVersion: v1 +name: redis +version: 5.0.7 +description: A Redis Chart +`, + "redis/values.yaml": fmt.Sprintf(`## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + # imageRegistry: myRegistryName + # imagePullSecrets: + # - myRegistryKeySecretName + # storageClass: myStorageClass + redis: {} + replicated: + some: value + channelName: channel-name + customerEmail: customer@example.com + customerName: Customer Name + dockerconfigjson: eyJhdXRocyI6eyJjdXN0b20ucHJveHkuY29tIjp7ImF1dGgiOiJiR2xqWlc1elpTMXBaRHBzYVdObGJuTmxMV2xrIn0sImN1c3RvbS5yZWdpc3RyeS5jb20iOnsiYXV0aCI6ImJHbGpaVzV6WlMxcFpEcHNhV05sYm5ObExXbGsifX19 + licenseFields: + license-field: + description: This is a license field + name: license-field + signature: {} + title: License Field + value: license-field-value + valueType: string + licenseID: license-id + licenseType: dev +# values related to the replicated subchart +%s: + some: value + appName: app-name + channelID: channel-id + channelName: channel-name + channelSequence: 2 + releaseCreatedAt: "2023-10-02T00:00:00Z" + releaseNotes: override my release notes + releaseSequence: 1 + statusInformers: + - deployment/replicated + - service/replicated + versionLabel: 1.0.0 + airgap: true + appID: app-id + license: | + apiVersion: kots.io/v1beta1 + kind: License + metadata: + creationTimestamp: null + name: kots-license + spec: + appSlug: app-slug + channelName: channel-name + customerEmail: customer@example.com + customerName: Customer Name + endpoint: https://replicated.app + entitlements: + license-field: + description: This is a license field + signature: {} + title: License Field + value: license-field-value + valueType: string + licenseID: license-id + licenseType: dev + signature: "" + status: {} + replicatedID: kotsadm-id + userAgent: KOTS/v0.0.0-unknown +`, chartName), + "redis/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "redis/charts/replicated/values.yaml": `# preserve this comment + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + wantErr: false, + }) + + tests = append(tests, Test{ + name: "online - a postgresql chart with replicated as subsubchart", + isAirgap: false, + chartContent: map[string]string{ + "postgresql/Chart.yaml": `apiVersion: v2 +name: postgresql +version: 11.6.0 +description: A Postgresql Chart +`, + "postgresql/values.yaml": `extraEnv: [] + +# override global values here +global: + postgresql: {} + +# additional values can be added here +`, + "postgresql/charts/guestbook/Chart.yaml": `apiVersion: v2 +name: guestbook +version: 1.16.0 +description: A Guestbook Chart +`, + "postgresql/charts/guestbook/values.yaml": `affinity: {} + +# use this value to override the chart name +fullnameOverride: "" + +# use this value to set the image pull policy +image: + pullPolicy: IfNotPresent +`, + "postgresql/charts/guestbook/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "postgresql/charts/guestbook/charts/replicated/values.yaml": `# this file should NOT change + +# global values should NOT be updated +global: + keep: this-value + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + want: map[string]string{ + "postgresql/Chart.yaml": `apiVersion: v2 +name: postgresql +version: 11.6.0 +description: A Postgresql Chart +`, + "postgresql/values.yaml": `extraEnv: [] + +# override global values here +global: + postgresql: {} + +# additional values can be added here +`, + "postgresql/charts/guestbook/Chart.yaml": `apiVersion: v2 +name: guestbook +version: 1.16.0 +description: A Guestbook Chart +`, + "postgresql/charts/guestbook/values.yaml": `affinity: {} + +# use this value to override the chart name +fullnameOverride: "" + +# use this value to set the image pull policy +image: + pullPolicy: IfNotPresent +`, + "postgresql/charts/guestbook/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "postgresql/charts/guestbook/charts/replicated/values.yaml": `# this file should NOT change + +# global values should NOT be updated +global: + keep: this-value + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + }) + + tests = append(tests, Test{ + name: "airgap - a postgresql chart with replicated as subsubchart", + isAirgap: true, + chartContent: map[string]string{ + "postgresql/Chart.yaml": `apiVersion: v2 +name: postgresql +version: 11.6.0 +description: A Postgresql Chart +`, + "postgresql/values.yaml": `extraEnv: [] + +# override global values here +global: + postgresql: {} + +# additional values can be added here +`, + "postgresql/charts/guestbook/Chart.yaml": `apiVersion: v2 +name: guestbook +version: 1.16.0 +description: A Guestbook Chart +`, + "postgresql/charts/guestbook/values.yaml": `affinity: {} + +# use this value to override the chart name +fullnameOverride: "" + +# use this value to set the image pull policy +image: + pullPolicy: IfNotPresent +`, + "postgresql/charts/guestbook/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "postgresql/charts/guestbook/charts/replicated/values.yaml": `# this file should NOT change + +# global values should NOT be updated +global: + keep: this-value + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + want: map[string]string{ + "postgresql/Chart.yaml": `apiVersion: v2 +name: postgresql +version: 11.6.0 +description: A Postgresql Chart +`, + "postgresql/values.yaml": `extraEnv: [] + +# override global values here +global: + postgresql: {} + +# additional values can be added here +`, + "postgresql/charts/guestbook/Chart.yaml": `apiVersion: v2 +name: guestbook +version: 1.16.0 +description: A Guestbook Chart +`, + "postgresql/charts/guestbook/values.yaml": `affinity: {} + +# use this value to override the chart name +fullnameOverride: "" + +# use this value to set the image pull policy +image: + pullPolicy: IfNotPresent +`, + "postgresql/charts/guestbook/charts/replicated/Chart.yaml": fmt.Sprintf(`apiVersion: v1 +name: %s +version: 1.0.0 +description: A Replicated Chart +`, chartName), + "postgresql/charts/guestbook/charts/replicated/values.yaml": `# this file should NOT change + +# global values should NOT be updated +global: + keep: this-value + +channelName: keep-this-channel-name +# and this comment + +some: value +# and this comment as well +`, + }, + }) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + chartBytes, err := util.FilesToTGZ(tt.chartContent) + require.NoError(t, err) + + upstream := &types.Upstream{ + License: &kotsv1beta1.License{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "License", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kots-license", + }, + Spec: kotsv1beta1.LicenseSpec{ + LicenseID: "license-id", + AppSlug: "app-slug", + ChannelName: "channel-name", + Endpoint: "https://replicated.app", + Entitlements: map[string]kotsv1beta1.EntitlementField{ + "license-field": { + Title: "License Field", + Description: "This is a license field", + ValueType: "string", + Value: kotsv1beta1.EntitlementValue{ + Type: kotsv1beta1.String, + StrVal: "license-field-value", + }, + }, + }, + CustomerEmail: "customer@example.com", + CustomerName: "Customer Name", + LicenseType: "dev", + Signature: []byte{}, + }, + }, + ReplicatedRegistryDomain: "custom.registry.com", + ReplicatedProxyDomain: "custom.proxy.com", + ReplicatedChartNames: testReplicatedChartNames, + } + + writeOptions := types.WriteOptions{ + KotsadmID: "kotsadm-id", + AppID: "app-id", + IsAirgap: tt.isAirgap, + } + + got, err := configureChart(chartBytes, upstream, writeOptions) + if (err != nil) != tt.wantErr { + t.Errorf("configureChart() error = %v, wantErr %v", err, tt.wantErr) + return + } + + gotFiles, err := util.TGZToFiles(got) + require.NoError(t, err) + + for filename, wantContent := range tt.want { + gotContent := gotFiles[filename] + if gotContent != wantContent { + t.Errorf("configureChart() %s: %v", filename, diffString(gotContent, wantContent)) + } + } + }) + } +} + +func diffString(got, want string) string { + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(got), + B: difflib.SplitLines(want), + FromFile: "Got", + ToFile: "Want", + Context: 1, + } + diffStr, _ := difflib.GetUnifiedDiffString(diff) + return fmt.Sprintf("got:\n%s \n\nwant:\n%s \n\ndiff:\n%s", got, want, diffStr) +} diff --git a/pkg/upstream/replicated.go b/pkg/upstream/replicated.go index 78529b0349..2e4b09a65b 100644 --- a/pkg/upstream/replicated.go +++ b/pkg/upstream/replicated.go @@ -55,6 +55,7 @@ type Release struct { ReleasedAt *time.Time ReplicatedRegistryDomain string ReplicatedProxyDomain string + ReplicatedChartNames []string Manifests map[string][]byte } @@ -125,6 +126,7 @@ func downloadReplicated( isRequired bool, replicatedRegistryDomain string, replicatedProxyDomain string, + replicatedChartNames []string, appSlug string, appSequence int64, isAirgap bool, @@ -136,7 +138,7 @@ func downloadReplicated( var release *Release if localPath != "" { - parsedLocalRelease, err := readReplicatedAppFromLocalPath(localPath, updateCursor, versionLabel, isRequired, replicatedRegistryDomain, replicatedProxyDomain) + parsedLocalRelease, err := readReplicatedAppFromLocalPath(localPath, updateCursor, versionLabel, isRequired, replicatedRegistryDomain, replicatedProxyDomain, replicatedChartNames) if err != nil { return nil, errors.Wrap(err, "failed to read replicated app from local path") } @@ -287,6 +289,7 @@ func downloadReplicated( Files: files, Type: "replicated", UpdateCursor: release.UpdateCursor.Cursor, + License: license, ChannelID: channelID, ChannelName: channelName, VersionLabel: release.VersionLabel, @@ -295,12 +298,13 @@ func downloadReplicated( ReleasedAt: release.ReleasedAt, ReplicatedRegistryDomain: release.ReplicatedRegistryDomain, ReplicatedProxyDomain: release.ReplicatedProxyDomain, + ReplicatedChartNames: release.ReplicatedChartNames, } return upstream, nil } -func readReplicatedAppFromLocalPath(localPath string, localCursor replicatedapp.ReplicatedCursor, versionLabel string, isRequired bool, replicatedRegistryDomain string, replicatedProxyDomain string) (*Release, error) { +func readReplicatedAppFromLocalPath(localPath string, localCursor replicatedapp.ReplicatedCursor, versionLabel string, isRequired bool, replicatedRegistryDomain string, replicatedProxyDomain string, replicatedChartNames []string) (*Release, error) { release := Release{ Manifests: make(map[string][]byte), UpdateCursor: localCursor, @@ -308,6 +312,7 @@ func readReplicatedAppFromLocalPath(localPath string, localCursor replicatedapp. IsRequired: isRequired, ReplicatedRegistryDomain: replicatedRegistryDomain, ReplicatedProxyDomain: replicatedProxyDomain, + ReplicatedChartNames: replicatedChartNames, } err := filepath.Walk(localPath, @@ -370,6 +375,7 @@ func downloadReplicatedApp(replicatedUpstream *replicatedapp.ReplicatedUpstream, releasedAtStr := getResp.Header.Get("X-Replicated-ReleasedAt") replicatedRegistryDomain := getResp.Header.Get("X-Replicated-ReplicatedRegistryDomain") replicatedProxyDomain := getResp.Header.Get("X-Replicated-ReplicatedProxyDomain") + replicatedChartNamesStr := getResp.Header.Get("X-Replicated-ReplicatedChartNames") var releasedAt *time.Time r, err := time.Parse(time.RFC3339, releasedAtStr) @@ -379,6 +385,11 @@ func downloadReplicatedApp(replicatedUpstream *replicatedapp.ReplicatedUpstream, isRequired, _ := strconv.ParseBool(isRequiredStr) + var replicatedChartNames []string + if replicatedChartNamesStr != "" { + replicatedChartNames = strings.Split(replicatedChartNamesStr, ",") + } + gzf, err := gzip.NewReader(getResp.Body) if err != nil { return nil, errors.Wrap(err, "failed to create new gzip reader") @@ -396,6 +407,7 @@ func downloadReplicatedApp(replicatedUpstream *replicatedapp.ReplicatedUpstream, ReleasedAt: releasedAt, ReplicatedRegistryDomain: replicatedRegistryDomain, ReplicatedProxyDomain: replicatedProxyDomain, + ReplicatedChartNames: replicatedChartNames, // NOTE: release notes come from Application spec } tarReader := tar.NewReader(gzf) diff --git a/pkg/upstream/types/types.go b/pkg/upstream/types/types.go index ae9df8ee7a..6f24527b6f 100644 --- a/pkg/upstream/types/types.go +++ b/pkg/upstream/types/types.go @@ -27,6 +27,7 @@ type Upstream struct { Type string Files []UpstreamFile UpdateCursor string + License *kotsv1beta1.License ChannelID string ChannelName string VersionLabel string @@ -35,6 +36,7 @@ type Upstream struct { ReleasedAt *time.Time ReplicatedRegistryDomain string ReplicatedProxyDomain string + ReplicatedChartNames []string EncryptionKey string } @@ -67,6 +69,9 @@ type WriteOptions struct { NoProxyEnvValue string IsMinimalRBAC bool AdditionalNamespaces []string + IsAirgap bool + KotsadmID string + AppID string // This should be set to true when updating due to license sync, config update, registry settings update. // and should be false when it's an upstream update. // When true, the channel name in Installation yaml will not be changed. @@ -98,6 +103,7 @@ type FetchOptions struct { CurrentVersionIsRequired bool CurrentReplicatedRegistryDomain string CurrentReplicatedProxyDomain string + CurrentReplicatedChartNames []string ChannelChanged bool AppSlug string AppSequence int64 diff --git a/pkg/upstream/write.go b/pkg/upstream/write.go index 0d569f8cef..2c8d8f8fb9 100644 --- a/pkg/upstream/write.go +++ b/pkg/upstream/write.go @@ -3,11 +3,11 @@ package upstream import ( "bytes" "encoding/base64" - "io/ioutil" "os" "path" "github.com/pkg/errors" + "github.com/replicatedhq/kots/pkg/archives" "github.com/replicatedhq/kots/pkg/crypto" "github.com/replicatedhq/kots/pkg/kotsutil" "github.com/replicatedhq/kots/pkg/upstream/types" @@ -39,7 +39,7 @@ func WriteUpstream(u *types.Upstream, options types.WriteOptions) error { if err == nil { _, err = os.Stat(path.Join(renderDir, "userdata", "installation.yaml")) if err == nil { - c, err := ioutil.ReadFile(path.Join(renderDir, "userdata", "installation.yaml")) + c, err := os.ReadFile(path.Join(renderDir, "userdata", "installation.yaml")) if err != nil { return errors.Wrap(err, "failed to read existing installation") } @@ -100,7 +100,16 @@ func WriteUpstream(u *types.Upstream, options types.WriteOptions) error { u.Files[i] = file } - if err := ioutil.WriteFile(fileRenderPath, file.Content, 0644); err != nil { + if archives.IsTGZ(file.Content) { + updatedContent, err := configureChart(file.Content, u, options) + if err != nil { + return errors.Wrap(err, "failed to configure replicated sdk") + } + file.Content = updatedContent + u.Files[i] = file + } + + if err := os.WriteFile(fileRenderPath, file.Content, 0644); err != nil { return errors.Wrap(err, "failed to write upstream file") } } @@ -131,6 +140,7 @@ func WriteUpstream(u *types.Upstream, options types.WriteOptions) error { ReleaseNotes: u.ReleaseNotes, ReplicatedRegistryDomain: u.ReplicatedRegistryDomain, ReplicatedProxyDomain: u.ReplicatedProxyDomain, + ReplicatedChartNames: u.ReplicatedChartNames, EncryptionKey: encryptionKey, }, } @@ -147,7 +157,7 @@ func WriteUpstream(u *types.Upstream, options types.WriteOptions) error { } installationBytes := kotsutil.MustMarshalInstallation(&installation) - err = ioutil.WriteFile(path.Join(renderDir, "userdata", "installation.yaml"), installationBytes, 0644) + err = os.WriteFile(path.Join(renderDir, "userdata", "installation.yaml"), installationBytes, 0644) if err != nil { return errors.Wrap(err, "failed to write installation") } diff --git a/pkg/util/file.go b/pkg/util/file.go new file mode 100644 index 0000000000..5583a2a3ed --- /dev/null +++ b/pkg/util/file.go @@ -0,0 +1,74 @@ +package util + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "strings" + + "github.com/pkg/errors" +) + +func FilesToTGZ(files map[string]string) ([]byte, error) { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + for path, content := range files { + header := &tar.Header{ + Name: path, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(header); err != nil { + return nil, errors.Wrapf(err, "failed to write tar header for %s", path) + } + _, err := io.Copy(tw, strings.NewReader(content)) + if err != nil { + return nil, errors.Wrapf(err, "failed to write %s to tar", path) + } + } + + if err := tw.Close(); err != nil { + return nil, errors.Wrap(err, "failed to close tar writer") + } + + if err := gw.Close(); err != nil { + return nil, errors.Wrap(err, "failed to close gzip writer") + } + + return buf.Bytes(), nil +} + +func TGZToFiles(tgzBytes []byte) (map[string]string, error) { + files := make(map[string]string) + + gr, err := gzip.NewReader(bytes.NewReader(tgzBytes)) + if err != nil { + return nil, errors.Wrap(err, "failed to create gzip reader") + } + defer gr.Close() + + tr := tar.NewReader(gr) + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if header.Typeflag == tar.TypeReg { + var contentBuf bytes.Buffer + if _, err := io.Copy(&contentBuf, tr); err != nil { + return nil, errors.Wrap(err, "failed to copy tar data") + } + files[header.Name] = contentBuf.String() + } + } + + return files, nil +} diff --git a/pkg/util/file_test.go b/pkg/util/file_test.go new file mode 100644 index 0000000000..31e188b3c9 --- /dev/null +++ b/pkg/util/file_test.go @@ -0,0 +1,51 @@ +package util + +import ( + "reflect" + "testing" +) + +func Test_filesToTGZAndTGZToFiles(t *testing.T) { + tests := []struct { + name string + files map[string]string + }{ + { + name: "SingleFile", + files: map[string]string{ + "file.txt": "File content", + }, + }, + { + name: "MultipleFiles", + files: map[string]string{ + "file1.txt": "File 1 content", + "file2.txt": "File 2 content", + }, + }, + { + name: "EmptyFiles", + files: map[string]string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tgzBytes, err := FilesToTGZ(tt.files) + if err != nil { + t.Errorf("FilesToTGZ() error = %v", err) + return + } + + actualFiles, err := TGZToFiles(tgzBytes) + if err != nil { + t.Errorf("TGZToFiles() error = %v", err) + return + } + + if !reflect.DeepEqual(actualFiles, tt.files) { + t.Errorf("filesToTGZAndTGZToFiles() = %v, want %v", actualFiles, tt.files) + } + }) + } +}