wip: configure sdk

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe
+	github.com/replicatedhq/kotskinds v0.0.0-20230925145827-c8f611d61fc1
 	github.com/replicatedhq/kurlkinds v1.3.6
 	github.com/replicatedhq/troubleshoot v0.72.1
 	github.com/replicatedhq/yaml/v3 v3.0.0-beta5-replicatedhq

go.sum

@@ -1527,6 +1527,10 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqn
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe h1:3AJInd06UxzqHmgy8+24CPsT2tYSE0zToJZyuX9q+MA=
 github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I=
+github.com/replicatedhq/kotskinds v0.0.0-20230921181252-176885c4c65d h1:P1PY8o/pvb9B+cCFepYUGhqzoAFYe9H0pz0SKq2GJR0=
+github.com/replicatedhq/kotskinds v0.0.0-20230921181252-176885c4c65d/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I=
+github.com/replicatedhq/kotskinds v0.0.0-20230925145827-c8f611d61fc1 h1:DqyT2B0ud9S1p78oTGV4mjtVvo3GqjyLbc17SS77kec=
+github.com/replicatedhq/kotskinds v0.0.0-20230925145827-c8f611d61fc1/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I=
 github.com/replicatedhq/kurlkinds v1.3.6 h1:/dhS32cSSZR4yS4vA8EquBvz+VgJCyTqBO9Xw+6eI4M=
 github.com/replicatedhq/kurlkinds v1.3.6/go.mod h1:c5+hoAkuARgftB2Ft3RCyWRZZPhL0clHEaw7XoGDAg4=
 github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851 h1:eRlNDHxGfVkPCRXbA4BfQJvt5DHjFiTtWy3R/t4djyY=

pkg/k8sutil/kotsadm.go

@@ -9,6 +9,7 @@ import (
 	types "github.com/replicatedhq/kots/pkg/k8sutil/types"
 	kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types"
 	"github.com/replicatedhq/kots/pkg/util"
+	"github.com/segmentio/ksuid"
 	corev1 "k8s.io/api/core/v1"
 	kuberneteserrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -81,11 +82,23 @@ func IsKotsadmClusterScoped(ctx context.Context, clientset kubernetes.Interface,
 	return false
 }
 
-func GetKotsadmIDConfigMap() (*corev1.ConfigMap, error) {
-	clientset, err := GetClientset()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get clientset")
+func GetKotsadmClusterID(clientset kubernetes.Interface) string {
+	var clusterID string
+	configMap, err := GetKotsadmIDConfigMap(clientset)
+	// if configmap is not found, generate a new guid and create a new configmap, if configmap is found, use the existing guid, otherwise generate
+	if err != nil && !kuberneteserrors.IsNotFound(err) {
+		clusterID = ksuid.New().String()
+	} else if configMap != nil {
+		clusterID = configMap.Data["id"]
+	} else {
+		// configmap is missing for some reason, recreate with new guid, this will appear as a new instance in the report
+		clusterID = ksuid.New().String()
+		CreateKotsadmIDConfigMap(clientset, clusterID)
 	}
+	return clusterID
+}
+
+func GetKotsadmIDConfigMap(clientset kubernetes.Interface) (*corev1.ConfigMap, error) {
 	namespace := util.PodNamespace
 	existingConfigmap, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), KotsadmIDConfigMapName, metav1.GetOptions{})
 	if err != nil && !kuberneteserrors.IsNotFound(err) {
@@ -96,12 +109,8 @@ func GetKotsadmIDConfigMap() (*corev1.ConfigMap, error) {
 	return existingConfigmap, nil
 }
 
-func CreateKotsadmIDConfigMap(kotsadmID string) error {
+func CreateKotsadmIDConfigMap(clientset kubernetes.Interface, kotsadmID string) error {
 	var err error = nil
-	clientset, err := GetClientset()
-	if err != nil {
-		return err
-	}
 	configmap := corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "v1",
@@ -136,11 +145,7 @@ func IsKotsadmIDConfigMapPresent() (bool, error) {
 	return true, nil
 }
 
-func UpdateKotsadmIDConfigMap(kotsadmID string) error {
-	clientset, err := GetClientset()
-	if err != nil {
-		return errors.Wrap(err, "failed to get clientset")
-	}
+func UpdateKotsadmIDConfigMap(clientset kubernetes.Interface, kotsadmID string) error {
 	namespace := util.PodNamespace
 	existingConfigMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), KotsadmIDConfigMapName, metav1.GetOptions{})
 	if err != nil && !kuberneteserrors.IsNotFound(err) {

pkg/k8sutil/kotsadm_test.go

@@ -0,0 +1,62 @@
+package k8sutil
+
+import (
+	"context"
+	"testing"
+
+	"gopkg.in/go-playground/assert.v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func TestGetKotsadmClusterID(t *testing.T) {
+
+	type args struct {
+		clientset kubernetes.Interface
+	}
+	tests := []struct {
+		name                  string
+		args                  args
+		want                  string
+		shouldCreateConfigMap bool
+	}{
+		{
+			name: "configmap exists",
+			args: args{
+				clientset: fake.NewSimpleClientset(&corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Name: KotsadmIDConfigMapName},
+					Data:       map[string]string{"id": "cluster-id"},
+				}),
+			},
+			want:                  "cluster-id",
+			shouldCreateConfigMap: false,
+		},
+		{
+			name: "configmap does not exist, should create",
+			args: args{
+				clientset: fake.NewSimpleClientset(),
+			},
+			want:                  "",
+			shouldCreateConfigMap: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := GetKotsadmClusterID(tt.args.clientset)
+			if tt.want != "" {
+				assert.Equal(t, tt.want, got)
+			} else {
+				// a random uuid is generated
+				assert.NotEqual(t, "", got)
+			}
+
+			if tt.shouldCreateConfigMap {
+				// should have created the configmap if it didn't exist
+				_, err := tt.args.clientset.CoreV1().ConfigMaps("").Get(context.TODO(), KotsadmIDConfigMapName, metav1.GetOptions{})
+				assert.Equal(t, nil, err)
+			}
+		})
+	}
+}

pkg/kotsutil/yaml.go

@@ -2,10 +2,13 @@ package kotsutil
 
 import (
 	"bytes"
+	"fmt"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/replicatedhq/kots/pkg/util"
 	yaml "github.com/replicatedhq/yaml/v3"
+	goyaml "gopkg.in/yaml.v3"
 	k8syaml "sigs.k8s.io/yaml"
 )
 
@@ -85,3 +88,159 @@ func removeNilFieldsFromMap(input map[string]interface{}) bool {
 
 	return removedItems
 }
+
+func MergeYAMLNodes(targetNodes []*goyaml.Node, overrideNodes []*goyaml.Node) []*goyaml.Node {
+	// Since inputs are arrays and not maps, we need to:
+	// 1. Copy all keys in targetNodes, overriding the ones that match from overrideNodes
+	// 2. Add all keys from overrideNodes that don't exist in targetNodes
+
+	if len(overrideNodes) == 0 {
+		return targetNodes
+	}
+
+	if len(targetNodes) == 0 {
+		return overrideNodes
+	}
+
+	// Special case where top level node is either a mapping node or an array
+	if len(targetNodes) == 1 && len(overrideNodes) == 1 {
+		if targetNodes[0].Kind == goyaml.MappingNode && overrideNodes[0].Kind == goyaml.MappingNode {
+			return []*goyaml.Node{
+				{
+					Kind:    goyaml.MappingNode,
+					Content: MergeYAMLNodes(targetNodes[0].Content, overrideNodes[0].Content),
+				},
+			}
+		}
+
+		if targetNodes[0].Value == overrideNodes[0].Value {
+			return overrideNodes
+		}
+
+		return append(targetNodes, overrideNodes...)
+	}
+
+	// 1. Copy all keys in targetNodes, overriding the ones that match from overrideNodes
+	newNodes := make([]*goyaml.Node, 0)
+	for i := 0; i < len(targetNodes)-1; i += 2 {
+		var additionalNode *goyaml.Node
+		for j := 0; j < len(overrideNodes)-1; j += 2 {
+			nodeNameI := targetNodes[i]
+			nodeValueI := targetNodes[i+1]
+
+			nodeNameJ := overrideNodes[j]
+			nodeValueJ := overrideNodes[j+1]
+
+			if nodeNameI.Value != nodeNameJ.Value {
+				continue
+			}
+
+			additionalNode = &goyaml.Node{
+				Kind:        nodeValueJ.Kind,
+				Tag:         nodeValueJ.Tag,
+				Line:        nodeValueJ.Line,
+				Style:       nodeValueJ.Style,
+				Anchor:      nodeValueJ.Anchor,
+				Value:       nodeValueJ.Value,
+				Alias:       nodeValueJ.Alias,
+				HeadComment: nodeValueJ.HeadComment,
+				LineComment: nodeValueJ.LineComment,
+				FootComment: nodeValueJ.FootComment,
+				Column:      nodeValueJ.Column,
+			}
+
+			if nodeValueI.Kind == goyaml.MappingNode && nodeValueJ.Kind == goyaml.MappingNode {
+				additionalNode.Content = MergeYAMLNodes(nodeValueI.Content, nodeValueJ.Content)
+			} else {
+				additionalNode.Content = nodeValueJ.Content
+			}
+
+			break
+		}
+
+		if additionalNode != nil {
+			newNodes = append(newNodes, targetNodes[i], additionalNode)
+		} else {
+			newNodes = append(newNodes, targetNodes[i], targetNodes[i+1])
+		}
+	}
+
+	// 2. Add all keys from overrideNodes that don't exist in targetNodes
+	for j := 0; j < len(overrideNodes)-1; j += 2 {
+		isFound := false
+		for i := 0; i < len(newNodes)-1; i += 2 {
+			nodeNameI := newNodes[i]
+			nodeValueI := newNodes[i+1]
+
+			additionalNodeName := overrideNodes[j]
+			additionalNodeValue := overrideNodes[j+1]
+
+			if nodeNameI.Value != additionalNodeName.Value {
+				continue
+			}
+
+			if nodeValueI.Kind == goyaml.MappingNode && additionalNodeValue.Kind == goyaml.MappingNode {
+				nodeValueI.Content = MergeYAMLNodes(nodeValueI.Content, additionalNodeValue.Content)
+			}
+
+			isFound = true
+			break
+		}
+
+		if !isFound {
+			newNodes = append(newNodes, overrideNodes[j], overrideNodes[j+1])
+		}
+	}
+
+	return newNodes
+}
+
+func ContentToDocNode(doc *goyaml.Node, nodes []*goyaml.Node) *goyaml.Node {
+	if doc == nil {
+		return &goyaml.Node{
+			Kind:    goyaml.DocumentNode,
+			Content: nodes,
+		}
+	}
+	return &goyaml.Node{
+		Kind:        doc.Kind,
+		Tag:         doc.Tag,
+		Line:        doc.Line,
+		Style:       doc.Style,
+		Anchor:      doc.Anchor,
+		Value:       doc.Value,
+		Alias:       doc.Alias,
+		HeadComment: doc.HeadComment,
+		LineComment: doc.LineComment,
+		FootComment: doc.FootComment,
+		Column:      doc.Column,
+		Content:     nodes,
+	}
+}
+
+func NodeToYAML(node *goyaml.Node) ([]byte, error) {
+	var renderedContents bytes.Buffer
+	yamlEncoder := goyaml.NewEncoder(&renderedContents)
+	yamlEncoder.SetIndent(2) // this may change indentations of the original values.yaml, but this matches out tests
+	err := yamlEncoder.Encode(node)
+	if err != nil {
+		return nil, errors.Wrap(err, "marshal")
+	}
+
+	return renderedContents.Bytes(), nil
+}
+
+// Handy functions for printing YAML nodes
+func PrintNodes(nodes []*goyaml.Node, i int) {
+	for _, n := range nodes {
+		PrintNode(n, i)
+	}
+}
+func PrintNode(n *goyaml.Node, i int) {
+	if n == nil {
+		return
+	}
+	indent := strings.Repeat(" ", i*2)
+	fmt.Printf("%stag:%v, style:%v, kind:%v, value:%v\n", indent, n.Tag, n.Style, n.Kind, n.Value)
+	PrintNodes(n.Content, i+1)
+}

pkg/reporting/app.go

@@ -138,13 +138,18 @@ func initFromDownstream() error {
 		return errors.Wrap(err, "failed to check configmap")
 	}
 
+	clientset, err := k8sutil.GetClientset()
+	if err != nil {
+		return errors.Wrap(err, "failed to get clientset")
+	}
+
 	if isKotsadmIDGenerated && !cmpExists {
 		kotsadmID := ksuid.New().String()
-		err = k8sutil.CreateKotsadmIDConfigMap(kotsadmID)
+		err = k8sutil.CreateKotsadmIDConfigMap(clientset, kotsadmID)
 	} else if !isKotsadmIDGenerated && !cmpExists {
-		err = k8sutil.CreateKotsadmIDConfigMap(clusterID)
+		err = k8sutil.CreateKotsadmIDConfigMap(clientset, clusterID)
 	} else if !isKotsadmIDGenerated && cmpExists {
-		err = k8sutil.UpdateKotsadmIDConfigMap(clusterID)
+		err = k8sutil.UpdateKotsadmIDConfigMap(clientset, clusterID)
 	} else {
 		// id exists and so as configmap, noop
 	}
@@ -181,16 +186,7 @@ func GetReportingInfo(appID string) *types.ReportingInfo {
 	if util.IsHelmManaged() {
 		r.ClusterID = clusterID
 	} else {
-		configMap, err := k8sutil.GetKotsadmIDConfigMap()
-		if err != nil {
-			r.ClusterID = ksuid.New().String()
-		} else if configMap != nil {
-			r.ClusterID = configMap.Data["id"]
-		} else {
-			// configmap is missing for some reason, recreate with new guid, this will appear as a new instance in the report
-			r.ClusterID = ksuid.New().String()
-			k8sutil.CreateKotsadmIDConfigMap(r.ClusterID)
-		}
+		r.ClusterID = k8sutil.GetKotsadmClusterID(clientset)
 
 		di, err := getDownstreamInfo(appID)
 		if err != nil {

pkg/rewrite/rewrite.go

@@ -74,6 +74,7 @@ func Rewrite(rewriteOptions RewriteOptions) error {
 		CurrentVersionIsRequired:        rewriteOptions.Installation.Spec.IsRequired,
 		CurrentReplicatedRegistryDomain: rewriteOptions.Installation.Spec.ReplicatedRegistryDomain,
 		CurrentReplicatedProxyDomain:    rewriteOptions.Installation.Spec.ReplicatedProxyDomain,
+		CurrentReplicatedChartNames:     rewriteOptions.Installation.Spec.ReplicatedChartNames,
 		EncryptionKey:                   rewriteOptions.Installation.Spec.EncryptionKey,
 		License:                         rewriteOptions.License,
 		AppSequence:                     rewriteOptions.AppSequence,

pkg/upstream/fetch.go

@@ -49,6 +49,7 @@ func downloadUpstream(upstreamURI string, fetchOptions *types.FetchOptions) (*ty
 			pickVersionIsRequired(fetchOptions),
 			pickReplicatedRegistryDomain(fetchOptions),
 			pickReplicatedProxyDomain(fetchOptions),
+			pickReplicatedChartNames(fetchOptions),
 			fetchOptions.AppSlug,
 			fetchOptions.AppSequence,
 			fetchOptions.Airgap != nil,
@@ -110,3 +111,11 @@ func pickCursor(fetchOptions *types.FetchOptions) replicatedapp.ReplicatedCursor
 		Cursor:      fetchOptions.CurrentCursor,
 	}
 }
+
+func pickReplicatedChartNames(fetchOptions *types.FetchOptions) []string {
+	// TODO: airgap
+	// if fetchOptions.Airgap != nil {
+	// 	return fetchOptions.Airgap.Spec.ReplicatedChartNames
+	// }
+	return fetchOptions.CurrentReplicatedChartNames
+}