Implement cloud flow for plural up (#530)

* Implement cloud flow for `plural up`

This adds a `--cloud` flag to plural up to create a console-less management cluster and register it w/ a hosted console.  There are definitely some other ux things we should smooth here, but for now it's good.

* fix go.mod

* fix pluralctl setup

* fix some minor bugs

* Use temp dir to clone bootstrap repo in

* improve cloud flag

* fix after rebase

---------

Co-authored-by: Lukasz Zajaczkowski <[email protected]>

cmd/command/cd/cd.go

@@ -106,7 +106,7 @@ func Commands(clients client.Plural, helmConfiguration *action.Configuration) []
 		},
 		{
 			Name:   "login",
-			Action: p.handleCdLogin,
+			Action: p.HandleCdLogin,
 			Usage:  "logs into your plural console",
 			Flags: []cli.Flag{
 				cli.StringFlag{Name: "url", Usage: "console url"},
@@ -224,7 +224,17 @@ func confirmCluster(url, token string) (bool, error) {
 	return common.Confirm(fmt.Sprintf("Are you sure you want to install deploy operator for the cluster:\nName: %s\nHandle: %s\nProvider: %s\n", myCluster.MyCluster.Name, handle, provider), "PLURAL_INSTALL_AGENT_CONFIRM"), nil
 }
 
-func (p *Plural) handleCdLogin(c *cli.Context) (err error) {
+func (p *Plural) HandleCdLogin(c *cli.Context) (err error) {
+	prior := console.ReadConfig()
+	if prior.Url != "" {
+		if common.Affirm(
+			fmt.Sprintf("You've already configured your console at %s, continue using those credentials?", prior.Url),
+			"PLURAL_CD_USE_EXISTING_CREDENTIALS",
+		) {
+			return
+		}
+	}
+
 	url := c.String("url")
 	if url == "" {
 		url, err = utils.ReadLine("Enter the url of your console: ")

cmd/command/cd/cd_clusters.go

@@ -148,6 +148,19 @@ func (p *Plural) handleListClusters(_ *cli.Context) error {
 	})
 }
 
+func (p *Plural) GetClusterId(handle string) (string, string, error) {
+	if err := p.InitConsoleClient(consoleToken, consoleURL); err != nil {
+		return "", "", err
+	}
+
+	existing, err := p.ConsoleClient.GetCluster(nil, lo.ToPtr(handle))
+	if err != nil {
+		return "", "", err
+	}
+
+	return existing.ID, existing.Name, nil
+}
+
 func (p *Plural) handleDescribeCluster(c *cli.Context) error {
 	if err := p.InitConsoleClient(consoleToken, consoleURL); err != nil {
 		return err
@@ -384,10 +397,10 @@ func (p *Plural) handleClusterReinstall(c *cli.Context) error {
 	}
 
 	id, name := common.GetIdAndName(c.Args().Get(0))
-	return p.reinstallOperator(c, id, name)
+	return p.ReinstallOperator(c, id, name)
 }
 
-func (p *Plural) reinstallOperator(c *cli.Context, id, handle *string) error {
+func (p *Plural) ReinstallOperator(c *cli.Context, id, handle *string) error {
 	deployToken, err := p.ConsoleClient.GetDeployToken(id, handle)
 	if err != nil {
 		return err
@@ -446,7 +459,7 @@ func (p *Plural) handleClusterBootstrap(c *cli.Context) error {
 			if attrs.Handle != nil {
 				handle = attrs.Handle
 			}
-			return p.reinstallOperator(c, nil, handle)
+			return p.ReinstallOperator(c, nil, handle)
 		}
 
 		return err

cmd/command/up/backfill.go

@@ -0,0 +1,54 @@
+package up
+
+import (
+	"fmt"
+	"strings"
+
+	"encoding/base64"
+
+	"github.com/pluralsh/gqlclient"
+	"github.com/pluralsh/plural-cli/pkg/console"
+	"github.com/pluralsh/plural-cli/pkg/crypto"
+	"github.com/samber/lo"
+)
+
+func (p *Plural) backfillEncryption() error {
+	instances, err := p.Plural.Client.GetConsoleInstances()
+	if err != nil {
+		return err
+	}
+
+	conf := console.ReadConfig()
+
+	if conf.Url == "" {
+		return fmt.Errorf("You haven't configured your Plural Console client yet")
+	}
+
+	var id string
+	for _, inst := range instances {
+		if strings.Contains(conf.Url, inst.URL) {
+			id = inst.ID
+		}
+	}
+	if id == "" {
+		return fmt.Errorf("Your configuration doesn't match to any existing Plural Console")
+	}
+
+	prov, err := crypto.Build()
+	if err != nil {
+		return err
+	}
+
+	raw, err := prov.SymmetricKey()
+	if err != nil {
+		return err
+	}
+
+	encoded := base64.StdEncoding.EncodeToString(raw)
+
+	return p.Plural.Client.UpdateConsoleInstance(id, gqlclient.ConsoleInstanceUpdateAttributes{
+		Configuration: &gqlclient.ConsoleConfigurationUpdateAttributes{
+			EncryptionKey: lo.ToPtr(encoded),
+		},
+	})
+}

cmd/command/up/up.go

@@ -2,12 +2,15 @@ package up
 
 import (
 	"fmt"
+	"os"
 
+	"github.com/pluralsh/plural-cli/cmd/command/cd"
 	"github.com/pluralsh/plural-cli/pkg/client"
 	"github.com/pluralsh/plural-cli/pkg/common"
 	"github.com/pluralsh/plural-cli/pkg/up"
 	"github.com/pluralsh/plural-cli/pkg/utils"
 	"github.com/pluralsh/plural-cli/pkg/utils/git"
+	"github.com/samber/lo"
 	"github.com/urfave/cli"
 )
 
@@ -35,6 +38,10 @@ func Command(clients client.Plural) cli.Command {
 				Name:  "ignore-preflights",
 				Usage: "whether to ignore preflight check failures prior to init",
 			},
+			cli.BoolFlag{
+				Name:  "cloud",
+				Usage: "Whether you're provisioning against a cloud-hosted Plural Console",
+			},
 			cli.StringFlag{
 				Name:  "commit",
 				Usage: "commits your changes with this message",
@@ -51,21 +58,45 @@ func (p *Plural) handleUp(c *cli.Context) error {
 	}
 	p.InitPluralClient()
 
+	cd := &cd.Plural{Plural: p.Plural}
+
+	if c.Bool("cloud") {
+		if err := cd.HandleCdLogin(c); err != nil {
+			return err
+		}
+
+		if err := p.backfillEncryption(); err != nil {
+			return err
+		}
+	}
+
 	repoRoot, err := git.Root()
 	if err != nil {
 		return err
 	}
 
-	ctx, err := up.Build()
+	ctx, err := up.Build(c.Bool("cloud"))
 	if err != nil {
 		return err
 	}
 
+	if c.Bool("cloud") {
+		id, name, err := cd.GetClusterId("mgmt")
+		if err != nil {
+			return err
+		}
+
+		ctx.ImportCluster = lo.ToPtr(id)
+		ctx.CloudCluster = name
+	}
+
 	if err := ctx.Backfill(); err != nil {
 		return err
 	}
 
-	if err := ctx.Generate(); err != nil {
+	dir, err := ctx.Generate()
+	defer func() { os.RemoveAll(dir) }()
+	if err != nil {
 		return err
 	}
 
@@ -85,6 +116,6 @@ func (p *Plural) handleUp(c *cli.Context) error {
 	}
 
 	utils.Success("Finished setting up your management cluster!\n")
-	utils.Highlight("Feel free to use `terrafrom` as you normally would, and leverage the gitops setup we've generated in the `apps/` subfolder\n")
+	utils.Highlight("Feel free to use terraform as you normally would, and leverage the gitops setup we've generated in the apps/ subfolder\n")
 	return nil
 }

go.mod

@@ -55,8 +55,8 @@ require (
 	github.com/packethost/packngo v0.29.0
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
 	github.com/pluralsh/cluster-api-migration v0.2.16
-	github.com/pluralsh/console/go/client v1.5.0
-	github.com/pluralsh/gqlclient v1.12.1
+	github.com/pluralsh/console/go/client v1.12.0
+	github.com/pluralsh/gqlclient v1.12.2
 	github.com/pluralsh/plural-operator v0.5.5
 	github.com/pluralsh/polly v0.1.8
 	github.com/pluralsh/terraform-delinker v0.0.2

go.sum

@@ -1872,12 +1872,12 @@ github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
 github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
 github.com/pluralsh/cluster-api-migration v0.2.16 h1:MQGrLQAhGSSpyjEDamhnJZaQ8MkxlHzR8PZxIVavLIM=
 github.com/pluralsh/cluster-api-migration v0.2.16/go.mod h1:24PjMsYv3vSlUiYw7BeUQ0GAtK0Jk2B1iwh35WGQLx8=
-github.com/pluralsh/console/go/client v1.5.0 h1:O5pHJUXqBJWUt66MiCRlyatarNFQGy1Fy6QaC5P0avY=
-github.com/pluralsh/console/go/client v1.5.0/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k=
+github.com/pluralsh/console/go/client v1.12.0 h1:1djVBV9GMEe1I6yKv0bWu2Qx8dnsVgGzhntGSY+KLCw=
+github.com/pluralsh/console/go/client v1.12.0/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k=
 github.com/pluralsh/controller-reconcile-helper v0.0.4 h1:1o+7qYSyoeqKFjx+WgQTxDz4Q2VMpzprJIIKShxqG0E=
 github.com/pluralsh/controller-reconcile-helper v0.0.4/go.mod h1:AfY0gtteD6veBjmB6jiRx/aR4yevEf6K0M13/pGan/s=
-github.com/pluralsh/gqlclient v1.12.1 h1:JDOkP9jjqkPdTYdpH5hooG4F8T6FDH90XfipeXJmJFY=
-github.com/pluralsh/gqlclient v1.12.1/go.mod h1:OEjN9L63x8m3A3eQBv5kVkFgiY9fp2aZ0cgOF0uII58=
+github.com/pluralsh/gqlclient v1.12.2 h1:BrEFAASktf4quFw57CIaLAd+NZUTLhG08fe6tnhBQN4=
+github.com/pluralsh/gqlclient v1.12.2/go.mod h1:OEjN9L63x8m3A3eQBv5kVkFgiY9fp2aZ0cgOF0uII58=
 github.com/pluralsh/helm-docs v1.11.3-0.20230914191425-6d14ebab8817 h1:J7SGxH6nJGdRoNtqdzhyr2VMpbl4asolul7xqqW++EA=
 github.com/pluralsh/helm-docs v1.11.3-0.20230914191425-6d14ebab8817/go.mod h1:rLqec59NO7YF57Rq9VlubQHMp7wcRTJhzpkcgs4lOG4=
 github.com/pluralsh/oauth v0.9.2 h1:tM9hBK4tCnJUeCOgX0ctxBBCS3hiCDPoxkJLODtedmQ=

pkg/api/client.go

@@ -99,6 +99,8 @@ type Client interface {
 	DeleteTrust(id string) error
 	OidcToken(provider gqlclient.ExternalOidcProvider, token, email string) (string, error)
 	MarkSynced(repo string) error
+	GetConsoleInstances() ([]*gqlclient.ConsoleInstanceFragment, error)
+	UpdateConsoleInstance(id string, attrs gqlclient.ConsoleInstanceUpdateAttributes) error
 }
 
 type client struct {

pkg/api/cloud.go

@@ -0,0 +1,24 @@
+package api
+
+import (
+	"github.com/pluralsh/gqlclient"
+)
+
+func (client *client) GetConsoleInstances() ([]*gqlclient.ConsoleInstanceFragment, error) {
+	res := []*gqlclient.ConsoleInstanceFragment{}
+	resp, err := client.pluralClient.GetConsoleInstances(client.ctx, 100)
+	if err != nil {
+		return res, err
+	}
+
+	for _, node := range resp.ConsoleInstances.Edges {
+		res = append(res, node.Node)
+	}
+
+	return res, nil
+}
+
+func (client *client) UpdateConsoleInstance(id string, attrs gqlclient.ConsoleInstanceUpdateAttributes) error {
+	_, err := client.pluralClient.UpdateConsoleInstance(client.ctx, id, attrs)
+	return err
+}

pkg/client/plural.go

@@ -126,7 +126,7 @@ func (p *Plural) HandleInit(c *cli.Context) error {
 		return nil
 	}
 
-	prov, err := common.RunPreflights()
+	prov, err := common.RunPreflights(c)
 	if err != nil && !c.Bool("ignore-preflights") {
 		return err
 	}

pkg/common/app.go

@@ -71,7 +71,7 @@ func HandleDown(_ *cli.Context) error {
 		return fmt.Errorf("cancelled destroy")
 	}
 
-	ctx, err := up.Build()
+	ctx, err := up.Build(false)
 	if err != nil {
 		return err
 	}

pkg/common/common.go

@@ -150,11 +150,12 @@ func postLogin(conf *config.Config, client api.Client, c *cli.Context, persist b
 	return conf.Flush()
 }
 func Preflights(c *cli.Context) error {
-	_, err := RunPreflights()
+	_, err := RunPreflights(c)
 	return err
 }
 
-func RunPreflights() (provider.Provider, error) {
+func RunPreflights(c *cli.Context) (provider.Provider, error) {
+	provider.SetCloudFlag(c.Bool("cloud"))
 	prov, err := provider.GetProvider()
 	if err != nil {
 		return prov, err

pkg/console/config.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"gopkg.in/yaml.v2"
+	"sigs.k8s.io/yaml"
 )
 
 const (
@@ -19,14 +19,14 @@ var (
 )
 
 type VersionedConfig struct {
-	ApiVersion string  `yaml:"apiVersion"`
-	Kind       string  `yaml:"kind"`
-	Spec       *Config `yaml:"spec"`
+	ApiVersion string  `json:"apiVersion"`
+	Kind       string  `json:"kind"`
+	Spec       *Config `json:"spec"`
 }
 
 type Config struct {
-	Url   string `yaml:"url"`
-	Token string `yaml:"token"`
+	Url   string `json:"url"`
+	Token string `json:"token"`
 }
 
 func configFile() string {

pkg/console/console.go

@@ -90,6 +90,8 @@ func NormalizeExtUrl(uri string) string {
 		uri = fmt.Sprintf("https://%s", uri)
 	}
 
+	uri = strings.TrimSuffix(uri, "/")
+
 	parsed, err := url.Parse(uri)
 	if err != nil {
 		panic(err)
@@ -106,6 +108,8 @@ func NormalizeUrl(url string) string {
 		url = fmt.Sprintf("%s/gql", url)
 	}
 
+	url = strings.TrimSuffix(url, "/")
+
 	return url
 }