Merge remote-tracking branch 'origin/main' into app_level_ch_cache

rilldata · Dec 23, 2024 · 4c5cf9c · 4c5cf9c
2 parents 9eb17e7 + 53a125b
commit 4c5cf9c
Show file tree

Hide file tree

Showing 347 changed files with 12,928 additions and 12,306 deletions.
diff --git a/.github/workflows/rill-cloud.yml b/.github/workflows/rill-cloud.yml
@@ -13,6 +13,7 @@ on:
 
 env:
   RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }}
+  DEPLOY_CLOUD: 1
 
 jobs:
   release:
@@ -39,10 +40,15 @@ jobs:
 
       - name: Build & Publish Rill docker image
         run: |-
+          if [ "$RELEASE" == "true" ] || [ "$GITHUB_REF_NAME" == "main" ]; then
+            echo "DEPLOY_CLOUD=1" >> $GITHUB_ENV
+          else
+            echo "DEPLOY_CLOUD=$(git branch -r | grep release-0 | sort | tail -1 | grep -c $GITHUB_REF_NAME)" >> $GITHUB_ENV
+          fi
+          
           if [ ${RELEASE} == "false" ]; then
             echo "Fetch tags to get the last tagged version"
-            git fetch --all --tags;            
-            echo "LATEST_BRANCH=$(git branch -r | grep release-0 | sort | tail -1 | grep -c ${GITHUB_REF_NAME})" >> $GITHUB_ENV
+            git fetch --all --tags;
           fi
 
           # Embed DuckDB extensions in the Rill binary
@@ -64,7 +70,7 @@ jobs:
           fi
 
       - name: Trigger Rill Cloud deployment
-        if: env.LATEST_BRANCH == '1'
+        if: env.DEPLOY_CLOUD == '1'
         run: |-
           set -e
           curl -X POST https://api.github.com/repos/rilldata/rill-helm-charts/dispatches \

diff --git a/admin/database/database.go b/admin/database/database.go
@@ -293,6 +293,7 @@ type DB interface {
 	DeleteProjectVariables(ctx context.Context, projectID, environment string, vars []string) error
 
 	FindProvisionerResourcesForDeployment(ctx context.Context, deploymentID string) ([]*ProvisionerResource, error)
+	FindProvisionerResourceByTypeAndName(ctx context.Context, deploymentID, typ, name string) (*ProvisionerResource, error)
 	InsertProvisionerResource(ctx context.Context, opts *InsertProvisionerResourceOptions) (*ProvisionerResource, error)
 	UpdateProvisionerResource(ctx context.Context, id string, opts *UpdateProvisionerResourceOptions) (*ProvisionerResource, error)
 	DeleteProvisionerResource(ctx context.Context, id string) error
@@ -783,26 +784,28 @@ type OrganizationRole struct {
 
 // ProjectRole represents roles for projects.
 type ProjectRole struct {
-	ID                    string
-	Name                  string
-	ReadProject           bool `db:"read_project"`
-	ManageProject         bool `db:"manage_project"`
-	ReadProd              bool `db:"read_prod"`
-	ReadProdStatus        bool `db:"read_prod_status"`
-	ManageProd            bool `db:"manage_prod"`
-	ReadDev               bool `db:"read_dev"`
-	ReadDevStatus         bool `db:"read_dev_status"`
-	ManageDev             bool `db:"manage_dev"`
-	ReadProjectMembers    bool `db:"read_project_members"`
-	ManageProjectMembers  bool `db:"manage_project_members"`
-	CreateMagicAuthTokens bool `db:"create_magic_auth_tokens"`
-	ManageMagicAuthTokens bool `db:"manage_magic_auth_tokens"`
-	CreateReports         bool `db:"create_reports"`
-	ManageReports         bool `db:"manage_reports"`
-	CreateAlerts          bool `db:"create_alerts"`
-	ManageAlerts          bool `db:"manage_alerts"`
-	CreateBookmarks       bool `db:"create_bookmarks"`
-	ManageBookmarks       bool `db:"manage_bookmarks"`
+	ID                         string
+	Name                       string
+	ReadProject                bool `db:"read_project"`
+	ManageProject              bool `db:"manage_project"`
+	ReadProd                   bool `db:"read_prod"`
+	ReadProdStatus             bool `db:"read_prod_status"`
+	ManageProd                 bool `db:"manage_prod"`
+	ReadDev                    bool `db:"read_dev"`
+	ReadDevStatus              bool `db:"read_dev_status"`
+	ManageDev                  bool `db:"manage_dev"`
+	ReadProvisionerResources   bool `db:"read_provisioner_resources"`
+	ManageProvisionerResources bool `db:"manage_provisioner_resources"`
+	ReadProjectMembers         bool `db:"read_project_members"`
+	ManageProjectMembers       bool `db:"manage_project_members"`
+	CreateMagicAuthTokens      bool `db:"create_magic_auth_tokens"`
+	ManageMagicAuthTokens      bool `db:"manage_magic_auth_tokens"`
+	CreateReports              bool `db:"create_reports"`
+	ManageReports              bool `db:"manage_reports"`
+	CreateAlerts               bool `db:"create_alerts"`
+	ManageAlerts               bool `db:"manage_alerts"`
+	CreateBookmarks            bool `db:"create_bookmarks"`
+	ManageBookmarks            bool `db:"manage_bookmarks"`
 }
 
 // MemberUser is a convenience type used for display-friendly representation of an org or project member.

diff --git a/admin/database/postgres/migrations/0055.sql b/admin/database/postgres/migrations/0055.sql
@@ -0,0 +1,5 @@
+ALTER TABLE project_roles ADD read_provisioner_resources BOOLEAN DEFAULT false NOT NULL;
+UPDATE project_roles SET read_provisioner_resources = read_prod_status;
+
+ALTER TABLE project_roles ADD manage_provisioner_resources BOOLEAN DEFAULT false NOT NULL;
+UPDATE project_roles SET manage_provisioner_resources = manage_prod;
diff --git a/admin/database/postgres/postgres.go b/admin/database/postgres/postgres.go
@@ -2232,6 +2232,15 @@ func (c *connection) FindProvisionerResourcesForDeployment(ctx context.Context,
 	return c.provisionerResourcesFromDTOs(res)
 }
 
+func (c *connection) FindProvisionerResourceByTypeAndName(ctx context.Context, deploymentID, typ, name string) (*database.ProvisionerResource, error) {
+	res := &provisionerResourceDTO{}
+	err := c.getDB(ctx).QueryRowxContext(ctx, `SELECT * FROM provisioner_resources WHERE deployment_id = $1 AND "type" = $2 AND name = $3`, deploymentID, typ, name).StructScan(res)
+	if err != nil {
+		return nil, parseErr("provisioner resource", err)
+	}
+	return c.provisionerResourceFromDTO(res)
+}
+
 func (c *connection) InsertProvisionerResource(ctx context.Context, opts *database.InsertProvisionerResourceOptions) (*database.ProvisionerResource, error) {
 	if err := database.Validate(opts); err != nil {
 		return nil, err

diff --git a/admin/deployments.go b/admin/deployments.go
@@ -9,7 +9,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/hashicorp/go-version"
 	"github.com/rilldata/rill/admin/database"
 	"github.com/rilldata/rill/admin/provisioner"
@@ -478,107 +477,42 @@ type provisionRuntimeOptions struct {
 }
 
 func (s *Service) provisionRuntime(ctx context.Context, opts *provisionRuntimeOptions) (*database.ProvisionerResource, error) {
-	// Get provisioner from the set.
 	// Use default if no provisioner is specified.
 	if opts.Provisioner == "" {
 		opts.Provisioner = s.opts.DefaultProvisioner
 	}
-	p, ok := s.ProvisionerSet[opts.Provisioner]
-	if !ok {
-		return nil, fmt.Errorf("provisioner: %q is not in the provisioner set", opts.Provisioner)
-	}
 
 	// Create provisioner args
 	args := &provisioner.RuntimeArgs{
 		Slots:   opts.Slots,
 		Version: opts.Version,
 	}
 
-	// Attempt to find an existing provisioned runtime for the deployment
-	pr, ok, err := s.findProvisionedRuntimeResource(ctx, opts.DeploymentID)
-	if err != nil {
-		return nil, err
-	}
-	if ok && pr.Provisioner != opts.Provisioner {
-		return nil, fmt.Errorf("provisioner: cannot change provisioner from %q to %q for deployment %q", pr.Provisioner, opts.Provisioner, opts.DeploymentID)
-	}
-
-	// If we didn't find an existing DB entry, create one
-	if !ok {
-		pr, err = s.DB.InsertProvisionerResource(ctx, &database.InsertProvisionerResourceOptions{
-			ID:            uuid.New().String(),
-			DeploymentID:  opts.DeploymentID,
-			Type:          string(provisioner.ResourceTypeRuntime),
-			Name:          "", // Not giving runtime resources a name since there should only be one per deployment.
-			Status:        database.ProvisionerResourceStatusPending,
-			StatusMessage: "Provisioning...",
-			Provisioner:   opts.Provisioner,
-			Args:          args.AsMap(),
-			State:         nil, // Will be populated after provisioning
-			Config:        nil, // Will be populated after provisioning
-		})
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Provision the runtime
-	r := &provisioner.Resource{
-		ID:     pr.ID,
-		Type:   provisioner.ResourceTypeRuntime,
-		State:  pr.State,  // Empty if inserting
-		Config: pr.Config, // Empty if inserting
-	}
-	r, err = p.Provision(ctx, r, &provisioner.ResourceOptions{
-		Args:        args.AsMap(),
-		Annotations: opts.Annotations,
-		RillVersion: s.resolveRillVersion(),
-	})
-	if err != nil {
-		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-		defer cancel()
-		_, _ = s.DB.UpdateProvisionerResource(ctx, pr.ID, &database.UpdateProvisionerResourceOptions{
-			Status:        database.ProvisionerResourceStatusError,
-			StatusMessage: fmt.Sprintf("Failed provisioning runtime: %v", err),
-			Args:          pr.Args,
-			State:         pr.State,
-			Config:        pr.Config,
-		})
-		return nil, err
-	}
-
-	// Update the provisioner resource
-	pr, err = s.DB.UpdateProvisionerResource(ctx, pr.ID, &database.UpdateProvisionerResourceOptions{
-		Status:        database.ProvisionerResourceStatusOK,
-		StatusMessage: "",
-		Args:          args.AsMap(),
-		State:         r.State,
-		Config:        r.Config,
+	// Call into the generic provision function
+	pr, err := s.Provision(ctx, &ProvisionOptions{
+		DeploymentID: opts.DeploymentID,
+		Type:         provisioner.ResourceTypeRuntime,
+		Name:         "", // Not giving runtime resources a name since there should only be one per deployment.
+		Provisioner:  opts.Provisioner,
+		Args:         args.AsMap(),
+		Annotations:  opts.Annotations,
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	// Await the runtime to be ready
-	err = p.AwaitReady(ctx, r)
-	if err != nil {
-		return nil, err
-	}
-
 	return pr, nil
 }
 
 func (s *Service) findProvisionedRuntimeResource(ctx context.Context, deploymentID string) (*database.ProvisionerResource, bool, error) {
-	prs, err := s.DB.FindProvisionerResourcesForDeployment(ctx, deploymentID)
+	pr, err := s.DB.FindProvisionerResourceByTypeAndName(ctx, deploymentID, string(provisioner.ResourceTypeRuntime), "")
 	if err != nil {
-		return nil, false, err
-	}
-	for _, val := range prs {
-		if provisioner.ResourceType(val.Type) == provisioner.ResourceTypeRuntime {
-			return val, true, nil
+		if errors.Is(err, database.ErrNotFound) {
+			return nil, false, nil
 		}
+		return nil, false, err
 	}
-	return nil, false, nil
+	return pr, true, nil
 }
 
 func (s *Service) resolveRillVersion() string {