Merge branch 'main' into fix/url-history-issues

.github/workflows/rill-cloud.yml

@@ -13,6 +13,7 @@ on:
 
 env:
   RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }}
+  DEPLOY_CLOUD: 1
 
 jobs:
   release:
@@ -39,10 +40,15 @@ jobs:
 
       - name: Build & Publish Rill docker image
         run: |-
+          if [ "$RELEASE" == "true" ] || [ "$GITHUB_REF_NAME" == "main" ]; then
+            echo "DEPLOY_CLOUD=1" >> $GITHUB_ENV
+          else
+            echo "DEPLOY_CLOUD=$(git branch -r | grep release-0 | sort | tail -1 | grep -c $GITHUB_REF_NAME)" >> $GITHUB_ENV
+          fi
+          
           if [ ${RELEASE} == "false" ]; then
             echo "Fetch tags to get the last tagged version"
-            git fetch --all --tags;            
-            echo "LATEST_BRANCH=$(git branch -r | grep release-0 | sort | tail -1 | grep -c ${GITHUB_REF_NAME})" >> $GITHUB_ENV
+            git fetch --all --tags;
           fi
 
           # Embed DuckDB extensions in the Rill binary
@@ -64,7 +70,7 @@ jobs:
           fi
 
       - name: Trigger Rill Cloud deployment
-        if: env.LATEST_BRANCH == '1'
+        if: env.DEPLOY_CLOUD == '1'
         run: |-
           set -e
           curl -X POST https://api.github.com/repos/rilldata/rill-helm-charts/dispatches \

admin/provisioner/clickhousestatic/provisioner.go

@@ -102,12 +102,20 @@ func (p *Provisioner) Provision(ctx context.Context, r *provisioner.Resource, op
 		return nil, fmt.Errorf("failed to create clickhouse database: %w", err)
 	}
 
-	// Idempotently create the user
+	// Idempotently create the user.
 	_, err = p.ch.ExecContext(ctx, fmt.Sprintf("CREATE USER IF NOT EXISTS %s IDENTIFIED WITH sha256_password BY ? DEFAULT DATABASE %s GRANTEES NONE", user, dbName), password)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create clickhouse user: %w", err)
 	}
 
+	// When creating the user, the password assignment is not idempotent (if there are two concurrent invocations, we don't know which password was used).
+	// By adding the password separately, we ensure all passwords will work.
+	// NOTE: Requires ClickHouse 24.9 or later.
+	_, err = p.ch.ExecContext(ctx, fmt.Sprintf("ALTER USER %s ADD IDENTIFIED WITH sha256_password BY ?", user), password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to add password for clickhouse user: %w", err)
+	}
+
 	// Grant privileges on the database to the user
 	_, err = p.ch.ExecContext(ctx, fmt.Sprintf(`
 		GRANT

admin/provisioner/clickhousestatic/provisioner_test.go

@@ -17,11 +17,11 @@ import (
 	"go.uber.org/zap"
 )
 
-func Test(t *testing.T) {
+func TestClickHouseStatic(t *testing.T) {
 	// Create a test ClickHouse cluster
 	container, err := testcontainersclickhouse.Run(
 		context.Background(),
-		"clickhouse/clickhouse-server:24.6.2.17",
+		"clickhouse/clickhouse-server:24.11.1.2557",
 		// Add a user config file that enables access management for the "default" user
 		testcontainers.CustomizeRequestOption(func(req *testcontainers.GenericContainerRequest) error {
 			req.Files = append(req.Files, testcontainers.ContainerFile{

cli/pkg/local/app.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"strconv"
 	"time"
 
 	"github.com/c2h5oh/datasize"
@@ -197,26 +196,13 @@ func NewApp(ctx context.Context, opts *AppOptions) (*App, error) {
 		}
 	}
 
-	// If the OLAP is the default OLAP (DuckDB in stage.db), we make it relative to the project directory (not the working directory)
-	defaultOLAP := false
 	olapCfg := make(map[string]string)
-	if opts.OlapDriver == DefaultOLAPDriver && opts.OlapDSN == DefaultOLAPDSN {
-		defaultOLAP = true
-		val, err := isExternalStorageEnabled(vars)
-		if err != nil {
-			return nil, err
-		}
-		olapCfg["external_table_storage"] = strconv.FormatBool(val)
-	}
-
 	if opts.OlapDriver == "duckdb" {
+		if opts.OlapDSN != DefaultOLAPDSN {
+			return nil, fmt.Errorf("setting DSN for DuckDB is not supported")
+		}
 		// Set default DuckDB pool size to 4
 		olapCfg["pool_size"] = "4"
-		if !defaultOLAP {
-			// dsn is automatically computed by duckdb driver so we set only when non default dsn is passed
-			olapCfg["dsn"] = opts.OlapDSN
-			olapCfg["error_on_incompatible_version"] = "true"
-		}
 	}
 
 	// Add OLAP connector
@@ -609,14 +595,3 @@ func (s skipFieldZapEncoder) AddString(key, val string) {
 		s.Encoder.AddString(key, val)
 	}
 }
-
-// isExternalStorageEnabled determines if external storage can be enabled.
-func isExternalStorageEnabled(variables map[string]string) (bool, error) {
-	// check if flag explicitly passed
-	val, ok := variables["connector.duckdb.external_table_storage"]
-	if !ok {
-		// mark enabled by default
-		return true, nil
-	}
-	return strconv.ParseBool(val)
-}

docs/docs/build/credentials/credentials.md

@@ -8,7 +8,7 @@ Rill requires credentials to connect to remote data sources such as private buck
 
 At a high level, configuring credentials and credentials management in Rill can be broken down into three categories:
 - Setting credentials for Rill Developer
-- Setting credentials for a Rill Cloud project
+- [Setting credentials for a Rill Cloud project](/manage/variables-and-credentials)
 - Pushing and pulling credentials to / from Rill Cloud
 
 ## Setting credentials for Rill Developer
@@ -28,37 +28,38 @@ If you plan to deploy a project (to Rill Cloud), it is not recommended to pass i
 :::
 
 
-## Setting credentials for a Rill Cloud project
+## Variables
 
-When deploying a project, Rill Cloud will need appropriate credentials passed to it that can be used to read from a source (or to be used to establish a connection with the OLAP engine). This can be done either by:
-1. Using [rill env configure](../../reference/cli/env/configure.md) to pass in connector level credentials (make sure to run this CLI command from the root directory of your Rill project)
-2. Specifying all corresponding source level and OLAP engine credentials in the project's `.env` file and "pushing it" to Rill Cloud ([see below](#pushing-and-pulling-credentials-to--from-rill-cloud))
+Project variables work exactly the same way as credentials and can be defined when starting rill via `--env key=value` or set in the .env file in the project directory.
 
-For more details, please refer to the corresponding [connector](../../reference/connectors/connectors.md) or [OLAP engine](../../reference/olap-engines/olap-engines.md) page.
+```bash
+variable=xyz
+```
 
-:::tip Best practice
+This variable will then be usable and referenceable for [templating](../../deploy/templating.md) purposes in the local instance of your project. 
 
-For projects that you deploy to production, it is recommended to use service accounts when possible instead of individual user credentials.
+:::info Fun Fact
 
+Connector credentials are essentially a form of project variable, prefixed using the `connector.<connector_name>.<property>` syntax. For example, `connector.druid.dsn` and `connector.clickhouse.dsn` are both hardcoded project variables (that happen to correspond to the [Druid](/reference/olap-engines/druid.md) and [ClickHouse](/reference/olap-engines/clickhouse.md) OLAP engines respectively).
 
 :::
 
-## Pushing and pulling credentials to / from Rill Cloud
+:::tip Avoid committing sensitive information to Git
 
-If you have a project deployed to Rill Cloud, Rill provides the ability to **sync** the credentials between your local instance of Rill Developer and Rill cloud. This provides the ability to quickly reuse existing credentials, if configured, instead of having to manually input credentials each time. This can be accomplished by leveraging the `rill env push` and `rill env pull` CLI commands respectively.
+It's never a good idea to commit sensitive information to Git and goes against security best practices. Similar to credentials, if there are sensitive variables that you don't want to commit publicly to your `rill.yaml` configuration file (and thus potentially accessible by others), it's recommended to set them in your `.env` file directly and/or use `rill env set` via the CLI (and then optionally push / pull them as necessary).
 
-### rill env push
+:::
 
-As a project admin, you can either use `rill env configure` after deploying a project or `rill env push` to specify a particular set of credentials that your Rill Cloud project will use. If choosing the latter, you can update your *`<RILL_PROJECT_HOME>/.env`* file with the appropriate variables and credentials that are required. Alternatively, if this file has already been updated, you can run `rill env push` from your project's root directory.
-- Rill Cloud will use the specified credentials and variables in this `.env` file for the deployed project.
-- Other users will also be able to use `rill env pull` to retrieve these defined credentials for local use (with Rill Developer).
+## Deploying to Rill Cloud 
 
-:::warning Overriding Cloud credentials
+:::tip Ready to Deploy?
+Please see our [deploy credentials page](/deploy/deploy-credentials#configure-environmental-variables-and-credentials-on-rill-cloud) to configure your credentials on Rill Cloud.
+:::
 
-If a credential and/or variable has already been configured in Rill Cloud, Rill will warn you about overriding if you attempt to push a new value in your `.env` file. This is because overriding credentials can impact your deployed project and/or other users (if they pull these credentials locally).
-![Pushing credentials that already exist to Rill Cloud](/img/build/credentials/rill-env-push.png)
 
-:::
+## Pulling Credentials and Variables from a Deployed Project on Rill Cloud
+
+If you are making changes to an already deployed instance from Rill Cloud, it is possible to **sync** the credentials and variables from the Rill Cloud to your local instance of Rill Developer. 
 
 ### rill env pull
 
@@ -72,26 +73,17 @@ Please note when you run `rill env pull`, Rill will *automatically override any
 
 :::
 
-### variables
-
-Project variables work exactly the same way as credentials and can be pushed to Cloud / pulled locally using the same `rill env push` and `rill env pull` commands specifically. To do this, if you're in Rill Developer and want to set a variable through your `<RILL_PROJECT_HOME>/.env` file (and save):
 
-```bash
-variable=xyz
-```
-
-This variable should then be usable and referenceable for [templating](../../deploy/templating.md) purposes in the local instance of your project. Then, if you want to push these variable definitions to your deployed Cloud project, you can use `rill env push`. Similarly, if these variables had already been set in Rill Cloud for your project, you can use `rill env pull` to clone these variables locally (in your `.env` file).
-
-:::info Fun Fact
-
-Connector credentials are essentially a form of project variable, prefixed using the `connector.<connector_name>.<property>` syntax. For example, `connector.druid.dsn` and `connector.clickhouse.dsn` are both hardcoded project variables (that happen to correspond to the [Druid](/reference/olap-engines/druid.md) and [ClickHouse](/reference/olap-engines/clickhouse.md) OLAP engines respectively).
+### rill env push
 
-:::
+As a project admin, you can either use `rill env configure` after deploying a project or `rill env push` to specify a particular set of credentials that your Rill Cloud project will use. If choosing the latter, you can update your *`<RILL_PROJECT_HOME>/.env`* file with the appropriate variables and credentials that are required. Alternatively, if this file has already been updated, you can run `rill env push` from your project's root directory.
+- Rill Cloud will use the specified credentials and variables in this `.env` file for the deployed project.
+- Other users will also be able to use `rill env pull` to retrieve these defined credentials for local use (with Rill Developer).
 
-:::tip Avoid committing sensitive information to Git
+:::warning Overriding Cloud credentials
 
-It's never a good idea to commit sensitive information to Git and goes against security best practices. Similar to credentials, if there are sensitive variables that you don't want to commit publicly to your `rill.yaml` configuration file (and thus potentially accessible by others), it's recommended to set them in your `.env` file directly and/or use `rill env set` via the CLI (and then optionally push / pull them as necessary).
+If a credential and/or variable has already been configured in Rill Cloud, Rill will warn you about overriding if you attempt to push a new value in your `.env` file. This is because overriding credentials can impact your deployed project and/or other users (if they pull these credentials locally).
+![Pushing credentials that already exist to Rill Cloud](/img/build/credentials/rill-env-push.png)
 
 :::
 
-

docs/docs/build/dashboards/customize.md

@@ -16,36 +16,35 @@ For a full list of available dashboard properties and configurations, please see
 :::
 
 
-**`time_ranges`**
+**`time_ranges:`**
 
 One of the more important configurations, available time ranges allow you to change the defaults in the time dropdown for periods to select. Updating this list allows users to quickly change between the most common analyses like day over day, recent weeks, or period to date. The range must be a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) or one of the [Rill ISO 8601 extensions](../../reference/rill-iso-extensions.md#extensions).
 
 ```yaml
-  available_time_ranges:
-  - PT15M 
-  - PT1H
-  - P7D
-  - P4W
-  - rill-TD ## Today
-  - rill-WTD ## Week-To-date
+- PT15M 
+- PT1H
+- P7D
+- P4W
+- rill-TD ## Today
+- rill-WTD ## Week-To-date
 ```
 
-**`time_zones`**
+**`time_zones:`**
 
 Rill will automatically select several time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). You can add or remove from this list for the relevant time zones for your team.
 
 ```yaml
-  - America/Los_Angeles
-  - America/Chicago
-  - America/New_York
-  - Europe/London
-  - Europe/Paris
-  - Asia/Jerusalem
-  - Europe/Moscow
-  - Asia/Kolkata
-  - Asia/Shanghai
-  - Asia/Tokyo
-  - Australia/Sydney
+- America/Los_Angeles
+- America/Chicago
+- America/New_York
+- Europe/London
+- Europe/Paris
+- Asia/Jerusalem
+- Europe/Moscow
+- Asia/Kolkata
+- Asia/Shanghai
+- Asia/Tokyo
+- Australia/Sydney
 ```
 
 ### Setting Default Views for Dashboards
@@ -122,8 +121,8 @@ title: Title of your explore dashboard
 description: a description
 metrics_view: <your-metric-view-file-name>
 
-dimensions: '*' #can use regex
-measures: '*' #can use regex
+dimensions: '*' #can use expressions
+measures: '*' #can use expressions
 
 theme: #your default theme
 
@@ -132,10 +131,7 @@ time_zones: #was available_time_zones
 
 defaults: #define all the defaults within here
     dimensions:
-    measures:
-    time_range:
-    comparison_mode:
-    comparison_dimension:
+  ...
 
 security:
     access: #only access can be set on dashboard level, see metric view for detailed access policies

docs/docs/build/dashboards/dashboards.md

@@ -4,16 +4,18 @@ description: Create dashboards using source data and models with time, dimension
 sidebar_label: Create Explore Dashboards
 sidebar_position: 00
 ---
+
+<img src = '/img/build/dashboard/explore-dashboard.gif' class='rounded-gif' />
+<br />
+
+
+In Rill, explore dashboards are used to visually understand your data with real-time filtering based on your defined dimensions and measures in your metrics view. In the explore dashboard YAML, you can define which measures and dimensions are visible as well as define the default view when a user sees your dashboard. 
 :::tip
 Starting in version 0.50, metrics views has been separated from explore dashboards. This allows for a cleaner, more accessible metrics layer and the ability to build various dashboards and components on top of a single metrics view. For more information on what a metrics view is please see: [What is a Metrics View?](/concepts/metrics-layer)
 
 For migration steps, see [Migrations](/latest-changes/v50-dashboard-changes#how-to-migrate-your-current-dashboards).
 :::
 
-In Rill, explore dashboards are used to visually understand your data with real-time filtering based on your defined dimensions and measures in your metrics view. In the explore dashboard YAML, you can define which measures and dimensions are visible as well as define the default view when a user sees your dashboard. 
-
-![img](/img/build/dashboard/explore-dashboard.png)
-
 * _**metrics_view**_ - A metrics view that powers the dashboard
 * _**measures**_ - `*` Which measures to include or exclude from the metrics view, using a wildcard will include all.
 * _**dimensions**_ -  `*` Which dimensions to include or exclude from the metrics view, using a wildcard will include all.
@@ -29,11 +31,8 @@ title: Title of your Explore Dashboard
 description: a description for your explore dashboard
 metrics_view: my_metricsview
 
-dimensions: '*' #can use regex
-measures: '*' #can use regex
-
-time_ranges: #was available_time_ranges, list the time of available time ranges that can be selected in your dashboard
-time_zones: #was available_time_zones, list the time zones that are selectable in the dashboard
+dimensions: '*' #can use expressions
+measures: '*' #can use expressions
 
 defaults: #define all the defaults within here, was default_* in previous dashboard YAML
     dimensions: 
@@ -44,8 +43,6 @@ security:
 ```
 
 
-
-
 :::note Dashboard Properties
 For more details about available configurations and properties, check our [Dashboard YAML](/reference/project-files/explore-dashboards) reference page.
 :::

docs/docs/build/structure/structure.md

@@ -26,6 +26,12 @@ However, if you'd like to create a resource outside one of these native folders,
 
 :::
 
+## Navigating Upstream / Downstream Objects
+
+<img src = '/img/build/structure/breadcrumb.png' class='rounded-gif' />
+<br />
+When selecting between a source, model, metrics view and dashboard, you can view the upstream/downstream objects to the current view. For example, if you are selecting a metrics view, you can see all of the dashboards (in a dropdown) that are built on the metrics view. Likewise, if your model references several sources, this will be available to select. 
+
 ## Moving resources within your project
 
 From the UI, within the file explorer, you should be able to drag resources / objects around and move them from / to folders as necessary. 

docs/docs/deploy/deploy-credentials.md

@@ -14,6 +14,22 @@ As a general best practice, it is strongly recommended to use service accounts a
 
 :::
 
+## Configure Environmental Variables and Credentials on Rill Cloud 
+
+Once you are ready to deploy or have already deployed and are experiencing issues connecting to your source, you will need to run the following command, `rill env configure`. When running this command, Rill will detect any connectors that are being used by the project and prompt you to fill in the required fields. When completed, this will be pushed to your Rill Cloud Deployment and automatically refresh the required objects. 
+
+
+```bash
+$rill env configure
+Finish deploying your project by providing access to the connectors. Rill requires credentials for the following connectors:
+
+ - your connectors here (used by models and sources)
+
+Configuring connector "bigquery":
+...
+
+Updated project variables
+```
 ## Service Accounts
 
 Service accounts are non-human user accounts that provide an identity for processes or services running on a server to interact with external resources, such as databases, APIs, and cloud services. Unlike personal user accounts, service accounts are intended for use by software applications or automated tools and do not require interactive login. In the context of Rill, service accounts are credentials that should be used for projects deployed to Rill Cloud.