From 1bc7b73a193e4580bbcd52032b68d624b92466cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Fri, 27 Oct 2023 09:55:12 +0200 Subject: [PATCH] Document create/drop catalog --- .../main/sphinx/admin/properties-catalog.md | 85 +++++++++++++++++++ docs/src/main/sphinx/admin/properties.md | 1 + docs/src/main/sphinx/language/sql-support.md | 7 ++ docs/src/main/sphinx/sql.md | 2 + docs/src/main/sphinx/sql/create-catalog.md | 82 ++++++++++++++++++ docs/src/main/sphinx/sql/drop-catalog.md | 38 +++++++++ 6 files changed, 215 insertions(+) create mode 100644 docs/src/main/sphinx/admin/properties-catalog.md create mode 100644 docs/src/main/sphinx/sql/create-catalog.md create mode 100644 docs/src/main/sphinx/sql/drop-catalog.md diff --git a/docs/src/main/sphinx/admin/properties-catalog.md b/docs/src/main/sphinx/admin/properties-catalog.md new file mode 100644 index 000000000000..52dddec22617 --- /dev/null +++ b/docs/src/main/sphinx/admin/properties-catalog.md @@ -0,0 +1,85 @@ +# Catalog management properties + +(prop-catalog-management)= +## `catalog.management` + +- **Type:** [](prop-type-string) +- **Allowed values:** `static`, `dynamic` +- **Default value:** `static` + +When set to `static`, Trino reads catalog property files and configures +available catalogs only on server startup. When set to `dynamic`, catalog +configuration can also be managed using [](/sql/create-catalog) and +[](/sql/drop-catalog). New worker nodes joining the cluster receive the current +catalog configuration from the coordinator node. + +:::{warning} +This feature is experimental only. Because of the security implications the +syntax might change and be backward incompatible. +::: + +:::{warning} +Some connectors are known not to release all resources when dropping a catalog +that uses such connector. This includes all connectors that can read data from +HDFS, S3, GCS, or Azure, which are [](/connector/hive), +[](/connector/iceberg), [](/connector/delta-lake), and +[](/connector/hudi). +::: + +:::{warning} +The complete `CREATE CATALOG` query is logged, and visible in the [Web +UI](/admin/web-interface). This includes any sensitive properties, like +passwords and other credentials. See [](/security/secrets). +::: + +## `catalog.prune.update-interval` + +- **Type:** [](prop-type-duration) +- **Default value:** `5s` +- **Minimum value:** `1s` + +Requires [](prop-catalog-management) to be set to `dynamic`. Interval for +pruning dropped catalogs. Dropping a catalog does not interrupt any running +queries that use it, but makes it unavailable to any new queries. + +(prop-catalog-store)= +## `catalog.store` + +- **Type:** [](prop-type-string) +- **Allowed values:** `file`, `memory` +- **Default value:** `file` + +Requires [](prop-catalog-management) to be set to `dynamic`. When set to +`file`, creating and dropping catalogs using the SQL commands adds and removes +catalog property files on the coordinator node. Trino server process requires +write access in the catalog configuration directory. Existing catalog files are +also read on the coordinator startup. When set to `memory`, catalog +configuration is only managed in memory, and any existing files are ignored on +startup. + +## `catalog.config-dir` + +- **Type:** [](prop-type-string) +- **Default value:** `etc/catalog/` + +Requires [](prop-catalog-management) to be set to `static` or +[](prop-catalog-store) to be set to `file`. The directory with catalog property +files. + +## `catalog.disabled-catalogs` + +- **Type:** [](prop-type-string) + +Requires [](prop-catalog-management) to be set to `static` or +[](prop-catalog-store) to be set to `file`. Comma-separated list of catalogs to +ignore on startup. + +## `catalog.read-only` + +- **Type:** [](prop-type-string) +- **Default value:** `false` + +Requires [](prop-catalog-store) to be set to `file`. If true, existing catalog +property files cannot be removed with `DROP CATALOG`, and now new catalog files +can be written with identical names with `CREATE CATALOG`. As a result, a +coordinator restart resets the known catalogs to the existing files only. diff --git a/docs/src/main/sphinx/admin/properties.md b/docs/src/main/sphinx/admin/properties.md index c0965221a0ae..c039d5742fda 100644 --- a/docs/src/main/sphinx/admin/properties.md +++ b/docs/src/main/sphinx/admin/properties.md @@ -17,6 +17,7 @@ properties, refer to the {doc}`connector documentation `. General Resource management Query management +Catalog management SQL environment Spilling Exchange diff --git a/docs/src/main/sphinx/language/sql-support.md b/docs/src/main/sphinx/language/sql-support.md index 209bbd1a6924..a30b3d1c59d5 100644 --- a/docs/src/main/sphinx/language/sql-support.md +++ b/docs/src/main/sphinx/language/sql-support.md @@ -95,6 +95,13 @@ connector to connector: - {doc}`/sql/drop-materialized-view` - {doc}`/sql/refresh-materialized-view` +(sql-catalog-management)= + +### Catalog management + +- {doc}`/sql/create-catalog` +- {doc}`/sql/drop-catalog` + (sql-schema-table-management)= ### Schema and table management diff --git a/docs/src/main/sphinx/sql.md b/docs/src/main/sphinx/sql.md index da54bc0e1bea..cd138d289b44 100644 --- a/docs/src/main/sphinx/sql.md +++ b/docs/src/main/sphinx/sql.md @@ -19,6 +19,7 @@ sql/analyze sql/call sql/comment sql/commit +sql/create-catalog sql/create-function sql/create-materialized-view sql/create-role @@ -32,6 +33,7 @@ sql/deny sql/describe sql/describe-input sql/describe-output +sql/drop-catalog sql/drop-function sql/drop-materialized-view sql/drop-role diff --git a/docs/src/main/sphinx/sql/create-catalog.md b/docs/src/main/sphinx/sql/create-catalog.md new file mode 100644 index 000000000000..727e96d7e0e1 --- /dev/null +++ b/docs/src/main/sphinx/sql/create-catalog.md @@ -0,0 +1,82 @@ +# CREATE CATALOG + +## Synopsis + +```text +CREATE CATALOG +catalog_name +USING connector_name +[ WITH ( property_name = expression [, ...] ) ] +``` + +## Description + +Create a new catalog using the specified connector. + +The optional `WITH` clause is used to set properties on the newly created +catalog. Property names can be double quoted, which is required if they contain +special characters, like `-`. Refer to the [connectors +documentation](/connector) to learn about all available properties. All +property values must be varchars (single quoted), including numbers and boolean +values. + +The query fails in the following circumstances: + +* A required property is missing. +* An invalid property is set, for example there is a typo in the property name, + or a property name from a different connector was used. +* The value of the property is invalid, for example a numeric value is out of + range, or a string value doesn't match the required pattern. +* The value references an environmental variable that is not set on the + coordinator node. + +:::{warning} +The complete `CREATE CATALOG` query is logged, and visible in the [Web +UI](/admin/web-interface). This includes any sensitive properties, like +passwords and other credentials. See [](/security/secrets). +::: + +:::{note} +This command requires the [catalog management type](/admin/properties-catalog) +to be set to `dynamic`. +::: + +## Examples + +Create a new catalog called `tpch` using the [](/connector/tpch): + +```sql +CREATE CATALOG tpch USING tpch; +``` + +Create a new catalog called `brain` using the [](/connector/memory): + +```sql +CREATE CATALOG brain USING memory +WITH ("memory.max-data-per-node" = '128MB'); +``` + +Notice that the connector property contains dashes (`-`) and needs to quoted +using a double quote (`"`). The value `128MB` is quoted using single quotes, +because it is a string literal. + +Create a new catalog called `example` using the [](/connector/postgresql): + +```sql +CREATE CATALOG example USING postgresql +WITH ( + "connection-url" = 'jdbc:pg:localhost:5432', + "connection-user" = '${ENV:POSTGRES_USER}', + "connection-password" = '${ENV:POSTGRES_PASSWORD}', + "case-insensitive-name-matching" = 'true' +); +``` + +This example assumes that the `POSTGRES_USER` and `POSTGRES_PASSWORD` +environmental variables are set as [secrets](/security/secrets) on the +coordinator node. + +## See also + +* [](/sql/drop-catalog) +* [](/admin/properties-catalog) diff --git a/docs/src/main/sphinx/sql/drop-catalog.md b/docs/src/main/sphinx/sql/drop-catalog.md new file mode 100644 index 000000000000..3ec312172371 --- /dev/null +++ b/docs/src/main/sphinx/sql/drop-catalog.md @@ -0,0 +1,38 @@ +# DROP CATALOG + +## Synopsis + +```text +DROP CATALOG catalog_name +``` + +## Description + +Drops an existing catalog. Dropping a catalog does not interrupt any running +queries that use it, but makes it unavailable to any new queries. + +:::{warning} +Some connectors are known not to release all resources when dropping a catalog +that uses such connector. This includes all connectors that can read data from +HDFS, S3, GCS, or Azure, which are [](/connector/hive), +[](/connector/iceberg), [](/connector/delta-lake), and +[](/connector/hudi). +::: + +:::{note} +This command requires the [catalog management type](/admin/properties-catalog) +to be set to `dynamic`. +::: + +## Examples + +Drop the catalog `example`: + +``` +DROP CATALOG example; +``` + +## See also + +* [](/sql/create-catalog) +* [](/admin/properties-catalog)