update autogenerate-settings.sh for core settings, format settings an…

…d also add merge tree settings

scripts/settings/autogenerate-settings.sh

@@ -2,7 +2,7 @@
 
 SCRIPT_NAME=$(basename "$0")
 
-echo "[$SCRIPT_NAME] Autogenerating settings"
+echo "[$SCRIPT_NAME] Autogenerating markdown from settings"
 
 # Install ClickHouse
 if [ ! -f ./clickhouse ]; then
@@ -11,61 +11,91 @@ if [ ! -f ./clickhouse ]; then
 fi
 
 # Autogenerate Format settings
+echo "[$SCRIPT_NAME] Autogenerating format settings"
 ./clickhouse -q "
 WITH
 'FormatFactorySettings.h' AS cpp_file,
-settings_from_cpp AS
+format_settings_from_source AS
 (
     SELECT extract(line, 'DECLARE\\(\\w+, (\\w+),') AS name
     FROM file(cpp_file, LineAsString)
     WHERE match(line, '^\\s*DECLARE\\(')
-),
-main_content AS
-(
-    SELECT format('## {} {}\\n\\nType: {}\\n\\nDefault value: {}\\n\\n{}\\n\\n', name, '{#'||name||'}', type, default, trim(BOTH '\\n' FROM description))
-    FROM system.settings WHERE name IN settings_from_cpp
-    ORDER BY name
-),
-'---
-title: Format Settings
-sidebar_label: Format Settings
-slug: /en/operations/settings/formats
-toc_max_heading_level: 2
----
-<!-- Autogenerated -->
-These settings are autogenerated from [source](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/FormatFactorySettings.h).
-' AS prefix
-SELECT prefix || (SELECT groupConcat(*) FROM main_content)
-INTO OUTFILE 'docs/en/operations/settings/settings-formats.md' TRUNCATE FORMAT LineAsString
+)
+SELECT
+    name,
+    description,
+    type,
+    default,
+    tier,
+FROM system.settings
+WHERE name IN format_settings_from_source
+ORDER BY
+  CASE
+    WHEN tier = 'Production' THEN 1
+    WHEN tier = 'Beta' THEN 2
+    WHEN tier = 'Experimental' THEN 3
+    WHEN tier = 'Obsolete' THEN 4
+  END, name ASC
+INTO OUTFILE 'docs/en/operations/settings/settings-formats.md' TRUNCATE
+FORMAT Template
+SETTINGS
+format_template_resultset = 'scripts/settings/resultset_format_settings.format',
+format_template_row = 'scripts/settings/row_settings.format'
 "
 
-# Autogenerate settings
+# Autogenerate system.settings (Core Settings)
+echo "[$SCRIPT_NAME] Autogenerating core settings"
 ./clickhouse -q "
 WITH
 'Settings.cpp' AS cpp_file,
-settings_from_cpp AS
+system_settings_from_source AS
 (
     SELECT extract(line, 'DECLARE\\(\\w+, (\\w+),') AS name
     FROM file(cpp_file, LineAsString)
     WHERE match(line, '^\\s*DECLARE\\(')
-),
-main_content AS
-(
-    SELECT format('## {} {}\\n\\nType: {}\\n\\nDefault value: {}\\n\\n{}\\n\\n', name, '{#'||name||'}', type, default, trim(BOTH '\\n' FROM description))
-    FROM system.settings WHERE name IN settings_from_cpp
-    ORDER BY name
-),
-'---
-title: Core Settings
-sidebar_label: Core Settings
-slug: /en/operations/settings/settings
-toc_max_heading_level: 2
----
-<!-- Autogenerated -->
-All below settings are also available in table [system.settings](/docs/en/operations/system-tables/settings). These settings are autogenerated from [source](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.cpp).
-' AS prefix
-SELECT prefix || (SELECT groupConcat(*) FROM main_content)
-INTO OUTFILE 'docs/en/operations/settings/settings.md' TRUNCATE FORMAT LineAsString
+)
+SELECT
+    name,
+    description,
+    type,
+    default,
+    tier
+FROM system.settings
+WHERE name IN system_settings_from_source
+ORDER BY
+  CASE
+    WHEN tier = 'Production' THEN 1
+    WHEN tier = 'Beta' THEN 2
+    WHEN tier = 'Experimental' THEN 3
+    WHEN tier = 'Obsolete' THEN 4
+  END, name ASC
+INTO OUTFILE 'docs/en/operations/settings/settings.md' TRUNCATE
+FORMAT Template
+SETTINGS format_template_resultset = 'scripts/settings/resultset_system_settings.format',
+format_template_row = 'scripts/settings/row_settings.format'
+"
+
+# Autogenerate system.merge_tree_settings
+echo "[$SCRIPT_NAME] Autogenerating merge tree settings"
+./clickhouse -q "
+SELECT
+    name,
+    description,
+    type,
+    value,
+    tier
+FROM system.merge_tree_settings
+ORDER BY
+  CASE
+    WHEN tier = 'Production' THEN 1
+    WHEN tier = 'Beta' THEN 2
+    WHEN tier = 'Experimental' THEN 3
+    WHEN tier = 'Obsolete' THEN 4
+  END, name ASC
+INTO OUTFILE 'docs/en/operations/settings/merge-tree-settings.md' TRUNCATE
+FORMAT Template
+SETTINGS format_template_resultset = 'scripts/settings/resultset_merge_tree_settings.format',
+format_template_row = 'scripts/settings/row_settings.format'
 "
 
 # Delete ClickHouse

scripts/settings/autogenerated-markdown-readme.md

@@ -0,0 +1,91 @@
+# Autogenerated markdown
+
+This is a small explanation of how to work with the autogenerated markdown documentation, in case of future updates.
+
+In this folder you will find the following files:
+- `autogenerate-settings.sh` (defines the queries for generating each markdown page)
+- `resultset_{}_settings.format` (one file per generated page defining the template of the markdown page)
+- `row_settings.format` (defines the structure of each setting entry which will get inserted into one of the template pages mentioned above).
+
+## Example
+
+The [MergeTree Table Settings]() page is generated using these files: `autogenerate-settings.sh`, `resultset_{}_settings.format` and `row_settings.format`.
+
+In `autogenerate-settings.sh` ClickHouse will be cloned and the following query is run:
+
+```sql
+WITH
+'FormatFactorySettings.h' AS cpp_file,
+format_settings_from_source AS
+(
+    SELECT extract(line, 'DECLARE\\(\\w+, (\\w+),') AS name
+    FROM file(cpp_file, LineAsString)
+    WHERE match(line, '^\\s*DECLARE\\(')
+)
+SELECT
+    name,
+    description,
+    type,
+    default,
+    tier,
+FROM system.settings
+WHERE name IN format_settings_from_source
+ORDER BY
+  CASE
+    WHEN tier = 'Production' THEN 1
+    WHEN tier = 'Beta' THEN 2
+    WHEN tier = 'Experimental' THEN 3
+    WHEN tier = 'Obsolete' THEN 4
+  END, name ASC
+INTO OUTFILE 'docs/en/operations/settings/settings-formats.md' TRUNCATE
+FORMAT Template
+SETTINGS
+format_template_resultset = 'scripts/settings/resultset_format_settings.format',
+format_template_row = 'scripts/settings/row_settings.format'
+```
+For this markdown file it is necessary to first extract only the format settings from the `system.settings` table which contains not only the settings for formats, but also the core settings.
+The format settings are defined in [FormatSettings.h]() and so we first extract the names of all the format settings into `format_settings_from_source`.
+
+Once this is done we can use this list as a condition to select only the format settings from `system.settings`.
+
+We are using the [Template](http://www.clickhouse.com/docs/en/interfaces/formats#format-template) format to generate the markdown. We specify the general page template using `resultset_format_settings.format` which is specified using setting `format_template_resultset`:
+
+```text
+---
+title: Format Settings
+sidebar_label: Format Settings
+slug: /en/operations/settings/formats
+---
+
+<!--Do not edit – this file is autogenerated-->
+
+## Format Settings
+
+${data}
+```
+
+the `Template` format will insert the selected rows with the structure we define (as markdown in our case) in `row_settings.format`, replacing `${data}`.
+
+`row_settings.format` defines the structure of each individual setting entry and the file to use is defined by setting `format_template_row`:
+
+```text
+## ${0:XML}
+
+${1:XML}
+
+**Type**: ${2:XML}<br/>
+**Default value**: `${3:XML}`<br/>
+**Tier**: ${4:XML}<br/>
+```
+
+Where:
+- `${0:XML}` will insert the selected `name`
+- `${1:XML}` will insert the selected `description`
+- `${2:XML}` will insert the selected `type`
+- `${3:XML}` will insert the selected `default`
+- `${4:XML}` will insert the selected `tier`
+
+**Changing this file will change the layout for ALL generated markdown files**
+
+If different row formatting is needed for a specific markdown file, then please create a new `row_setttings_{page}.format` and specify to use this new file with the `format_template_row` setting in the query.
+

scripts/settings/resultset_format_settings.format

@@ -0,0 +1,11 @@
+---
+title: Format Settings
+sidebar_label: Format Settings
+slug: /en/operations/settings/formats
+---
+
+<!--Do not edit – this file is autogenerated-->
+
+## Format Settings
+
+${data}

scripts/settings/resultset_merge_tree_settings.format

@@ -0,0 +1,45 @@
+---
+slug: /en/operations/settings/merge-tree-settings
+title: "MergeTree Table Settings"
+---
+
+<!--Do not edit – this file it is autogenerated-->
+
+System table `system.merge_tree_settings` shows the globally set MergeTree settings.
+
+MergeTree settings can be set in the `merge_tree` section of the server config file, or specified for each `MergeTree` table individually in
+the `SETTINGS` clause of the `CREATE TABLE` statement.
+
+Example for customizing setting `max_suspicious_broken_parts`:
+
+Configure the default for all `MergeTree` tables in the server configuration file:
+
+``` text
+<merge_tree>
+    <max_suspicious_broken_parts>5</max_suspicious_broken_parts>
+</merge_tree>
+```
+
+Set for a particular table:
+
+``` sql
+CREATE TABLE tab
+(
+    `A` Int64
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS max_suspicious_broken_parts = 500;
+```
+
+Change the settings for a particular table using `ALTER TABLE ... MODIFY SETTING`:
+
+```sql
+ALTER TABLE tab MODIFY SETTING max_suspicious_broken_parts = 100;
+
+-- reset to global default (value from system.merge_tree_settings)
+ALTER TABLE tab RESET SETTING max_suspicious_broken_parts;
+```
+# Settings
+
+${data}

scripts/settings/resultset_system_settings.format

@@ -0,0 +1,12 @@
+---
+title: Core Settings
+sidebar_label: Core Settings
+slug: /en/operations/settings/settings
+toc_max_heading_level: 2
+---
+
+<!--Do not edit – this file is autogenerated-->
+
+## System Settings
+
+${data}

scripts/settings/row_settings.format

@@ -0,0 +1,7 @@
+## ${0:XML}
+
+${1:XML}
+
+**Type**: ${2:XML}<br/>
+**Default value**: `${3:XML}`<br/>
+**Tier**: ${4:XML}<br/>