#997: Added documentation for Rollback of Script-Languages-Container (#…

…999)

fixes #997

.github/workflows/check_rollback_7.1.30_script.yml

@@ -0,0 +1,92 @@
+name: Check Rollback Script 7.1.30
+
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+
+jobs:
+  Check:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - db-version: "7.1.29"
+            expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3'"
+            expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/exaudf/exaudfclient_py3';"
+            support-python2: "no"
+          - db-version: "7.1.19"
+            expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON=builtin_python R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3';"
+            expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 PYTHON=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/exaudf/exaudfclient R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/exaudf/exaudfclient_py3';"
+            support-python2: "yes"
+          - db-version: "7.1.6"
+            expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON=builtin_python R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3';"
+            expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 PYTHON=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/exaudf/exaudfclient R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/exaudf/exaudfclient_py3';"
+            support-python2: "yes"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: "3.10"
+          poetry-version: '1.8.2'
+      - name: Start ITDE
+        run: poetry run itde spawn-test-environment --environment-name exasol_test --database-port-forward 8888 --bucketfs-port-forward 6583 --docker-db-image-version "${{ matrix.db-version }}"
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y perl default-jre
+      - name: Install EXAPlus
+        run: |
+          curl -o exaplus.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/EXAplus-7.0.11.tar.gz && tar xf exaplus.tar.gz
+          echo "/tmp/EXAplus-7.0.11" >> $GITHUB_PATH
+        working-directory: /tmp
+      - name: Check if get_original_script_languages_parameters.sql works
+        run: |
+          RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./get_original_script_languages_parameter.sql`
+          echo "$RESULT" | grep  "$EXPECTED_ORIGINAL_ALTER_SYSTEM_PARAMETER" || (echo SCRIPT_LANGUAGES parameter not as expected && exit 1)
+        env:
+          EXPECTED_ORIGINAL_ALTER_SYSTEM_PARAMETER: "${{ matrix.expected-original-alter-system-parameter }}"
+
+        working-directory: doc/user_guide/resources
+      - name: Install a simple R UDF
+        run: |
+          sleep 120
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -f install_r_udf.sql
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.R_DEMO();"
+        working-directory: .github/workflows/scripts
+      - name: Install a simple Python2 UDF
+        if:  ${{ matrix.support-python2=='yes' }}
+        run: |
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -f install_python2_udf.sql
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.TEST_PYTHON_VERSION();"
+        working-directory: .github/workflows/scripts
+      - name: Check if pre_update_check.sql works for R only
+        if:  ${{ matrix.support-python2=='no' }}
+        run: |
+          RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./pre_update_check.sql`
+          echo "$RESULT" | grep "You have 1 UDFs using the pre-shipped R/Python2"
+        working-directory: doc/user_guide/resources
+      - name: Check if pre_update_check.sql works for R and Python2
+        if:  ${{ matrix.support-python2=='yes' }}
+        run: |
+          RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./pre_update_check.sql`
+          echo "$RESULT" | grep "You have 2 UDFs using the pre-shipped R/Python2"
+        working-directory: doc/user_guide/resources
+      - name: Run construct_alter_system_command_before_update.sql
+        run: |
+          RESULT=`exaplus -c localhost:8888 -u sys -p exasol -f ./construct_alter_system_command_before_update.sql`
+          echo "$RESULT" | tr -d '\n' | grep "$EXPECTED_NEW_ALTER_SYSTEM_PARAMETER" || (echo ALTER SYSTEM command not as expected && exit 1)
+          exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "$EXPECTED_NEW_ALTER_SYSTEM_PARAMETER"
+        working-directory: doc/user_guide/resources
+        env:
+          EXPECTED_NEW_ALTER_SYSTEM_PARAMETER: "${{ matrix.expected-new-alter-system-command }}"
+      - name: Test if R UDF still works
+        run: exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.R_DEMO();"
+      - name: Test if Python2 UDF still works
+        if:  ${{ matrix.support-python2=='yes' }}
+        run: exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.TEST_PYTHON_VERSION();"

.github/workflows/scripts/install_python2_udf.sql

@@ -0,0 +1,10 @@
+CREATE SCHEMA IF NOT EXISTS TEST;
+OPEN SCHEMA TEST;
+--/
+
+CREATE OR REPLACE PYTHON SCALAR SCRIPT "TEST_PYTHON_VERSION" () RETURNS VARCHAR(2000) AS
+import sys
+def run(ctx):
+    return sys.version
+/
+

.github/workflows/scripts/install_r_udf.sql

@@ -0,0 +1,10 @@
+CREATE SCHEMA IF NOT EXISTS TEST;
+OPEN SCHEMA TEST;
+
+--/
+CREATE OR REPLACE R SCALAR SCRIPT TEST.R_DEMO() RETURNS VARCHAR(2000) AS
+
+run <- function(ctx) {
+ "Minimal R language UDF"
+}
+/

doc/user_guide/resources/construct_alter_system_command_before_update.sql

@@ -0,0 +1,66 @@
+with minor_version as (
+  select CAST(SUBSTR(PARAM_VALUE,INSTR(PARAM_VALUE,'.',-1,1)+1) as INT) as minor_version
+  from  EXA_METADATA
+  where PARAM_NAME='databaseProductVersion' and PARAM_VALUE like '7.1%'
+),
+container_name as (
+  select 'ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36' as container_name
+  from minor_version
+  where minor_version>=20
+  union all
+  select 'ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R' as container_name
+  from minor_version
+  where minor_version>=7 AND minor_version<20
+  union all
+  select 'ScriptLanguages-release-standard-exasol-7.1.0-1.1.0' as container_name
+  from minor_version
+  where minor_version<7
+
+),
+num_of_pairs as(
+  SELECT
+    p.system_value,
+    length(p.system_value)-length(replace(p.system_value, '=')) as num_of_pairs
+  FROM exa_parameters p
+  WHERE p.parameter_name in ('SCRIPT_LANGUAGES')
+),
+lang_pairs as(
+  SELECT
+    regexp_substr(nop.system_value, '[^ =]+=[^ ]+', 1, level) as pair_val,
+    nop.system_value
+  FROM num_of_pairs nop
+  connect by level <= nop.num_of_pairs
+),
+parsed_lang_pairs as(
+  SELECT
+    instr(lp.pair_val, '=') as eq_pos,
+    substr(lp.pair_val, 1, local.eq_pos-1) as alias_name,
+    substr(lp.pair_val, local.eq_pos + 1, length(lp.pair_val) - local.eq_pos) as alias_content
+  FROM lang_pairs lp
+),
+new_alias_content as (
+  select
+    alias_name,
+    'localzmq+protobuf:///bfsdefault/default/EXAClusterOS/' || container_name || '/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/' || container_name || '/exaudf/exaudfclient_py3'  as alias_content
+  from container_name
+  join parsed_lang_pairs on true
+  where alias_content='builtin_r'
+  union all
+  select
+    alias_name,
+    'localzmq+protobuf:///bfsdefault/default/EXAClusterOS/' || container_name || '/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/' || container_name || '/exaudf/exaudfclient'  as alias_content
+  from container_name
+  join parsed_lang_pairs on true
+  where alias_content='builtin_python'
+  union all
+  select
+    alias_name,
+    alias_content
+  from parsed_lang_pairs
+  where alias_content<>'builtin_r' and alias_content<>'builtin_python'
+),
+new_system_value as (
+  select GROUP_CONCAT(alias_name || '=' || alias_content SEPARATOR ' ') as new_system_value
+  from new_alias_content
+)
+select 'ALTER SYSTEM SET SCRIPT_LANGUAGES=''' || new_system_value || ''';' as command_text from new_system_value;

doc/user_guide/resources/get_original_script_languages_parameter.sql

@@ -0,0 +1,3 @@
+SELECT 'ALTER SYSTEM SET SCRIPT_LANGUAGES=''' || system_value || ''';'
+FROM  EXA_PARAMETERS
+WHERE PARAMETER_NAME='SCRIPT_LANGUAGES';

doc/user_guide/resources/pre_update_check.sql

@@ -0,0 +1,45 @@
+with
+num_of_pairs as(
+    SELECT
+        p.system_value
+        , length(p.system_value)-length(replace(p.system_value, '=')) as num_of_pairs
+    FROM
+        exa_parameters p
+    WHERE
+        1=1
+        and p.parameter_name in ('SCRIPT_LANGUAGES')
+)
+, lang_pairs as(
+    SELECT
+        regexp_substr(nop.system_value, '[^ =]+=[^ ]+', 1, level) as pair_val
+        , nop.system_value
+    FROM
+        num_of_pairs nop
+    connect by
+        level <= nop.num_of_pairs
+)
+, parsed as(
+    SELECT
+        lp.*
+        , instr(lp.pair_val, '=') as eq_pos
+        , substr(lp.pair_val, 1, local.eq_pos-1) as alias_name
+        , substr(lp.pair_val, local.eq_pos + 1, length(lp.pair_val) - local.eq_pos) as alias_content
+    FROM
+        lang_pairs lp
+)
+select
+    case
+        when count(*) = 0 then 'You are not using the pre-shipped R/Python2 in UDFs.'
+        else 'You have '||to_char(count(*))||' UDFs using the pre-shipped R/Python2:
+
+' || GROUP_CONCAT('"'||s.script_schema||'"."'||s.script_name||'"' separator '
+')
+    end as check_results
+FROM
+    parsed p
+        join exa_dba_scripts s
+        on upper(p.alias_name)=s.script_language
+WHERE
+    1=1
+    and (p.alias_content = 'builtin_r' or p.alias_content = 'builtin_python')
+;

doc/user_guide/use_R3_and_Python2_on_7.1.30_and_above.md

@@ -0,0 +1,72 @@
+# Use R3.6 UDF's with Exasol 7.1.30 and later
+
+## Background
+
+The Script-Languages-Container was updated to version [8.1.0](https://github.com/exasol/script-languages-release/releases/tag/8.1.0) in Database v7.1.30. Release 8.1.0 included a major update for the Python and R languages compared to 6.0.0. For R we detected some performance drops in special circumstances: Newer versions of R have a higher memory consumption which can lead to throttling of the UDFs and decrease their performance. 
+Also, some R packages were removed. 
+In case, customers want to use the previous Script-Languages-Container [6.0.0](https://github.com/exasol/script-languages-release/releases/tag/6.0.0), they can follow the steps described in this document. Alternatively, if the customer experiences performance drops with R UDF's, [limiting the number](https://docs.exasol.com/db/latest/database_concepts/udf_scripts/udf_instance_limit.htm) of UDF instances might avoid this problem.
+
+⚠️These script language containers are provided as is and are not maintained anymore.
+
+## Pre-update Check
+
+ⓘ This script will detect most occurrences of R and Python2 usage in UDFs. We cannot guarantee that all occurrences are detected.
+
+See [pre_update_check.sql](./resources/pre_update_check.sql)
+
+
+## Update From 7.1.x To 7.1.30 or newer
+
+To use Python 2.7 or R3.4 with UDFs after updating from Exasol 7.1.x to Exasol 7.1.30 or later you need to change the SCRIPT_LANGUAGES parameter to activate a script language container that supports Python 2.7/R3.4. The new parameter value can be generated automatically or manually.
+
+### Automatic generation of the new SCRIPT_LANGUAGES parameter value
+
+1. Get the current value of the SCRIPT_LANGUAGES parameter and save the output in case you need to restore it.
+
+See [get_original_script_languages_parameter.sql](./resources/get_original_script_languages_parameter.sql)
+
+Expected result for an Exasol 7.1.20 database or later (the example shows the default values - actual values may be different if the parameter has been changed):
+```sql
+ALTER SYSTEM SET SCRIPT_LANGUAGES='R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3';
+```
+Expected result for an Exasol 7.1.19 database or lower (the example shows the default values - actual values may be different if the parameter has been changed):
+```sql
+ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON=builtin_python R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3';
+```
+
+2. Run the following query before updating to 7.1.30:
+
+See [construct_alter_system_command_before_update.sql](./resources/construct_alter_system_command_before_update.sql)
+
+3. Run the ALTER SYSTEM statement returned by the query. For example:
+
+```sql
+ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/exaudf/exaudfclient_py3';
+```
+ⓘ We recommend that you test parameter changes using ALTER SESSION before making system-wide changes using ALTER SYSTEM.
+
+### Manual generation of the new SCRIPT_LANGUAGES parameter value
+
+1. Get the current value of the SCRIPT_LANGUAGES parameter and save the output in case you need to restore it.
+
+See [get_original_script_languages_parameter.sql](./resources/get_original_script_languages_parameter.sql)
+
+2. Using the output from the query in the previous step, replace:
+- `builtin_r` with `localzmq+protobuf:///bfsdefault/default/EXAClusterOS/<container_name>/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/<container_name>/exaudf/exaudfclient_py3`
+- `builtin_python` with `localzmq+protobuf:///bfsdefault/default/EXAClusterOS/<container_name>/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/<container_name>/exaudf/exaudfclient` (in case you need Python2 in UDF's)
+
+Replace <container_name> in this string with the following value depending on which version you are upgrading from:
+
+| Exasol DB Min Version        | Exasol DB Max Version            | Container Name                                           |
+|:-----------------------------|:---------------------------------|:---------------------------------------------------------|
+|  7.1.20                   | 7.1.29	                       |ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36 |
+|  7.1.7                    | 7.1.19	                       |ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R |
+|  7.1.0                    | 7.1.6	                           |ScriptLanguages-release-standard-exasol-7.1.0-1.1.0       |
+
+ⓘ If you already had updated the `SCRIPT_LANGUAGES` parameter during a previous update, following steps in https://docs.exasol.com/db/7.1/database_concepts/udf_scripts/python2_extended_use.htm, leave the value for Python2 unchanged.
+
+3. Run the updated ALTER SYSTEM statement after changing the values as described in the previous step. For example:
+```sql
+ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/exaudf/exaudfclient_py3 PYTHON=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/exaudf/exaudfclient';
+```
+ⓘ We recommend that you test parameter changes using ALTER SESSION before making system-wide changes using ALTER SYSTEM.

doc/user_guide/user_guide.md

@@ -6,3 +6,4 @@
 - [Python Dataframe Support](py_dataframe.md)
 - [Using Vagrant as development environment](vagrant.md)
 - [FAQ](FAQ.md)
+- [Use R3.x and Python2 on an Exasol 7.1.3.0 database and above](use_R3_and_Python2_on_7.1.30_and_above.md)