diff --git a/.github/workflows/check_rollback_7.1.30_script.yml b/.github/workflows/check_rollback_7.1.30_script.yml new file mode 100644 index 00000000..dd3d6092 --- /dev/null +++ b/.github/workflows/check_rollback_7.1.30_script.yml @@ -0,0 +1,92 @@ +name: Check Rollback Script 7.1.30 + +on: + push: + branches: + - develop + pull_request: + +jobs: + Check: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + include: + - db-version: "7.1.29" + expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3'" + expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/exaudf/exaudfclient_py3';" + support-python2: "no" + - db-version: "7.1.19" + expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON=builtin_python R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3';" + expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 PYTHON=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/exaudf/exaudfclient R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/exaudf/exaudfclient_py3';" + support-python2: "yes" + - db-version: "7.1.6" + expected-original-alter-system-parameter: "ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON=builtin_python R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3';" + expected-new-alter-system-command: "ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 PYTHON=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/exaudf/exaudfclient R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-release-standard-exasol-7.1.0-1.1.0/exaudf/exaudfclient_py3';" + support-python2: "yes" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.12.0 + with: + python-version: "3.10" + poetry-version: '1.8.2' + - name: Start ITDE + run: poetry run itde spawn-test-environment --environment-name exasol_test --database-port-forward 8888 --bucketfs-port-forward 6583 --docker-db-image-version "${{ matrix.db-version }}" + - name: Install dependencies + run: sudo apt update && sudo apt install -y perl default-jre + - name: Install EXAPlus + run: | + curl -o exaplus.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/EXAplus-7.0.11.tar.gz && tar xf exaplus.tar.gz + echo "/tmp/EXAplus-7.0.11" >> $GITHUB_PATH + working-directory: /tmp + - name: Check if get_original_script_languages_parameters.sql works + run: | + RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./get_original_script_languages_parameter.sql` + echo "$RESULT" | grep "$EXPECTED_ORIGINAL_ALTER_SYSTEM_PARAMETER" || (echo SCRIPT_LANGUAGES parameter not as expected && exit 1) + env: + EXPECTED_ORIGINAL_ALTER_SYSTEM_PARAMETER: "${{ matrix.expected-original-alter-system-parameter }}" + + working-directory: doc/user_guide/resources + - name: Install a simple R UDF + run: | + sleep 120 + exaplus -x -q -c localhost:8888 -u sys -p exasol -f install_r_udf.sql + exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.R_DEMO();" + working-directory: .github/workflows/scripts + - name: Install a simple Python2 UDF + if: ${{ matrix.support-python2=='yes' }} + run: | + exaplus -x -q -c localhost:8888 -u sys -p exasol -f install_python2_udf.sql + exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.TEST_PYTHON_VERSION();" + working-directory: .github/workflows/scripts + - name: Check if pre_update_check.sql works for R only + if: ${{ matrix.support-python2=='no' }} + run: | + RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./pre_update_check.sql` + echo "$RESULT" | grep "You have 1 UDFs using the pre-shipped R/Python2" + working-directory: doc/user_guide/resources + - name: Check if pre_update_check.sql works for R and Python2 + if: ${{ matrix.support-python2=='yes' }} + run: | + RESULT=`exaplus -x -q -c localhost:8888 -u sys -p exasol -f ./pre_update_check.sql` + echo "$RESULT" | grep "You have 2 UDFs using the pre-shipped R/Python2" + working-directory: doc/user_guide/resources + - name: Run construct_alter_system_command_before_update.sql + run: | + RESULT=`exaplus -c localhost:8888 -u sys -p exasol -f ./construct_alter_system_command_before_update.sql` + echo "$RESULT" | tr -d '\n' | grep "$EXPECTED_NEW_ALTER_SYSTEM_PARAMETER" || (echo ALTER SYSTEM command not as expected && exit 1) + exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "$EXPECTED_NEW_ALTER_SYSTEM_PARAMETER" + working-directory: doc/user_guide/resources + env: + EXPECTED_NEW_ALTER_SYSTEM_PARAMETER: "${{ matrix.expected-new-alter-system-command }}" + - name: Test if R UDF still works + run: exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.R_DEMO();" + - name: Test if Python2 UDF still works + if: ${{ matrix.support-python2=='yes' }} + run: exaplus -x -q -c localhost:8888 -u sys -p exasol -sql "SELECT TEST.TEST_PYTHON_VERSION();" \ No newline at end of file diff --git a/.github/workflows/scripts/install_python2_udf.sql b/.github/workflows/scripts/install_python2_udf.sql new file mode 100644 index 00000000..d9b7929b --- /dev/null +++ b/.github/workflows/scripts/install_python2_udf.sql @@ -0,0 +1,10 @@ +CREATE SCHEMA IF NOT EXISTS TEST; +OPEN SCHEMA TEST; +--/ + +CREATE OR REPLACE PYTHON SCALAR SCRIPT "TEST_PYTHON_VERSION" () RETURNS VARCHAR(2000) AS +import sys +def run(ctx): + return sys.version +/ + diff --git a/.github/workflows/scripts/install_r_udf.sql b/.github/workflows/scripts/install_r_udf.sql new file mode 100644 index 00000000..69fb0415 --- /dev/null +++ b/.github/workflows/scripts/install_r_udf.sql @@ -0,0 +1,10 @@ +CREATE SCHEMA IF NOT EXISTS TEST; +OPEN SCHEMA TEST; + +--/ +CREATE OR REPLACE R SCALAR SCRIPT TEST.R_DEMO() RETURNS VARCHAR(2000) AS + +run <- function(ctx) { + "Minimal R language UDF" +} +/ diff --git a/doc/user_guide/resources/construct_alter_system_command_before_update.sql b/doc/user_guide/resources/construct_alter_system_command_before_update.sql new file mode 100644 index 00000000..96518749 --- /dev/null +++ b/doc/user_guide/resources/construct_alter_system_command_before_update.sql @@ -0,0 +1,66 @@ +with minor_version as ( + select CAST(SUBSTR(PARAM_VALUE,INSTR(PARAM_VALUE,'.',-1,1)+1) as INT) as minor_version + from EXA_METADATA + where PARAM_NAME='databaseProductVersion' and PARAM_VALUE like '7.1%' +), +container_name as ( + select 'ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36' as container_name + from minor_version + where minor_version>=20 + union all + select 'ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R' as container_name + from minor_version + where minor_version>=7 AND minor_version<20 + union all + select 'ScriptLanguages-release-standard-exasol-7.1.0-1.1.0' as container_name + from minor_version + where minor_version<7 + +), +num_of_pairs as( + SELECT + p.system_value, + length(p.system_value)-length(replace(p.system_value, '=')) as num_of_pairs + FROM exa_parameters p + WHERE p.parameter_name in ('SCRIPT_LANGUAGES') +), +lang_pairs as( + SELECT + regexp_substr(nop.system_value, '[^ =]+=[^ ]+', 1, level) as pair_val, + nop.system_value + FROM num_of_pairs nop + connect by level <= nop.num_of_pairs +), +parsed_lang_pairs as( + SELECT + instr(lp.pair_val, '=') as eq_pos, + substr(lp.pair_val, 1, local.eq_pos-1) as alias_name, + substr(lp.pair_val, local.eq_pos + 1, length(lp.pair_val) - local.eq_pos) as alias_content + FROM lang_pairs lp +), +new_alias_content as ( + select + alias_name, + 'localzmq+protobuf:///bfsdefault/default/EXAClusterOS/' || container_name || '/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/' || container_name || '/exaudf/exaudfclient_py3' as alias_content + from container_name + join parsed_lang_pairs on true + where alias_content='builtin_r' + union all + select + alias_name, + 'localzmq+protobuf:///bfsdefault/default/EXAClusterOS/' || container_name || '/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/' || container_name || '/exaudf/exaudfclient' as alias_content + from container_name + join parsed_lang_pairs on true + where alias_content='builtin_python' + union all + select + alias_name, + alias_content + from parsed_lang_pairs + where alias_content<>'builtin_r' and alias_content<>'builtin_python' +), +new_system_value as ( + select GROUP_CONCAT(alias_name || '=' || alias_content SEPARATOR ' ') as new_system_value + from new_alias_content +) +select 'ALTER SYSTEM SET SCRIPT_LANGUAGES=''' || new_system_value || ''';' as command_text from new_system_value; diff --git a/doc/user_guide/resources/get_original_script_languages_parameter.sql b/doc/user_guide/resources/get_original_script_languages_parameter.sql new file mode 100644 index 00000000..4cb58b26 --- /dev/null +++ b/doc/user_guide/resources/get_original_script_languages_parameter.sql @@ -0,0 +1,3 @@ +SELECT 'ALTER SYSTEM SET SCRIPT_LANGUAGES=''' || system_value || ''';' +FROM EXA_PARAMETERS +WHERE PARAMETER_NAME='SCRIPT_LANGUAGES'; diff --git a/doc/user_guide/resources/pre_update_check.sql b/doc/user_guide/resources/pre_update_check.sql new file mode 100644 index 00000000..29d19a9d --- /dev/null +++ b/doc/user_guide/resources/pre_update_check.sql @@ -0,0 +1,45 @@ +with +num_of_pairs as( + SELECT + p.system_value + , length(p.system_value)-length(replace(p.system_value, '=')) as num_of_pairs + FROM + exa_parameters p + WHERE + 1=1 + and p.parameter_name in ('SCRIPT_LANGUAGES') +) +, lang_pairs as( + SELECT + regexp_substr(nop.system_value, '[^ =]+=[^ ]+', 1, level) as pair_val + , nop.system_value + FROM + num_of_pairs nop + connect by + level <= nop.num_of_pairs +) +, parsed as( + SELECT + lp.* + , instr(lp.pair_val, '=') as eq_pos + , substr(lp.pair_val, 1, local.eq_pos-1) as alias_name + , substr(lp.pair_val, local.eq_pos + 1, length(lp.pair_val) - local.eq_pos) as alias_content + FROM + lang_pairs lp +) +select + case + when count(*) = 0 then 'You are not using the pre-shipped R/Python2 in UDFs.' + else 'You have '||to_char(count(*))||' UDFs using the pre-shipped R/Python2: + +' || GROUP_CONCAT('"'||s.script_schema||'"."'||s.script_name||'"' separator ' +') + end as check_results +FROM + parsed p + join exa_dba_scripts s + on upper(p.alias_name)=s.script_language +WHERE + 1=1 + and (p.alias_content = 'builtin_r' or p.alias_content = 'builtin_python') +; diff --git a/doc/user_guide/use_R3_and_Python2_on_7.1.30_and_above.md b/doc/user_guide/use_R3_and_Python2_on_7.1.30_and_above.md new file mode 100644 index 00000000..fa3ab7c1 --- /dev/null +++ b/doc/user_guide/use_R3_and_Python2_on_7.1.30_and_above.md @@ -0,0 +1,72 @@ +# Use R3.6 UDF's with Exasol 7.1.30 and later + +## Background + +The Script-Languages-Container was updated to version [8.1.0](https://github.com/exasol/script-languages-release/releases/tag/8.1.0) in Database v7.1.30. Release 8.1.0 included a major update for the Python and R languages compared to 6.0.0. For R we detected some performance drops in special circumstances: Newer versions of R have a higher memory consumption which can lead to throttling of the UDFs and decrease their performance. +Also, some R packages were removed. +In case, customers want to use the previous Script-Languages-Container [6.0.0](https://github.com/exasol/script-languages-release/releases/tag/6.0.0), they can follow the steps described in this document. Alternatively, if the customer experiences performance drops with R UDF's, [limiting the number](https://docs.exasol.com/db/latest/database_concepts/udf_scripts/udf_instance_limit.htm) of UDF instances might avoid this problem. + +⚠️These script language containers are provided as is and are not maintained anymore. + +## Pre-update Check + +ⓘ This script will detect most occurrences of R and Python2 usage in UDFs. We cannot guarantee that all occurrences are detected. + +See [pre_update_check.sql](./resources/pre_update_check.sql) + + +## Update From 7.1.x To 7.1.30 or newer + +To use Python 2.7 or R3.4 with UDFs after updating from Exasol 7.1.x to Exasol 7.1.30 or later you need to change the SCRIPT_LANGUAGES parameter to activate a script language container that supports Python 2.7/R3.4. The new parameter value can be generated automatically or manually. + +### Automatic generation of the new SCRIPT_LANGUAGES parameter value + +1. Get the current value of the SCRIPT_LANGUAGES parameter and save the output in case you need to restore it. + +See [get_original_script_languages_parameter.sql](./resources/get_original_script_languages_parameter.sql) + +Expected result for an Exasol 7.1.20 database or later (the example shows the default values - actual values may be different if the parameter has been changed): +```sql +ALTER SYSTEM SET SCRIPT_LANGUAGES='R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3'; +``` +Expected result for an Exasol 7.1.19 database or lower (the example shows the default values - actual values may be different if the parameter has been changed): +```sql +ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON=builtin_python R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3'; +``` + +2. Run the following query before updating to 7.1.30: + +See [construct_alter_system_command_before_update.sql](./resources/construct_alter_system_command_before_update.sql) + +3. Run the ALTER SYSTEM statement returned by the query. For example: + +```sql +ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/exaudf/exaudfclient_py3'; +``` +ⓘ We recommend that you test parameter changes using ALTER SESSION before making system-wide changes using ALTER SYSTEM. + +### Manual generation of the new SCRIPT_LANGUAGES parameter value + +1. Get the current value of the SCRIPT_LANGUAGES parameter and save the output in case you need to restore it. + +See [get_original_script_languages_parameter.sql](./resources/get_original_script_languages_parameter.sql) + +2. Using the output from the query in the previous step, replace: +- `builtin_r` with `localzmq+protobuf:///bfsdefault/default/EXAClusterOS//?lang=r#/buckets/bfsdefault/default/EXAClusterOS//exaudf/exaudfclient_py3` +- `builtin_python` with `localzmq+protobuf:///bfsdefault/default/EXAClusterOS//?lang=r#/buckets/bfsdefault/default/EXAClusterOS//exaudf/exaudfclient` (in case you need Python2 in UDF's) + +Replace in this string with the following value depending on which version you are upgrading from: + +| Exasol DB Min Version | Exasol DB Max Version | Container Name | +|:-----------------------------|:---------------------------------|:---------------------------------------------------------| +| 7.1.20 | 7.1.29 |ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36 | +| 7.1.7 | 7.1.19 |ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R | +| 7.1.0 | 7.1.6 |ScriptLanguages-release-standard-exasol-7.1.0-1.1.0 | + +ⓘ If you already had updated the `SCRIPT_LANGUAGES` parameter during a previous update, following steps in https://docs.exasol.com/db/7.1/database_concepts/udf_scripts/python2_extended_use.htm, leave the value for Python2 unchanged. + +3. Run the updated ALTER SYSTEM statement after changing the values as described in the previous step. For example: +```sql +ALTER SYSTEM SET SCRIPT_LANGUAGES='JAVA=builtin_java PYTHON3=builtin_python3 R=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/?lang=r#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v6.0.0-VYP23K36/exaudf/exaudfclient_py3 PYTHON=localzmq+protobuf:///bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/?lang=python#/buckets/bfsdefault/default/EXAClusterOS/ScriptLanguages-standard-EXASOL-7.1.0-slc-v4.0.0-CM4RWW6R/exaudf/exaudfclient'; +``` +ⓘ We recommend that you test parameter changes using ALTER SESSION before making system-wide changes using ALTER SYSTEM. diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 413d8af1..18dd2ebd 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -6,3 +6,4 @@ - [Python Dataframe Support](py_dataframe.md) - [Using Vagrant as development environment](vagrant.md) - [FAQ](FAQ.md) +- [Use R3.x and Python2 on an Exasol 7.1.3.0 database and above](use_R3_and_Python2_on_7.1.30_and_above.md)