From a2a2f0bf0be31e7897beb934542e43ddbb609567 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:08:35 -0300 Subject: [PATCH] #969: Use new CTPG parser in java vm (#455) related to exasol/script-languages-release#969 --------- Co-authored-by: Torsten Kilias --- .github/workflows/check_bazel_tests.yml | 104 +++++++++---- .../test_package_management_scripts.yaml | 2 + exaudfclient/.bazelrc | 13 ++ exaudfclient/BUILD | 4 +- exaudfclient/base/javacontainer/BUILD | 7 +- .../base/javacontainer/javacontainer.cc | 5 +- .../base/javacontainer/javacontainer.h | 13 +- .../javacontainer/javacontainer_builder.cc | 32 ++++ .../javacontainer/javacontainer_builder.h | 35 +++++ .../base/javacontainer/javacontainer_impl.cc | 37 ++--- .../base/javacontainer/javacontainer_impl.h | 13 +- .../base/javacontainer/script_options/BUILD | 8 +- .../javacontainer/script_options/extractor.cc | 10 +- .../javacontainer/script_options/extractor.h | 7 +- .../javacontainer/script_options/keywords.cc | 33 ++++ .../javacontainer/script_options/keywords.h | 12 +- .../javacontainer/script_options/parser.h | 5 +- .../script_options/parser_ctpg.cc | 142 ++++++++++++++++++ .../script_options/parser_ctpg.h | 41 +++++ .../parser_ctpg_script_importer.cc | 99 ++++++++++++ .../parser_ctpg_script_importer.h | 46 ++++++ .../script_options/parser_legacy.cc | 15 +- .../script_options/parser_legacy.h | 14 +- .../javacontainer/script_options/test/BUILD | 10 ++ .../test/ctpg_script_importer_test.cc | 66 ++++++++ .../script_options/test/swig_factory_test.cc | 105 +++++++++++++ .../script_options/test/swig_factory_test.h | 20 +++ exaudfclient/base/javacontainer/test/BUILD | 45 +++++- .../test/cpp/javacontainer_ctpg_test.cc | 109 ++++++++++++++ .../test/cpp/javacontainer_perf_test.cc | 27 ++++ .../test/cpp/javacontainer_test.cc | 58 +++---- .../javacontainer/test/cpp/javavm_test.cc | 27 ++-- .../base/javacontainer/test/cpp/javavm_test.h | 5 +- .../test/cpp/swig_factory_test.cc | 4 +- .../ctpg/script_option_lines_ctpg.cc | 47 +++--- .../ctpg/script_option_lines_ctpg.h | 2 - .../ctpg/test/script_option_lines_test.cpp | 4 + exaudfclient/base/swig_factory/swig_factory.h | 1 + exaudfclient/exaudfclient.cc | 7 +- scripts/installUdfClientDeps.sh | 56 +++++++ 40 files changed, 1123 insertions(+), 167 deletions(-) create mode 100644 exaudfclient/base/javacontainer/javacontainer_builder.cc create mode 100644 exaudfclient/base/javacontainer/javacontainer_builder.h create mode 100644 exaudfclient/base/javacontainer/script_options/keywords.cc create mode 100644 exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc create mode 100644 exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h create mode 100644 exaudfclient/base/javacontainer/script_options/test/BUILD create mode 100644 exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc create mode 100644 exaudfclient/base/javacontainer/script_options/test/swig_factory_test.cc create mode 100644 exaudfclient/base/javacontainer/script_options/test/swig_factory_test.h create mode 100644 exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc create mode 100644 exaudfclient/base/javacontainer/test/cpp/javacontainer_perf_test.cc create mode 100644 scripts/installUdfClientDeps.sh diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index def9fd7a..9d0e28dc 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -6,38 +6,84 @@ on: - master pull_request: - +env: + USE_BAZEL_VERSION: 7.2.1 + UDF_CLIENT_ENV_FILE: /tmp/.udf_client_env jobs: build: - runs-on: ubuntu-latest - - env: - USE_BAZEL_VERSION: 7.2.1 + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - name: Search for duplicated error codes + run: bash find_duplicate_error_codes.sh + - name: Install UDF Client Dependencies + run: | + sudo bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" + - name: Build + run: | + source "$UDF_CLIENT_ENV_FILE" + bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --verbose_failures + working-directory: ./exaudfclient/ - - name: Search for duplicated error codes - run: bash find_duplicate_error_codes.sh - - - name: Install bazel - run: | - curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x bazel - - name: Install JDK and ZMQ - run: | - sudo apt-get update - sudo apt-get install -y openjdk-11-jdk libzmq3-dev - - name: Java Tests - run: | - bazel test //base/javacontainer/test/... - working-directory: ./exaudfclient/ - - name: ExaudfLib Tests - run: | - bazel test //base/exaudflib/test/... - working-directory: ./exaudfclient/ - - name: Script Options Parser Tests - run: | - bazel test //base/script_options_parser/... - working-directory: ./exaudfclient/ + tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - test: "//base/javacontainer/test:ExaStackTraceCleanerTest" + name: "ExaStackTraceCleanerTest" + - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" + name: "javacontainer-test-legacy-parser" + - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" + name: "javacontainer-test-ctpg-parser" + - test: "//base/javacontainer/script_options/..." + name: "javacontainer-script_options" + - test: "//base/exaudflib/test/..." + name: "exaudflib" + - test: "//base/script_options_parser/ctpg/..." + name: "script_options_parser_ctpg" + - test: "//base/script_options_parser/legacy/..." + name: "script_options_parser_legacy" + - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/javacontainer/test:javacontainer-test-legacy-parser" + name: "javacontainer-test-legacy-parser-with-valgrind" + - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/javacontainer/test:javacontainer-test-ctpg-parser" + name: "javacontainer-test-ctpg-parser-with-valgrind" + - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/script_options_parser/ctpg/..." + name: "script_options_parser_ctpg_with_valgrind" + - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/script_options_parser/legacy/..." + name: "script_options_parser_legacy_with_valgrind" + - test: "--config=asan //base/javacontainer/test:javacontainer-test-legacy-parser" + name: "javacontainer-test-legacy-parser-with-asan" + - test: "--config=asan //base/javacontainer/test:javacontainer-test-ctpg-parser" + name: "javacontainer-test-ctpg-parser-with-asan" + - test: "--config=asan //base/script_options_parser/ctpg/..." + name: "script_options_parser_ctpg_with_asan" + - test: "--config=asan //base/script_options_parser/legacy/..." + name: "script_options_parser_legacy_with_asan" + steps: + - uses: actions/checkout@v4 + - name: Install JDK and ZMQ + run: | + sudo apt-get update + sudo apt-get install -y openjdk-11-jdk libzmq3-dev valgrind + - name: Install bazel + run: | + BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" + BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" + curl -L --output /tmp/"$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" + sudo apt install -y "/tmp/$BAZEL_PACKAGE_FILE" + rm "/tmp/$BAZEL_PACKAGE_FILE" + env: + BAZEL_PACKAGE_VERSION: "7.2.1" + - name: Run tests + run: | + bazel test ${{ matrix.additional_args }} ${{ matrix.test }} + working-directory: ./exaudfclient/ + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: "${{ matrix.name }}" + path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/**/test.log diff --git a/.github/workflows/test_package_management_scripts.yaml b/.github/workflows/test_package_management_scripts.yaml index 8757331a..32ec2a3f 100644 --- a/.github/workflows/test_package_management_scripts.yaml +++ b/.github/workflows/test_package_management_scripts.yaml @@ -17,6 +17,8 @@ jobs: with: python-version: "3.10" + - uses: r-lib/actions/setup-r@v2 + - name: Init submodules run: git submodule update --init --recursive diff --git a/exaudfclient/.bazelrc b/exaudfclient/.bazelrc index 248128c7..fe0d63ac 100644 --- a/exaudfclient/.bazelrc +++ b/exaudfclient/.bazelrc @@ -13,3 +13,16 @@ build:optimize --copt="-g0" --copt="-DNDEBUG" --copt=-fstack-protector-strong -- build:no-tty --curses=no --color=no build:debug-build --sandbox_debug --config=verbose build:no-symlinks --symlink_prefix=/ +build:asan --strip=never +build:asan --copt -fsanitize=address +build:asan --copt -DADDRESS_SANITIZER +build:asan --copt -O1 +build:asan --copt -g +build:asan --copt -fno-omit-frame-pointer +build:asan --linkopt -fsanitize=address +build:asan -c dbg +build:valgrind --copt -O1 +build:valgrind -c dbg +build:valgrind --copt -g +build:valgrind --strip=never +build:valgrind --copt -DVALGRIND_ACTIVE \ No newline at end of file diff --git a/exaudfclient/BUILD b/exaudfclient/BUILD index 656ed774..1c609375 100644 --- a/exaudfclient/BUILD +++ b/exaudfclient/BUILD @@ -78,7 +78,7 @@ cc_binary( srcs = ["exaudfclient.cc", "//base:load_dynamic"], linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace deps = ["//base/exaudflib:header", "//base/utils:utils"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ - ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"], + ["//base/exaudflib:exaudflib-deps"], defines = VM_ENABLED_DEFINES, data = ["//base:libexaudflib_complete.so"] ) @@ -101,7 +101,7 @@ cc_binary( srcs = ["exaudfclient.cc", "//base:load_dynamic"], linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace deps = ["//base/exaudflib:header", "//base/utils:utils"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ - ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"] + + ["//base/exaudflib:exaudflib-deps"] + [ "@zmq//:zmq", "@protobuf//:protobuf"], defines = VM_ENABLED_DEFINES, data = ["//base:libexaudflib_complete.so"], diff --git a/exaudfclient/base/javacontainer/BUILD b/exaudfclient/base/javacontainer/BUILD index fa42cd31..d63bbf5b 100644 --- a/exaudfclient/base/javacontainer/BUILD +++ b/exaudfclient/base/javacontainer/BUILD @@ -106,11 +106,12 @@ cc_library( cc_library( name = "javacontainer", - srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", ":dummy"], + srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", + ":javacontainer_builder.h", ":javacontainer_builder.cc", ":dummy"], hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], - deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//base/exaudflib:header", + deps = ["@java//:java", ":exascript_java", "//base/exaudflib:header", "//base/utils:utils","//base/javacontainer/script_options:java_script_option_lines", - "//base/swig_factory:swig_factory_if"], + "//base/swig_factory:swig_factory"], # copts= ["-O0","-fno-lto"], alwayslink=True, ) diff --git a/exaudfclient/base/javacontainer/javacontainer.cc b/exaudfclient/base/javacontainer/javacontainer.cc index 797506cb..f8de356c 100644 --- a/exaudfclient/base/javacontainer/javacontainer.cc +++ b/exaudfclient/base/javacontainer/javacontainer.cc @@ -1,12 +1,13 @@ #include "base/javacontainer/javacontainer.h" #include "base/javacontainer/javacontainer_impl.h" +#include "base/javacontainer/script_options/parser.h" using namespace SWIGVMContainers; using namespace std; -JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory) { +JavaVMach::JavaVMach(bool checkOnly, std::unique_ptr scriptOptionsParser) { try { - m_impl = new JavaVMImpl(checkOnly, false, swigFactory); + m_impl = new JavaVMImpl(checkOnly, false, std::move(scriptOptionsParser)); } catch (std::exception& err) { lock_guard lock(exception_msg_mtx); exception_msg = "F-UDF-CL-SL-JAVA-1000: "+std::string(err.what()); diff --git a/exaudfclient/base/javacontainer/javacontainer.h b/exaudfclient/base/javacontainer/javacontainer.h index dc7ba418..c8c1b91f 100644 --- a/exaudfclient/base/javacontainer/javacontainer.h +++ b/exaudfclient/base/javacontainer/javacontainer.h @@ -4,17 +4,26 @@ #include "base/exaudflib/vm/swig_vm.h" #include +#include #ifdef ENABLE_JAVA_VM namespace SWIGVMContainers { class JavaVMImpl; -class SwigFactory; + +namespace JavaScriptOptions { + +struct ScriptOptionsParser; + +} class JavaVMach: public SWIGVM { public: - JavaVMach(bool checkOnly, SwigFactory& swigFactory); + /* + * scriptOptionsParser: JavaVMach takes ownership of ScriptOptionsParser pointer. + */ + JavaVMach(bool checkOnly, std::unique_ptr scriptOptionsParser); virtual ~JavaVMach() {} virtual void shutdown(); virtual bool run(); diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.cc b/exaudfclient/base/javacontainer/javacontainer_builder.cc new file mode 100644 index 00000000..ebbaab74 --- /dev/null +++ b/exaudfclient/base/javacontainer/javacontainer_builder.cc @@ -0,0 +1,32 @@ +#include "base/javacontainer/javacontainer_builder.h" +#include "base/javacontainer/script_options/parser_ctpg.h" +#include "base/javacontainer/script_options/parser_legacy.h" +#include "base/swig_factory/swig_factory_impl.h" + +#ifdef ENABLE_JAVA_VM + +namespace SWIGVMContainers { + +JavaContainerBuilder::JavaContainerBuilder() +: m_useCtpgParser(false) {} + +JavaContainerBuilder& JavaContainerBuilder::useCtpgParser(const bool useCtpgParser) { + m_useCtpgParser = useCtpgParser; + return *this; +} + +JavaVMach* JavaContainerBuilder::build() { + std::unique_ptr parser; + if (m_useCtpgParser) { + parser = std::make_unique(std::make_unique()); + } else { + parser = std::make_unique(std::make_unique()); + } + return new JavaVMach(false, std::move(parser)); +} + + +} //namespace SWIGVMContainers + + +#endif //ENABLE_JAVA_VM diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.h b/exaudfclient/base/javacontainer/javacontainer_builder.h new file mode 100644 index 00000000..5893b6d2 --- /dev/null +++ b/exaudfclient/base/javacontainer/javacontainer_builder.h @@ -0,0 +1,35 @@ +#ifndef JAVACONTAINER_BUILDER_H +#define JAVACONTAINER_BUILDER_H + +#include +#include "base/javacontainer/javacontainer.h" + +#ifdef ENABLE_JAVA_VM + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +struct ScriptOptionsParser; + +} + +class JavaContainerBuilder { + public: + JavaContainerBuilder(); + + JavaContainerBuilder& useCtpgParser(const bool value); + + JavaVMach* build(); + + private: + bool m_useCtpgParser; +}; + +} //namespace SWIGVMContainers + + +#endif //ENABLE_JAVA_VM + + +#endif //JAVACONTAINER_BUILDER_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc index 96aa0898..8a9cb2a0 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.cc +++ b/exaudfclient/base/javacontainer/javacontainer_impl.cc @@ -12,14 +12,14 @@ #include "base/javacontainer/javacontainer.h" #include "base/javacontainer/javacontainer_impl.h" #include "base/javacontainer/script_options/extractor.h" -#include "base/javacontainer/script_options/parser_legacy.h" -#include "base/swig_factory/swig_factory.h" +#include "base/javacontainer/script_options/parser.h" using namespace SWIGVMContainers; using namespace std; -JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory) +JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, + std::unique_ptr scriptOptionsParser) : m_checkOnly(checkOnly) , m_exaJavaPath("") , m_localClasspath("/tmp") // **IMPORTANT**: /tmp needs to be in the classpath, otherwise ExaCompiler crashe with com.exasol.ExaCompilationException: /DATE_STRING.java:3: error: error while writing DATE_STRING: could not create parent directories @@ -28,26 +28,12 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory) , m_jvm(NULL) , m_env(NULL) , m_needsCompilation(true) -, m_swigFactory(swigFactory) { stringstream ss; m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path - JavaScriptOptions::ScriptOptionLinesParserLegacy scriptOptionsParser; - - JavaScriptOptions::Extractor extractor(scriptOptionsParser, swigFactory); - - DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used - - DBG_FUNC_CALL(cerr,setClasspath()); - - m_jvmOptions = std::move(extractor.moveJvmOptions()); - - for (set::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end(); - ++it) { - addJarToClasspath(*it); - } + parseScriptOptions(std::move(scriptOptionsParser)); m_needsCompilation = checkNeedsCompilation(); if (m_needsCompilation) { @@ -64,6 +50,21 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory) } } +void JavaVMImpl::parseScriptOptions(std::unique_ptr scriptOptionsParser) { + JavaScriptOptions::Extractor extractor(*scriptOptionsParser); + + DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); + + DBG_FUNC_CALL(cerr,setClasspath()); + + m_jvmOptions = std::move(extractor.moveJvmOptions()); + + for (set::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end(); + ++it) { + addJarToClasspath(*it); + } +} + void JavaVMImpl::shutdown() { if (m_checkOnly) throwException("F-UDF.CL.SL.JAVA-1159: Java VM in check only mode"); diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.h b/exaudfclient/base/javacontainer/javacontainer_impl.h index f4f093be..973b7ede 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.h +++ b/exaudfclient/base/javacontainer/javacontainer_impl.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "base/exaudflib/vm/swig_vm.h" #include @@ -14,10 +15,18 @@ class JavaVMTest; namespace SWIGVMContainers { +namespace JavaScriptOptions { + struct ScriptOptionsParser; +} + class JavaVMImpl { public: friend class ::JavaVMTest; - JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory); + /* + * scriptOptionsParser: JavaVMImpl takes ownership of ScriptOptionsParser pointer. + */ + JavaVMImpl(bool checkOnly, bool noJNI, + std::unique_ptr scriptOptionsParser); ~JavaVMImpl() {} void shutdown(); bool run(); @@ -37,6 +46,7 @@ class JavaVMImpl { void throwException(const std::string& ex); void setJvmOptions(); void addJarToClasspath(const std::string& path); + void parseScriptOptions(std::unique_ptr scriptOptionsParser); bool m_checkOnly; std::string m_exaJavaPath; std::string m_localClasspath; @@ -48,7 +58,6 @@ class JavaVMImpl { JavaVM *m_jvm; JNIEnv *m_env; bool m_needsCompilation; - SwigFactory& m_swigFactory; }; } //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/BUILD b/exaudfclient/base/javacontainer/script_options/BUILD index 585f8a31..6874aa4a 100644 --- a/exaudfclient/base/javacontainer/script_options/BUILD +++ b/exaudfclient/base/javacontainer/script_options/BUILD @@ -3,10 +3,12 @@ package(default_visibility = ["//visibility:public"]) cc_library( name = "java_script_option_lines", - hdrs = [":extractor.h", ":parser_legacy.h"], + hdrs = [":extractor.h", ":parser_legacy.h", ":parser_ctpg.h"], srcs = [":parser.h", ":converter.h", ":converter.cc", ":parser_legacy.cc", ":extractor.cc", - ":keywords.h", ":checksum.h", ":checksum.cc"], - deps = ["//base/script_options_parser/legacy:script_option_lines_parser_legacy", "//base/utils:utils", + ":keywords.h", ":keywords.cc", ":checksum.h", ":checksum.cc", ":parser_ctpg.cc", + ":parser_ctpg_script_importer.cc", ":parser_ctpg_script_importer.h"], + deps = ["@ssl//:ssl", "//base/script_options_parser/legacy:script_option_lines_parser_legacy", + "//base/script_options_parser/ctpg:script_option_lines_parser_ctpg", "//base/utils:utils", "//base/exaudflib:header", "//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory_if", "//base/script_options_parser:exception"], ) diff --git a/exaudfclient/base/javacontainer/script_options/extractor.cc b/exaudfclient/base/javacontainer/script_options/extractor.cc index ed33388b..9a75c1c1 100644 --- a/exaudfclient/base/javacontainer/script_options/extractor.cc +++ b/exaudfclient/base/javacontainer/script_options/extractor.cc @@ -10,17 +10,15 @@ namespace SWIGVMContainers { namespace JavaScriptOptions { -Extractor::Extractor(ScriptOptionsParser & parser, - SwigFactory& swigFactory) -: m_parser(parser) -, m_swigFactory(swigFactory) {} +Extractor::Extractor(ScriptOptionsParser & parser) +: m_parser(parser) {} void Extractor::extract(std::string & scriptCode) { m_parser.prepareScriptCode(scriptCode); EXTR_DBG_FUNC_CALL(m_parser.parseForScriptClass( [&](const std::string& value){ - EXTR_DBG_FUNC_CALL(m_converter.convertScriptClassName(value)); + EXTR_DBG_FUNC_CALL(m_converter.convertScriptClassName(value)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used })); - EXTR_DBG_FUNC_CALL(m_parser.extractImportScripts(m_swigFactory)); + EXTR_DBG_FUNC_CALL(m_parser.extractImportScripts()); EXTR_DBG_FUNC_CALL(m_parser.parseForJvmOptions( [&](const std::string& value){ EXTR_DBG_FUNC_CALL(m_converter.convertJvmOption(value)); })); diff --git a/exaudfclient/base/javacontainer/script_options/extractor.h b/exaudfclient/base/javacontainer/script_options/extractor.h index d69ae354..03ce623b 100644 --- a/exaudfclient/base/javacontainer/script_options/extractor.h +++ b/exaudfclient/base/javacontainer/script_options/extractor.h @@ -10,8 +10,6 @@ namespace SWIGVMContainers { -struct SwigFactory; - namespace JavaScriptOptions { class ScriptOptionsParser; @@ -19,8 +17,7 @@ class ScriptOptionsParser; class Extractor { public: - Extractor(ScriptOptionsParser & parser, - SwigFactory& swigFactory); + Extractor(ScriptOptionsParser & parser); const std::set & getJarPaths() const { return m_converter.getJarPaths(); @@ -36,8 +33,6 @@ class Extractor { Converter m_converter; ScriptOptionsParser & m_parser; - - SwigFactory& m_swigFactory; }; diff --git a/exaudfclient/base/javacontainer/script_options/keywords.cc b/exaudfclient/base/javacontainer/script_options/keywords.cc new file mode 100644 index 00000000..9d89bf38 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/keywords.cc @@ -0,0 +1,33 @@ +#include "base/javacontainer/script_options/keywords.h" +#include + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +Keywords::Keywords(bool withScriptOptionsPrefix) +: m_jarKeyword() +, m_scriptClassKeyword() +, m_importKeyword() +, m_jvmKeyword() { + const std::string_view jar{"%jar"}; + const std::string_view scriptClass{"%scriptclass"}; + const std::string_view import{"%import"}; + const std::string_view jvm{"%jvmoption"}; + if (withScriptOptionsPrefix) { + m_jarKeyword = jar; + m_scriptClassKeyword = scriptClass; + m_importKeyword = import; + m_jvmKeyword = jvm; + } else { + m_jarKeyword.assign(jar.substr(1)); + m_scriptClassKeyword.assign(scriptClass.substr(1)); + m_importKeyword.assign(import.substr(1)); + m_jvmKeyword.assign(jvm.substr(1)); + } +} + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + diff --git a/exaudfclient/base/javacontainer/script_options/keywords.h b/exaudfclient/base/javacontainer/script_options/keywords.h index 840509a9..314bf5b6 100644 --- a/exaudfclient/base/javacontainer/script_options/keywords.h +++ b/exaudfclient/base/javacontainer/script_options/keywords.h @@ -9,20 +9,16 @@ namespace JavaScriptOptions { class Keywords { public: - Keywords() - : m_jarKeyword("%jar") - , m_scriptClassKeyword("%scriptclass") - , m_importKeyword("%import") - , m_jvmOptionKeyword("%jvmoption") {} - const std::string & jarKeyword() { return m_jarKeyword; } + Keywords(bool withScriptOptionsPrefix); const std::string & scriptClassKeyword() { return m_scriptClassKeyword; } const std::string & importKeyword() { return m_importKeyword; } - const std::string & jvmOptionKeyword() { return m_jvmOptionKeyword; } + const std::string & jvmKeyword() { return m_jvmKeyword; } + const std::string & jarKeyword() { return m_jarKeyword; } private: std::string m_jarKeyword; std::string m_scriptClassKeyword; std::string m_importKeyword; - std::string m_jvmOptionKeyword; + std::string m_jvmKeyword; }; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/parser.h b/exaudfclient/base/javacontainer/script_options/parser.h index 68dd634f..abb497fc 100644 --- a/exaudfclient/base/javacontainer/script_options/parser.h +++ b/exaudfclient/base/javacontainer/script_options/parser.h @@ -8,11 +8,10 @@ namespace SWIGVMContainers { -struct SwigFactory; - namespace JavaScriptOptions { struct ScriptOptionsParser { + virtual ~ScriptOptionsParser() {}; /* Passes the script code for parsing to the parser. The parser might modify the script code, because it will remove known options. @@ -42,7 +41,7 @@ struct ScriptOptionsParser { Searches for the "%import" options and embeds the respective imported script code at the same location as the option in the script code. */ - virtual void extractImportScripts(SwigFactory & swigFactory) = 0; + virtual void extractImportScripts() = 0; /* Returns the (eventually modified) script code. diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc index cd73abb7..3f576242 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc @@ -1,9 +1,151 @@ #include "base/javacontainer/script_options/parser_ctpg.h" +#include "base/javacontainer/script_options/parser_ctpg_script_importer.h" +#include "base/utils/exceptions.h" +#include "base/script_options_parser/exception.h" +#include "base/swig_factory/swig_factory.h" +#include + + +namespace ctpg_parser = ExecutionGraph::OptionsLineParser::CTPG; namespace SWIGVMContainers { namespace JavaScriptOptions { +ScriptOptionLinesParserCTPG::ScriptOptionLinesParserCTPG(std::unique_ptr swigFactory) +: m_scriptCode() +, m_keywords(false) +, m_needParsing(true) +, m_swigFactory(std::move(swigFactory)) {} + +void ScriptOptionLinesParserCTPG::prepareScriptCode(const std::string & scriptCode) { + m_scriptCode = scriptCode; +} + +void ScriptOptionLinesParserCTPG::parseForScriptClass(std::function callback) { + try { + parseForSingleOption(m_keywords.scriptClassKeyword(), callback); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1623"); + } +} + +void ScriptOptionLinesParserCTPG::parseForJvmOptions(std::function callback) { + try { + parseForMultipleOption(m_keywords.jvmKeyword(), callback); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1624"); + } +} + +void ScriptOptionLinesParserCTPG::parseForExternalJars(std::function callback) { + try { + parseForMultipleOption(m_keywords.jarKeyword(), callback); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1625"); + } +} + +void ScriptOptionLinesParserCTPG::extractImportScripts() { + + try { + parse(); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1626"); + } + + const auto optionIt = m_foundOptions.find(m_keywords.importKeyword()); + if (optionIt != m_foundOptions.end()) { + CTPG::ScriptImporter scriptImporter(*m_swigFactory, m_keywords); + scriptImporter.importScript(m_scriptCode, m_foundOptions); + //The imported scripts will change the location of the other options in m_foundOptions + //Also there might be new JVM / External Jar options + //=> We need to clear the option map and reset the parser. + m_foundOptions.clear(); + m_needParsing = true; + } +} + +std::string && ScriptOptionLinesParserCTPG::getScriptCode() { + try { + parse(); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1627"); + } + //Remove all options from script code in reverse order + struct option_location { + size_t pos; + size_t len; + }; + struct comp { + bool operator()(option_location a, option_location b) const { + return a.pos > b.pos; + } + }; + std::set option_locations; + for (const auto & option: m_foundOptions) { + for (const auto & option_loc: option.second) { + option_location loc = { .pos = option_loc.idx_in_source, .len = option_loc.size}; + option_locations.insert(loc); + } + } + for (const auto option_loc: option_locations) { + m_scriptCode.erase(option_loc.pos, option_loc.len); + } + return std::move(m_scriptCode); +} + +void ScriptOptionLinesParserCTPG::parse() { + if (m_needParsing) { + if(!m_foundOptions.empty()) { + throw std::logic_error( + "F-UDF-CL-SL-JAVA-1620 Internal error. Parser result is not empty. " + "Please open a bug ticket at https://github.com/exasol/script-languages-release/issues/new."); + } + try { + ExecutionGraph::OptionsLineParser::CTPG::parseOptions(m_scriptCode, m_foundOptions); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1621"); + } + + m_needParsing = false; + + //Check for unknown options + for (const auto & option: m_foundOptions) { + if (m_keywords.jarKeyword() != option.first && + m_keywords.scriptClassKeyword() != option.first && + m_keywords.importKeyword() != option.first && + m_keywords.jvmKeyword() != option.first) { + std::stringstream ss; + ss << "F-UDF-CL-SL-JAVA-1622 " << "Unexpected option: " << option.first; + throw std::invalid_argument(ss.str()); + } + } + } +} + +void ScriptOptionLinesParserCTPG::parseForSingleOption(const std::string key, std::function callback) { + parse(); + const auto optionIt = m_foundOptions.find(key); + if (optionIt != m_foundOptions.end()) { + if (optionIt->second.size() != 1) { + std::stringstream ss; + ss << "F-UDF-CL-SL-JAVA-1628 found " << optionIt->second.size() << " instances for script option key '" << key << "' but expected at most one." << std::endl; + throw std::invalid_argument(ss.str()); + } + callback(optionIt->second[0].value); + } +} + +void ScriptOptionLinesParserCTPG::parseForMultipleOption(const std::string key, std::function callback) { + parse(); + const auto optionIt = m_foundOptions.find(key); + if (optionIt != m_foundOptions.end()) { + for (const auto & option : optionIt->second) { + callback(option.value); + } + } +} } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.h b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h index 3b71a5ee..0f76663a 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg.h +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h @@ -1,10 +1,51 @@ #ifndef SCRIPTOPTIONLINEPARSERCTPGY_H #define SCRIPTOPTIONLINEPARSERCTPGY_H 1 +#include "base/javacontainer/script_options/parser.h" +#include "base/javacontainer/script_options/keywords.h" +#include "base/script_options_parser/ctpg/script_option_lines_ctpg.h" +#include + namespace SWIGVMContainers { +struct SwigFactory; + namespace JavaScriptOptions { +class ScriptOptionLinesParserCTPG : public ScriptOptionsParser { + + public: + ScriptOptionLinesParserCTPG(std::unique_ptr swigFactory); + + virtual ~ScriptOptionLinesParserCTPG() {}; + + void prepareScriptCode(const std::string & scriptCode) override; + + void parseForScriptClass(std::function callback) override; + + void parseForJvmOptions(std::function callback) override; + + void parseForExternalJars(std::function callback) override; + + void extractImportScripts() override; + + std::string && getScriptCode() override; + + private: + void parse(); + + void parseForSingleOption(const std::string key, std::function callback); + void parseForMultipleOption(const std::string key, std::function callback); + + void importScripts(SwigFactory & swigFactory); + + private: + std::string m_scriptCode; + Keywords m_keywords; + ExecutionGraph::OptionsLineParser::CTPG::options_map_t m_foundOptions; + bool m_needParsing; + std::unique_ptr m_swigFactory; +}; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc new file mode 100644 index 00000000..226e2359 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc @@ -0,0 +1,99 @@ +#include "base/javacontainer/script_options/parser_ctpg_script_importer.h" +#include "base/swig_factory/swig_factory.h" +#include "base/utils/debug_message.h" +#include "base/utils/exceptions.h" +#include "base/script_options_parser/exception.h" + +#include +#include +#include + +namespace ctpg_parser = ExecutionGraph::OptionsLineParser::CTPG; + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +namespace CTPG { + +ScriptImporter::ScriptImporter(SwigFactory & swigFactory, Keywords & keywords) +: m_importedScriptChecksums() +, m_swigFactory(swigFactory) +, m_metaData() +, m_keywords(keywords) {} + +void ScriptImporter::importScript(std::string & scriptCode, + ctpg_parser::options_map_t & options) { + importScript(scriptCode, options, 0); +} + +void ScriptImporter::importScript(std::string & scriptCode, + ctpg_parser::options_map_t & options, + const size_t recursionDepth) { + const auto optionIt = options.find(std::string(m_keywords.importKeyword())); + + if (recursionDepth >= cMaxRecursionDepth) { + throw std::runtime_error("F-UDF-CL-SL-JAVA-1633: Maximal recursion depth for importing scripts reached."); + } + if (optionIt != options.end()) { + m_importedScriptChecksums.addScript(scriptCode.c_str()); + //Sort options from first in script to last in script + std::sort(optionIt->second.begin(), optionIt->second.end(), + [](const ctpg_parser::ScriptOption& first, const ctpg_parser::ScriptOption& second) + { + return first.idx_in_source < second.idx_in_source; + }); + struct ReplacedScripts { + ReplacedScripts(ReplacedScripts&&) = default; + std::string script; + size_t origPos; + size_t origLen; + }; + std::vector replacedScripts; + replacedScripts.reserve(optionIt->second.size()); + //In order to continue compatibility with legacy implementation we must collect import scripts in forward direction + //but then replace in reverse direction (in order to keep consistency of positions) + for (const auto & option: optionIt->second) { + const char *importScriptCode = findImportScript(option.value); + std::string importScriptCodeStr; + if (m_importedScriptChecksums.addScript(importScriptCode) ) { + // Script has not been imported yet + // If this imported script contains %import statements + // they will be resolved in the next recursion. + ctpg_parser::options_map_t newOptions; + try { + ExecutionGraph::OptionsLineParser::CTPG::parseOptions(importScriptCode, newOptions); + } catch(const ExecutionGraph::OptionParserException & ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1630"); + } + importScriptCodeStr.assign(importScriptCode); + importScript(importScriptCodeStr, newOptions, recursionDepth + 1); + } + ReplacedScripts replacedScript = {.script = std::move(importScriptCodeStr), .origPos = option.idx_in_source, .origLen = option.size }; + replacedScripts.push_back(std::move(replacedScript)); + } + //Now replace the imported script bodies from end to start. Doing it in forward order would invalidate the offsets of later import scripts. + for (auto optionIt = replacedScripts.rbegin(); optionIt != replacedScripts.rend(); optionIt++) { + scriptCode.replace(optionIt->origPos, optionIt->origLen, optionIt->script); + } + } +} + +const char* ScriptImporter::findImportScript(const std::string & scriptKey) { + if (!m_metaData) { + m_metaData.reset(m_swigFactory.makeSwigMetadata()); + if (!m_metaData) + throw std::runtime_error("F-UDF-CL-SL-JAVA-1631: Failure while importing scripts"); + } + const char *importScriptCode = m_metaData->moduleContent(scriptKey.c_str()); + const char *exception = m_metaData->checkException(); + if (exception) + throw std::runtime_error("F-UDF-CL-SL-JAVA-1632: " + std::string(exception)); + return importScriptCode; +} + +} //namespace CTPG + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h new file mode 100644 index 00000000..f0492e34 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h @@ -0,0 +1,46 @@ +#ifndef SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H +#define SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H 1 + +#include "base/javacontainer/script_options/checksum.h" +#include "base/javacontainer/script_options/keywords.h" +#include "base/exaudflib/swig/swig_meta_data.h" +#include "base/script_options_parser/ctpg/script_option_lines_ctpg.h" +#include + + +namespace SWIGVMContainers { + + struct SwigFactory; + +namespace JavaScriptOptions { + +namespace CTPG { + +class ScriptImporter { + + public: + ScriptImporter(SwigFactory & swigFactory, Keywords & keywords); + + void importScript(std::string & scriptCode, ExecutionGraph::OptionsLineParser::CTPG::options_map_t & options); + + private: + void importScript(std::string & scriptCode, + ExecutionGraph::OptionsLineParser::CTPG::options_map_t & options, + const size_t recursionDepth); + const char* findImportScript(const std::string & scriptKey); + private: + Checksum m_importedScriptChecksums; + SwigFactory & m_swigFactory; + std::unique_ptr m_metaData; + Keywords & m_keywords; + //The empirical maximal value for recursion depth is ~26000. So we choose 20000 to have a certain buffer. + const size_t cMaxRecursionDepth = 20000; +}; + +} //namespace CTPG + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + +#endif //SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc index 4462e73b..4a07ead4 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc @@ -13,17 +13,18 @@ namespace SWIGVMContainers { namespace JavaScriptOptions { -ScriptOptionLinesParserLegacy::ScriptOptionLinesParserLegacy() +ScriptOptionLinesParserLegacy::ScriptOptionLinesParserLegacy(std::unique_ptr swigFactory) : m_whitespace(" \t\f\v") , m_lineend(";") , m_scriptCode() -, m_keywords() {} +, m_keywords(true) +, m_swigFactory(std::move(swigFactory)) {} void ScriptOptionLinesParserLegacy::prepareScriptCode(const std::string & scriptCode) { m_scriptCode = scriptCode; } -void ScriptOptionLinesParserLegacy::extractImportScripts(SwigFactory & swigFactory) { +void ScriptOptionLinesParserLegacy::extractImportScripts() { std::unique_ptr metaData; // Attention: We must hash the parent script before modifying it (adding the // package definition). Otherwise we don't recognize if the script imports its self @@ -111,7 +112,7 @@ void ScriptOptionLinesParserLegacy::extractImportScripts(SwigFactory & swigFacto } if (!newScript.empty()) { if (!metaData) { - metaData.reset(swigFactory.makeSwigMetadata()); + metaData.reset(m_swigFactory->makeSwigMetadata()); if (!metaData) throw std::runtime_error("F-UDF-CL-SL-JAVA-1615: Failure while importing scripts"); } @@ -142,7 +143,7 @@ void ScriptOptionLinesParserLegacy::parseForScriptClass(std::function callback) { try { - parseForMultipleOptions(m_keywords.jvmOptionKeyword(), + parseForMultipleOptions(m_keywords.jvmKeyword(), [&](const std::string& value, size_t pos){callback(value);}); } catch(const ExecutionGraph::OptionParserException& ex) { Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1612"); @@ -162,7 +163,7 @@ std::string && ScriptOptionLinesParserLegacy::getScriptCode() { return std::move(m_scriptCode); } -void ScriptOptionLinesParserLegacy::parseForSingleOption(const std::string keyword, +void ScriptOptionLinesParserLegacy::parseForSingleOption(const std::string & keyword, std::function callback) { size_t pos; try { @@ -176,7 +177,7 @@ void ScriptOptionLinesParserLegacy::parseForSingleOption(const std::string keywo } } -void ScriptOptionLinesParserLegacy::parseForMultipleOptions(const std::string keyword, +void ScriptOptionLinesParserLegacy::parseForMultipleOptions(const std::string & keyword, std::function callback) { size_t pos; while (true) { diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.h b/exaudfclient/base/javacontainer/script_options/parser_legacy.h index 655f6e0f..447ca394 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.h +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.h @@ -5,15 +5,20 @@ #include "base/javacontainer/script_options/parser.h" #include "base/javacontainer/script_options/keywords.h" +#include namespace SWIGVMContainers { +struct SwigFactory; + namespace JavaScriptOptions { class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { public: - ScriptOptionLinesParserLegacy(); + ScriptOptionLinesParserLegacy(std::unique_ptr swigFactory); + + virtual ~ScriptOptionLinesParserLegacy() {}; void prepareScriptCode(const std::string & scriptCode) override; @@ -23,14 +28,14 @@ class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { void parseForExternalJars(std::function callback) override; - void extractImportScripts(SwigFactory & swigFactory) override; + void extractImportScripts() override; std::string && getScriptCode() override; private: - void parseForSingleOption(const std::string key, + void parseForSingleOption(const std::string& key, std::function callback); - void parseForMultipleOptions(const std::string key, + void parseForMultipleOptions(const std::string& key, std::function callback); private: @@ -38,6 +43,7 @@ class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { const std::string m_lineend; std::string m_scriptCode; Keywords m_keywords; + std::unique_ptr m_swigFactory; }; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/test/BUILD b/exaudfclient/base/javacontainer/script_options/test/BUILD new file mode 100644 index 00000000..5c7f0cf5 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/test/BUILD @@ -0,0 +1,10 @@ + +cc_test( + name = "java-script-options-tests", + srcs = ["ctpg_script_importer_test.cc", "swig_factory_test.cc", "swig_factory_test.h"], + deps = [ + "//base/javacontainer/script_options:java_script_option_lines", + "@googletest//:gtest_main", + ], +) + diff --git a/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc b/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc new file mode 100644 index 00000000..eb34a961 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc @@ -0,0 +1,66 @@ + +#include "include/gtest/gtest.h" +#include "gmock/gmock.h" +#include "base/javacontainer/script_options/parser_ctpg.h" + +#include "base/javacontainer/script_options/test/swig_factory_test.h" +#include + +using namespace SWIGVMContainers::JavaScriptOptions; + + +static const char* sc_scriptName = "script"; + + +void checkIndex(size_t currentIdx, const char* scriptKey) { + std::stringstream ss; + ss << sc_scriptName << currentIdx; + if (ss.str() != scriptKey) { + throw std::logic_error(std::string("Script Key does not match: '") + ss.str() + " != '" + scriptKey + "'"); + } +} + +const char* buildNewScriptCode(size_t currentIdx) { + std::stringstream ss; + ss << "%import " << sc_scriptName << currentIdx << ";something"; + static std::string ret; + ret = ss.str(); + return ret.c_str(); +} + +TEST(ScriptImporterTest, max_recursion_depth) { + /** + This test checks that running an infinite recursion of the script import will result in the expected exception. + For that, the test creates new "import scripts" on the fly: + Whenever the parser finds a new 'import script' option, + it calls SWIGVMContainers::SWIGMetadataIf::moduleContent(). + The mocked implementation redirects this request to `buildNewScriptCode()` which creates a + new dummy import script with another '%import ...` option. + */ + + size_t currentIdx = 0; + std::unique_ptr swigFactory = + std::make_unique([&](const char* scriptKey) { + checkIndex(currentIdx, scriptKey); + return buildNewScriptCode(++currentIdx); + }); + ScriptOptionLinesParserCTPG parser(std::move(swigFactory)); + + const std::string code = buildNewScriptCode(currentIdx); + parser.prepareScriptCode(code); + EXPECT_THROW({ + try + { + parser.extractImportScripts(); + } + catch( const std::runtime_error& e ) + { + //We need to deceive "find_duplicate_error_codes.sh" here + const std::string expectedError = + std::string("F-UDF-CL-SL-JAVA-") + "1633: Maximal recursion depth for importing scripts reached."; + EXPECT_STREQ( expectedError.c_str(), e.what()); + throw; + } + }, std::runtime_error ); +} + diff --git a/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.cc b/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.cc new file mode 100644 index 00000000..8744d421 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.cc @@ -0,0 +1,105 @@ +#include "base/javacontainer/script_options/test/swig_factory_test.h" +#include "base/exaudflib/swig/swig_meta_data.h" + +#include + +class NotImplemented : public std::logic_error +{ +public: + NotImplemented() : std::logic_error("Function not yet implemented") { }; + NotImplemented(const std::string funcName) : std::logic_error("Function " + funcName + " not yet implemented") { }; +}; + +class SWIGMetadataTest : public SWIGVMContainers::SWIGMetadataIf { + +public: + SWIGMetadataTest(std::function callback): m_callback(callback) {} + virtual const char* databaseName() { throw NotImplemented("databaseName"); return nullptr;} + virtual const char* databaseVersion() { throw NotImplemented("databaseVersion"); return nullptr;} + virtual const char* scriptName() { throw NotImplemented("scriptName"); return nullptr;} + virtual const char* scriptSchema() { throw NotImplemented("scriptSchema"); return nullptr;} + virtual const char* currentUser() { throw NotImplemented("currentUser"); return nullptr;} + virtual const char* scopeUser() { throw NotImplemented("scopeUser"); return nullptr;} + virtual const char* currentSchema() { throw NotImplemented("currentSchema"); return nullptr;} + virtual const char* scriptCode() { throw NotImplemented("scriptCode"); return nullptr;} + virtual const unsigned long long sessionID() { throw NotImplemented("sessionID"); return 0;} + virtual const char *sessionID_S() { throw NotImplemented("sessionID_S"); return nullptr;} + virtual const unsigned long statementID() { throw NotImplemented("statementID"); return 0;} + virtual const unsigned int nodeCount() { throw NotImplemented("nodeCount"); return 0;} + virtual const unsigned int nodeID() { throw NotImplemented("nodeID"); return 0;} + virtual const unsigned long long vmID() { throw NotImplemented("vmID"); return 0;} + virtual const unsigned long long memoryLimit() { throw NotImplemented("memoryLimit"); return 0;} + virtual const SWIGVMContainers::VMTYPE vmType() { + throw NotImplemented("vmType"); + return SWIGVMContainers::VM_UNSUPPORTED; + } + virtual const char *vmID_S() { throw NotImplemented("vmID_S"); return nullptr;} + virtual const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name) { + throw NotImplemented("connectionInformation"); return nullptr; + } + virtual const char* moduleContent(const char* name) { + return m_callback(name); + } + virtual const unsigned int inputColumnCount() { throw NotImplemented("inputColumnCount"); return 0;} + virtual const char *inputColumnName(unsigned int col) { + throw NotImplemented("inputColumnName"); + return nullptr; + } + virtual const SWIGVMContainers::SWIGVM_datatype_e inputColumnType(unsigned int col) { + throw NotImplemented("inputColumnType"); + return SWIGVMContainers::UNSUPPORTED; + } + virtual const char *inputColumnTypeName(unsigned int col) { + throw NotImplemented("inputColumnTypeName"); return nullptr; + } + virtual const unsigned int inputColumnSize(unsigned int col) { + throw NotImplemented("inputColumnSize"); return 0; + } + virtual const unsigned int inputColumnPrecision(unsigned int col) { + throw NotImplemented("inputColumnPrecision"); return 0; + } + virtual const unsigned int inputColumnScale(unsigned int col) { + throw NotImplemented("inputColumnScale"); return 0; + } + virtual const SWIGVMContainers::SWIGVM_itertype_e inputType() { + throw NotImplemented("inputType"); + return SWIGVMContainers::EXACTLY_ONCE; + } + virtual const unsigned int outputColumnCount() { throw NotImplemented("outputColumnCount"); return 0;} + virtual const char *outputColumnName(unsigned int col) { throw NotImplemented("outputColumnName"); return nullptr;} + virtual const SWIGVMContainers::SWIGVM_datatype_e outputColumnType(unsigned int col) { + throw NotImplemented("outputColumnType"); + return SWIGVMContainers::UNSUPPORTED; + } + virtual const char *outputColumnTypeName(unsigned int col) { + throw NotImplemented("outputColumnTypeName"); return nullptr; + } + virtual const unsigned int outputColumnSize(unsigned int col) { + throw NotImplemented("outputColumnSize"); return 0; + } + virtual const unsigned int outputColumnPrecision(unsigned int col) { + throw NotImplemented("outputColumnPrecision"); return 0; + } + virtual const unsigned int outputColumnScale(unsigned int col) { + throw NotImplemented("outputColumnScale"); return 0; + } + virtual const SWIGVMContainers::SWIGVM_itertype_e outputType() { + throw NotImplemented("outputType"); + return SWIGVMContainers::EXACTLY_ONCE; + } + virtual const bool isEmittedColumn(unsigned int col) { throw NotImplemented("isEmittedColumn"); return false;} + virtual const char* checkException() { return nullptr;} + virtual const char* pluginLanguageName() { throw NotImplemented("pluginLanguageName"); return nullptr;} + virtual const char* pluginURI() { throw NotImplemented("pluginURI"); return nullptr;} + virtual const char* outputAddress() { throw NotImplemented("outputAddress"); return nullptr;} + +private: + std::function m_callback; +}; + +SwigFactoryTestImpl::SwigFactoryTestImpl(std::function callback) +: m_callback(callback) {} + +SWIGVMContainers::SWIGMetadataIf* SwigFactoryTestImpl::makeSwigMetadata() { + return new SWIGMetadataTest(m_callback); +} diff --git a/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.h b/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.h new file mode 100644 index 00000000..f5fa9555 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.h @@ -0,0 +1,20 @@ +#ifndef SWIG_FACTORY_TEST_H +#define SWIG_FACTORY_TEST_H 1 + +#include +#include +#include "base/swig_factory/swig_factory.h" +#include + +struct SwigFactoryTestImpl : public SWIGVMContainers::SwigFactory { + + SwigFactoryTestImpl(std::function callback); + + virtual SWIGVMContainers::SWIGMetadataIf* makeSwigMetadata() override; + +private: + std::function m_callback; +}; + + +#endif //namespace SWIG_FACTORY_TEST_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/test/BUILD b/exaudfclient/base/javacontainer/test/BUILD index 32874b9f..36fb3620 100644 --- a/exaudfclient/base/javacontainer/test/BUILD +++ b/exaudfclient/base/javacontainer/test/BUILD @@ -10,14 +10,51 @@ java_test( deps = [":ExaStackTraceCleaner"] ) +JAVACONTAINER_TEST_SRCS = ["cpp/javacontainer_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", + "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"] + +JAVACONTAINER_PERF_TEST_SRCS = ["cpp/javacontainer_perf_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", + "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"] + +cc_test( + name = "javacontainer-test-legacy-parser", + srcs = JAVACONTAINER_TEST_SRCS, + deps = [ + "//base/javacontainer:javacontainer", + "@googletest//:gtest_main", + ], + data = ["test.jar", "other_test.jar"] +) cc_test( - name = "javacontainer-test", - srcs = ["cpp/javacontainer_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", - "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"], + name = "javacontainer-test-ctpg-parser", + srcs = JAVACONTAINER_TEST_SRCS + ["cpp/javacontainer_ctpg_test.cc"], deps = [ "//base/javacontainer:javacontainer", "@googletest//:gtest_main", ], + defines = ["USE_CTPG_PARSER"], data = ["test.jar", "other_test.jar"] -) \ No newline at end of file +) + + +cc_test( + name = "javacontainer-perf-test-legacy-parser", + srcs = JAVACONTAINER_PERF_TEST_SRCS, + deps = [ + "//base/javacontainer:javacontainer", + "@googletest//:gtest_main", + ], + data = ["test.jar", "other_test.jar"] +) + +cc_test( + name = "javacontainer-perf-test-ctpg-parser", + srcs = JAVACONTAINER_PERF_TEST_SRCS, + deps = [ + "//base/javacontainer:javacontainer", + "@googletest//:gtest_main", + ], + defines = ["USE_CTPG_PARSER"], + data = ["test.jar", "other_test.jar"] +) diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc new file mode 100644 index 00000000..e66168e5 --- /dev/null +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc @@ -0,0 +1,109 @@ + +#include "include/gtest/gtest.h" +#include "gmock/gmock.h" +#include "base/javacontainer/test/cpp/javavm_test.h" +#include "base/javacontainer/test/cpp/swig_factory_test.h" +#include +#include + +class JavaContainerEscapeSequenceTest : public ::testing::TestWithParam> {}; + +TEST_P(JavaContainerEscapeSequenceTest, quoted_jvm_option) { +const std::pair option_value = GetParam(); + const std::string script_code = + "%jvmoption " + option_value.first + ";\n\n" + "class JVMOPTION_TEST_WITH_SPACE {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" + " ctx.emit(\"Success!\");\n" + " }\n" + "}\n"; + JavaVMTest vm(script_code); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); + const std::string expected_script_code = + "package com.exasol;\r\n\n\n" + "class JVMOPTION_TEST_WITH_SPACE {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" + "\tctx.emit(\"Success!\");\n" + " }\n}\n"; + EXPECT_EQ(expected_script_code, vm.getJavaVMInternalStatus().m_scriptCode); + EXPECT_EQ("/exaudf/base/javacontainer/exaudf_deploy.jar", vm.getJavaVMInternalStatus().m_exaJarPath); + EXPECT_EQ("/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar", vm.getJavaVMInternalStatus().m_classpath); + EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); + + const std::vector expectedJVMOptions = { option_value.second, "-Xms128m", "-Xmx128m", "-Xss512k", + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Djava.class.path=/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar", + "-XX:+UseSerialGC" }; + EXPECT_EQ(expectedJVMOptions, vm.getJavaVMInternalStatus().m_jvmOptions); +} + +const std::vector> escape_sequences = + { + std::make_pair("-Dhttp.agent=ABC\\nDEF", "-Dhttp.agent=ABC\nDEF"), + std::make_pair("-Dhttp.agent=ABC\\rDEF", "-Dhttp.agent=ABC\rDEF"), + std::make_pair("-Dhttp.agent=ABC\\;DEF", "-Dhttp.agent=ABC;DEF"), + std::make_pair("-Dhttp.agent=ABC\\aDEF", "-Dhttp.agent=ABC\\aDEF"), //any other escape sequence must stay as is + std::make_pair("\\n-Dhttp.agent=ABCDEF", "\n-Dhttp.agent=ABCDEF"), + std::make_pair("\\r-Dhttp.agent=ABCDEF", "\r-Dhttp.agent=ABCDEF"), + std::make_pair("\\;-Dhttp.agent=ABCDEF", ";-Dhttp.agent=ABCDEF"), + std::make_pair("-Dhttp.agent=ABCDEF\\n", "-Dhttp.agent=ABCDEF\n"), + std::make_pair("-Dhttp.agent=ABCDEF\\r", "-Dhttp.agent=ABCDEF\r"), + std::make_pair("-Dhttp.agent=ABCDEF\\;", "-Dhttp.agent=ABCDEF;"), + std::make_pair("\\ -Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), + std::make_pair("\\t-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), + std::make_pair("\\f-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), + std::make_pair("\\v-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF") + }; + +INSTANTIATE_TEST_SUITE_P( + JavaContainer, + JavaContainerEscapeSequenceTest, + ::testing::ValuesIn(escape_sequences) +); + +TEST(JavaContainer, import_script_with_escaped_options) { + const std::string script_code = + "%import other_script;\n\n" + "%jvmoption -Dsomeoption=\"ABC\";\n\n" + "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" + "%jar base/javacontainer/test/test.jar;" + "class JVMOPTION_TEST {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" + " ctx.emit(\"Success!\");\n" + " }\n" + "}\n"; + std::unique_ptr swigFactory = std::make_unique(); + + const std::string other_script_code = + "%jvmoption -Dsomeotheroption=\"DE\\nF\";\n\n" + "%jar base/javacontainer/test/other_test.jar;" + "class OtherClass {\n" + "static void doSomething() {\n\n" + " }\n" + "}\n"; + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); + const std::string expected_script_code = + "package com.exasol;\r\n\n\n" + "class OtherClass {\n" + "static void doSomething() {\n\n" + " }\n" + "}\n\n\n\n\n\n" + "class JVMOPTION_TEST {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" + "\tctx.emit(\"Success!\");\n" + " }\n}\n"; + EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/other_test.jar:base/javacontainer/test/test.jar"); + EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); + const std::vector expectedJVMOptions = { "-Dexasol.scriptclass=com.exasol.udf_profiling.UdfProfiler", + "-Dsomeotheroption=\"DE\nF\"", "-Dsomeoption=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Djava.class.path=/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/other_test.jar:base/javacontainer/test/test.jar", + "-XX:+UseSerialGC" }; + EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); +} diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_perf_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_perf_test.cc new file mode 100644 index 00000000..8321b285 --- /dev/null +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_perf_test.cc @@ -0,0 +1,27 @@ +#include "include/gtest/gtest.h" +#include "gmock/gmock.h" +#include "base/javacontainer/test/cpp/javavm_test.h" +#include "base/javacontainer/test/cpp/swig_factory_test.h" +#include + +const uint32_t NumInlineJavaLines = 500000; +const uint32_t NumInlineJavaWordsPerLine = 100; + + +TEST(JavaContainerPerformance, large_inline_java_udf_test) { + std::string script_code = + "%jvmoption option1=abc;\n" + "%jvmoption option2=def;\n" + "class JVMOPTION_TEST_WITH_SPACE {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n"; + + for (uint32_t idxLine(0); idxLine < NumInlineJavaLines; idxLine++) { + for (uint32_t idxWord(0); idxWord < NumInlineJavaWordsPerLine; idxWord++) + script_code.append("somecode "); + script_code.append("\n"); + } + script_code.append(" }\n}\n"); + JavaVMTest vm(script_code); +} + + diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc index ff772f68..0c754206 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc @@ -5,6 +5,7 @@ #include "base/javacontainer/test/cpp/swig_factory_test.h" #include + TEST(JavaContainer, basic_jar) { const std::string script_code = "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" "%jar base/javacontainer/test/test.jar;"; @@ -123,15 +124,15 @@ TEST(JavaContainer, simple_import_script) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "class OtherClass {\n" "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -164,7 +165,7 @@ TEST(JavaContainer, import_script_with_recursion) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "%import other_script;\n\n" @@ -172,8 +173,8 @@ TEST(JavaContainer, import_script_with_recursion) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -205,7 +206,7 @@ TEST(JavaContainer, import_script_with_jvmoption) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "%jvmoption -Dhttp.agent=\"ABC\";\n\n" @@ -213,8 +214,8 @@ TEST(JavaContainer, import_script_with_jvmoption) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -247,7 +248,7 @@ TEST(JavaContainer, multiple_import_scripts) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_scipt_code_A = "%import other_script_B;\n\n" @@ -266,10 +267,10 @@ TEST(JavaContainer, multiple_import_scripts) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script_A", other_scipt_code_A); - swigFactory.addModule("other_script_B", other_scipt_code_B); - swigFactory.addModule("other_script_C", other_scipt_code_C); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script_A", other_scipt_code_A); + swigFactory->addModule("other_script_B", other_scipt_code_B); + swigFactory->addModule("other_script_C", other_scipt_code_C); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -312,7 +313,7 @@ TEST(JavaContainer, import_script_with_mixed_options) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "%jvmoption -Dsomeotheroption=\"DEF\";\n\n" @@ -321,8 +322,8 @@ TEST(JavaContainer, import_script_with_mixed_options) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -355,7 +356,7 @@ TEST(JavaContainer, import_script_script_class_option_ignored) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" @@ -363,13 +364,16 @@ TEST(JavaContainer, import_script_script_class_option_ignored) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = "package com.exasol;\r\n" - "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" +#ifndef USE_CTPG_PARSER //The parsers behave differently: The legacy parser incorrectly keeps imported scriptclass options + "%scriptclass com.exasol.udf_profiling.UdfProfiler;" +#endif + "\n" "class OtherClass {\n" "static void doSomething() {\n\n" " }\n" @@ -398,7 +402,7 @@ TEST(JavaContainer, import_scripts_deep_recursion) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_scipt_code_A = "%import other_script_B;\n\n" @@ -424,11 +428,11 @@ TEST(JavaContainer, import_scripts_deep_recursion) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script_A", other_scipt_code_A); - swigFactory.addModule("other_script_B", other_scipt_code_B); - swigFactory.addModule("other_script_C", other_scipt_code_C); - swigFactory.addModule("other_script_D", other_scipt_code_D); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script_A", other_scipt_code_A); + swigFactory->addModule("other_script_B", other_scipt_code_B); + swigFactory->addModule("other_script_C", other_scipt_code_C); + swigFactory->addModule("other_script_D", other_scipt_code_D); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc index 9ac82a59..6a26dc35 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc @@ -1,26 +1,33 @@ #include "base/javacontainer/test/cpp/javavm_test.h" #include "base/javacontainer/test/cpp/swig_factory_test.h" #include "base/javacontainer/javacontainer_impl.h" +#include "base/javacontainer/script_options/parser_ctpg.h" +#include "base/javacontainer/script_options/parser_legacy.h" #include - -SwigFactoryTestImpl & defaultSwigFactory() { - static SwigFactoryTestImpl swigFactory; - return swigFactory; +std::unique_ptr makeDefaultSwigFactory() { + return std::make_unique(); } JavaVMTest::JavaVMTest(std::string scriptCode) : javaVMInternalStatus() { - run(scriptCode, defaultSwigFactory()); + run(scriptCode, std::move(makeDefaultSwigFactory())); } -JavaVMTest::JavaVMTest(std::string scriptCode, SwigFactoryTestImpl & swigFactory) : javaVMInternalStatus() { - run(scriptCode, swigFactory); +JavaVMTest::JavaVMTest(std::string scriptCode, std::unique_ptr swigFactory) : javaVMInternalStatus() { + run(scriptCode, std::move(swigFactory)); } -void JavaVMTest::run(std::string scriptCode, SwigFactoryTestImpl & swigFactory) { +void JavaVMTest::run(std::string scriptCode, std::unique_ptr swigFactory) { char* script_code = ::strdup(scriptCode.c_str()); SWIGVMContainers::SWIGVM_params->script_code = script_code; - SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, swigFactory); +#ifndef USE_CTPG_PARSER + std::unique_ptr parser = + std::make_unique(std::move(swigFactory)); +#else + std::unique_ptr parser = + std::make_unique(std::move(swigFactory)); +#endif + SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, std::move(parser)); javaVMInternalStatus.m_exaJavaPath = javaVMImpl.m_exaJavaPath; javaVMInternalStatus.m_localClasspath = javaVMImpl.m_localClasspath; javaVMInternalStatus.m_scriptCode = javaVMImpl.m_scriptCode; @@ -28,6 +35,6 @@ void JavaVMTest::run(std::string scriptCode, SwigFactoryTestImpl & swigFactory) javaVMInternalStatus.m_classpath = javaVMImpl.m_classpath; javaVMInternalStatus.m_jvmOptions = javaVMImpl.m_jvmOptions; javaVMInternalStatus.m_needsCompilation = javaVMImpl.m_needsCompilation; - delete script_code; + ::free(script_code); } diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.h b/exaudfclient/base/javacontainer/test/cpp/javavm_test.h index d4e31844..a0e8e0b5 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.h +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.h @@ -4,6 +4,7 @@ #include #include #include +#include struct JavaVMInternalStatus { std::string m_exaJavaPath; @@ -21,12 +22,12 @@ class JavaVMTest { public: JavaVMTest(std::string scriptCode); - JavaVMTest(std::string scriptCode, SwigFactoryTestImpl & swigFactory); + JavaVMTest(std::string scriptCode, std::unique_ptr swigFactory); const JavaVMInternalStatus& getJavaVMInternalStatus() {return javaVMInternalStatus;} private: - void run(std::string scriptCode, SwigFactoryTestImpl & swigFactory); + void run(std::string scriptCode, std::unique_ptr swigFactory); private: JavaVMInternalStatus javaVMInternalStatus; }; diff --git a/exaudfclient/base/javacontainer/test/cpp/swig_factory_test.cc b/exaudfclient/base/javacontainer/test/cpp/swig_factory_test.cc index 7dbb6ee7..aa46fa7d 100644 --- a/exaudfclient/base/javacontainer/test/cpp/swig_factory_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/swig_factory_test.cc @@ -14,8 +14,8 @@ class SWIGMetadataTest : public SWIGVMContainers::SWIGMetadataIf { public: SWIGMetadataTest(const std::map & moduleContent, const std::string & exceptionMsg) - : m_moduleContent(moduleContent) - , m_exceptionMsg(exceptionMsg) {} + : m_exceptionMsg(exceptionMsg) + , m_moduleContent(moduleContent) {} virtual const char* databaseName() { throw NotImplemented("databaseName"); return nullptr;} virtual const char* databaseVersion() { throw NotImplemented("databaseVersion"); return nullptr;} virtual const char* scriptName() { throw NotImplemented("scriptName"); return nullptr;} diff --git a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc index 7c9229b8..74d799de 100644 --- a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc +++ b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc @@ -180,6 +180,7 @@ constexpr parser option_parser( ); void parse(std::string&& code, options_type& result) { + std::stringstream error_buffer; auto res = option_parser.parse( parse_options{}.set_skip_whitespace(false), @@ -204,30 +205,36 @@ void parseOptions(const std::string& code, options_map_t & result) { size_t current_pos = 0; do { + const size_t new_option_start_pos = code.find_first_of("%", current_pos); + if (new_option_start_pos == std::string::npos) + break; + current_pos = code.find_last_of("\r\n", new_option_start_pos); + if (std::string::npos == current_pos) + current_pos = 0; + else + current_pos++; const size_t new_pos = code.find_first_of("\r\n", current_pos); std::string line = code.substr(current_pos, new_pos); - if (!line.empty() && !std::all_of(line.begin(),line.end(), [](const char c) {return std::isspace(c);})) { - options_type parser_result; - ParserInternals::parse(std::move(line), parser_result); - for (const auto & option: parser_result) + options_type parser_result; + ParserInternals::parse(std::move(line), parser_result); + for (const auto & option: parser_result) + { + ScriptOption entry = { + .value = option.value, + .idx_in_source = current_pos + option.start.column - 1, + .size = option.end.column - option.start.column + 1 + }; + auto it_in_result = result.find(option.key); + if (it_in_result == result.end()) + { + options_t new_options; + new_options.push_back(entry); + result.insert(std::make_pair(option.key, new_options)); + } + else { - ScriptOption entry = { - .value = option.value, - .idx_in_source = current_pos + option.start.column - 1, - .size = option.end.column - option.start.column + 1 - }; - auto it_in_result = result.find(option.key); - if (it_in_result == result.end()) - { - options_t new_options; - new_options.push_back(entry); - result.insert(std::make_pair(option.key, new_options)); - } - else - { - it_in_result->second.push_back(entry); - } + it_in_result->second.push_back(entry); } } if (new_pos == std::string::npos) { diff --git a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.h b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.h index 21c29034..8f893a26 100644 --- a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.h +++ b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.h @@ -15,8 +15,6 @@ namespace OptionsLineParser namespace CTPG { -class ParserResult; - struct ScriptOption { std::string value; size_t idx_in_source; diff --git a/exaudfclient/base/script_options_parser/ctpg/test/script_option_lines_test.cpp b/exaudfclient/base/script_options_parser/ctpg/test/script_option_lines_test.cpp index f7cd3540..76ca72b0 100644 --- a/exaudfclient/base/script_options_parser/ctpg/test/script_option_lines_test.cpp +++ b/exaudfclient/base/script_options_parser/ctpg/test/script_option_lines_test.cpp @@ -17,6 +17,8 @@ inline ScriptOption buildOption(const char* value, size_t idx, size_t len) { return option; } +#ifndef VALGRIND_ACTIVE + class ScriptOptionLinesWhitespaceTest : public ::testing::TestWithParam> {}; TEST_P(ScriptOptionLinesWhitespaceTest, WhitespaceExtractOptionLineTest) { @@ -58,6 +60,8 @@ INSTANTIATE_TEST_SUITE_P( ) ); +#endif //VALGRIND_ACTIVE + TEST(ScriptOptionLinesTest, ignore_anything_other_than_whitepsace) { const std::string code = "abc %option myoption;\n" diff --git a/exaudfclient/base/swig_factory/swig_factory.h b/exaudfclient/base/swig_factory/swig_factory.h index 06e82952..48f93c1d 100644 --- a/exaudfclient/base/swig_factory/swig_factory.h +++ b/exaudfclient/base/swig_factory/swig_factory.h @@ -6,6 +6,7 @@ namespace SWIGVMContainers { struct SWIGMetadataIf; struct SwigFactory { + virtual ~SwigFactory() {}; virtual SWIGMetadataIf* makeSwigMetadata() = 0; diff --git a/exaudfclient/exaudfclient.cc b/exaudfclient/exaudfclient.cc index 00b644f6..40756f18 100644 --- a/exaudfclient/exaudfclient.cc +++ b/exaudfclient/exaudfclient.cc @@ -38,7 +38,7 @@ #ifdef ENABLE_JAVA_VM -#include "base/javacontainer/javacontainer.h" +#include "base/javacontainer/javacontainer_builder.h" #endif //ENABLE_JAVA_VM #ifdef ENABLE_PYTHON_VM @@ -49,8 +49,6 @@ #include "protegrityclient.h" #endif -#include "base/swig_factory/swig_factory_impl.h" - using namespace std; using namespace SWIGVMContainers; @@ -140,7 +138,6 @@ int main(int argc, char **argv) { ::setlocale(LC_ALL, "en_US.utf8"); std::functionvmMaker=[](){return nullptr;}; // the initial vm maker returns NULL - SwigFactoryImpl swigFactory; #ifdef UDF_PLUGIN_CLIENT vmMaker = [](){return new SWIGVMContainers::Protegrity(false);}; #else @@ -163,7 +160,7 @@ int main(int argc, char **argv) { } else if (strcmp(argv[2], "lang=java")==0) { #ifdef ENABLE_JAVA_VM - vmMaker = [&](){return new SWIGVMContainers::JavaVMach(false, swigFactory);}; + vmMaker = [&](){return SWIGVMContainers::JavaContainerBuilder().build();}; #else throw SWIGVM::exception("this exaudfclient has been compilied without Java support"); #endif diff --git a/scripts/installUdfClientDeps.sh b/scripts/installUdfClientDeps.sh new file mode 100644 index 00000000..075b5c06 --- /dev/null +++ b/scripts/installUdfClientDeps.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail + +source /etc/os-release + +if [ "$VERSION_CODENAME" != "jammy" ]; then + echo "Script can only work correctly under Ubuntu 22.04" + exit 1 +fi + +if [ $# -ne 1 ]; +then + me=$(basename "$0") + echo "Usage: '$me '. For example: '$me .env'. will contain necessary environment variables to run 'bazel build'." + exit 1 +fi + +OUT_ENV_FILE="$1" + +export DEBIAN_FRONTEND=noninteractive + +apt update +apt install -y curl openjdk-11-jdk libzmq3-dev python3 protobuf-compiler build-essential python3-pip libpcre3-dev chrpath tar locales coreutils libssl-dev + +BAZEL_PACKAGE_VERSION="7.2.1" +BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" +BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" + + +curl -L --output "$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" && \ +apt install -y "./$BAZEL_PACKAGE_FILE" && \ +rm "$BAZEL_PACKAGE_FILE" && \ + +apt -y clean +apt -y autoremove + + +curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \ + tar zxf swig-2.0.4.tar.gz && \ + (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \ + rm -rf swig-2.0.4 swig-2.0.4.tar.gz + + +pip install numpy + + +cat >"$OUT_ENV_FILE" <