#969: Use new CTPG parser in java vm (#455)

related to exasol/script-languages-release#969 --------- Co-authored-by: Torsten Kilias <[email protected]>
exasol · Oct 15, 2024 · a2a2f0b · a2a2f0b
1 parent c52fc44
commit a2a2f0b
Show file tree

Hide file tree

Showing 40 changed files with 1,123 additions and 167 deletions.
diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml
@@ -6,38 +6,84 @@ on:
       - master
   pull_request:
 
-
+env:
+  USE_BAZEL_VERSION: 7.2.1
+  UDF_CLIENT_ENV_FILE: /tmp/.udf_client_env
 jobs:
   build:
-    runs-on: ubuntu-latest
-
-    env:
-      USE_BAZEL_VERSION: 7.2.1
+    runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+      - name: Search for duplicated error codes
+        run: bash find_duplicate_error_codes.sh
 
+      - name: Install UDF Client Dependencies
+        run: |
+          sudo bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE"
+      - name: Build
+        run: |
+          source "$UDF_CLIENT_ENV_FILE"
+          bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --verbose_failures
+        working-directory: ./exaudfclient/
 
-    - name: Search for duplicated error codes
-      run: bash find_duplicate_error_codes.sh
-
-    - name: Install bazel
-      run: | 
-        curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64
-        chmod +x bazel
-    - name: Install JDK and ZMQ
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y openjdk-11-jdk libzmq3-dev
-    - name: Java Tests
-      run: |
-        bazel test //base/javacontainer/test/...
-      working-directory: ./exaudfclient/
-    - name: ExaudfLib Tests
-      run: |
-        bazel test //base/exaudflib/test/...
-      working-directory: ./exaudfclient/
-    - name: Script Options Parser Tests
-      run: |
-        bazel test //base/script_options_parser/...
-      working-directory: ./exaudfclient/
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+     fail-fast: false
+     matrix:
+        include:
+          - test: "//base/javacontainer/test:ExaStackTraceCleanerTest"
+            name: "ExaStackTraceCleanerTest"
+          - test: "//base/javacontainer/test:javacontainer-test-legacy-parser"
+            name: "javacontainer-test-legacy-parser"
+          - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser"
+            name: "javacontainer-test-ctpg-parser"
+          - test: "//base/javacontainer/script_options/..."
+            name: "javacontainer-script_options"
+          - test: "//base/exaudflib/test/..."
+            name: "exaudflib"
+          - test: "//base/script_options_parser/ctpg/..."
+            name: "script_options_parser_ctpg"
+          - test: "//base/script_options_parser/legacy/..."
+            name: "script_options_parser_legacy"
+          - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/javacontainer/test:javacontainer-test-legacy-parser"
+            name: "javacontainer-test-legacy-parser-with-valgrind"
+          - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/javacontainer/test:javacontainer-test-ctpg-parser"
+            name: "javacontainer-test-ctpg-parser-with-valgrind"
+          - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/script_options_parser/ctpg/..."
+            name: "script_options_parser_ctpg_with_valgrind"
+          - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/script_options_parser/legacy/..."
+            name: "script_options_parser_legacy_with_valgrind"
+          - test: "--config=asan //base/javacontainer/test:javacontainer-test-legacy-parser"
+            name: "javacontainer-test-legacy-parser-with-asan"
+          - test: "--config=asan //base/javacontainer/test:javacontainer-test-ctpg-parser"
+            name: "javacontainer-test-ctpg-parser-with-asan"
+          - test: "--config=asan //base/script_options_parser/ctpg/..."
+            name: "script_options_parser_ctpg_with_asan"
+          - test: "--config=asan //base/script_options_parser/legacy/..."
+            name: "script_options_parser_legacy_with_asan"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install JDK and ZMQ
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y openjdk-11-jdk libzmq3-dev valgrind
+      - name: Install bazel
+        run: | 
+          BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb"
+          BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE"
+          curl -L --output /tmp/"$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL"
+          sudo apt install -y "/tmp/$BAZEL_PACKAGE_FILE"
+          rm "/tmp/$BAZEL_PACKAGE_FILE"
+        env:
+          BAZEL_PACKAGE_VERSION: "7.2.1"
+      - name: Run tests
+        run: |
+          bazel test ${{ matrix.additional_args }} ${{ matrix.test }}
+        working-directory: ./exaudfclient/
 
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: "${{ matrix.name }}"
+          path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/**/test.log
diff --git a/.github/workflows/test_package_management_scripts.yaml b/.github/workflows/test_package_management_scripts.yaml
@@ -17,6 +17,8 @@ jobs:
       with:
         python-version: "3.10"
 
+    - uses: r-lib/actions/setup-r@v2
+
     - name: Init submodules
       run: git submodule update --init --recursive
 

diff --git a/exaudfclient/.bazelrc b/exaudfclient/.bazelrc
@@ -13,3 +13,16 @@ build:optimize --copt="-g0" --copt="-DNDEBUG" --copt=-fstack-protector-strong --
 build:no-tty --curses=no --color=no
 build:debug-build --sandbox_debug --config=verbose
 build:no-symlinks --symlink_prefix=/
+build:asan --strip=never
+build:asan --copt -fsanitize=address
+build:asan --copt -DADDRESS_SANITIZER
+build:asan --copt -O1
+build:asan --copt -g
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address
+build:asan -c dbg
+build:valgrind --copt -O1
+build:valgrind -c dbg
+build:valgrind --copt -g
+build:valgrind --strip=never
+build:valgrind --copt -DVALGRIND_ACTIVE
diff --git a/exaudfclient/BUILD b/exaudfclient/BUILD
@@ -78,7 +78,7 @@ cc_binary(
     srcs = ["exaudfclient.cc", "//base:load_dynamic"],
     linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
     deps = ["//base/exaudflib:header", "//base/utils:utils"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+
-           ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"],
+           ["//base/exaudflib:exaudflib-deps"],
     defines = VM_ENABLED_DEFINES,
     data = ["//base:libexaudflib_complete.so"]
 )
@@ -101,7 +101,7 @@ cc_binary(
     srcs = ["exaudfclient.cc", "//base:load_dynamic"],
     linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
     deps = ["//base/exaudflib:header", "//base/utils:utils"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+
-            ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"] +
+            ["//base/exaudflib:exaudflib-deps"] +
             [ "@zmq//:zmq", "@protobuf//:protobuf"],
     defines = VM_ENABLED_DEFINES,
     data = ["//base:libexaudflib_complete.so"],

diff --git a/exaudfclient/base/javacontainer/BUILD b/exaudfclient/base/javacontainer/BUILD
@@ -106,11 +106,12 @@ cc_library(
 
 cc_library(
     name = "javacontainer",
-    srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", ":dummy"],
+    srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h",
+            ":javacontainer_builder.h", ":javacontainer_builder.cc", ":dummy"],
     hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"],
-    deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//base/exaudflib:header",
+    deps = ["@java//:java", ":exascript_java", "//base/exaudflib:header",
             "//base/utils:utils","//base/javacontainer/script_options:java_script_option_lines",
-            "//base/swig_factory:swig_factory_if"],
+            "//base/swig_factory:swig_factory"],
 #    copts= ["-O0","-fno-lto"],
     alwayslink=True,
 )

diff --git a/exaudfclient/base/javacontainer/javacontainer.cc b/exaudfclient/base/javacontainer/javacontainer.cc
@@ -1,12 +1,13 @@
 #include "base/javacontainer/javacontainer.h"
 #include "base/javacontainer/javacontainer_impl.h"
+#include "base/javacontainer/script_options/parser.h"
 
 using namespace SWIGVMContainers;
 using namespace std;
 
-JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory) {
+JavaVMach::JavaVMach(bool checkOnly, std::unique_ptr<JavaScriptOptions::ScriptOptionsParser> scriptOptionsParser) {
     try {
-        m_impl = new JavaVMImpl(checkOnly, false, swigFactory);
+        m_impl = new JavaVMImpl(checkOnly, false, std::move(scriptOptionsParser));
     } catch (std::exception& err) {
         lock_guard<mutex> lock(exception_msg_mtx);
         exception_msg = "F-UDF-CL-SL-JAVA-1000: "+std::string(err.what());

diff --git a/exaudfclient/base/javacontainer/javacontainer.h b/exaudfclient/base/javacontainer/javacontainer.h
@@ -4,17 +4,26 @@
 
 #include "base/exaudflib/vm/swig_vm.h"
 #include <string.h>
+#include <memory>
 
 #ifdef ENABLE_JAVA_VM
 
 namespace SWIGVMContainers {
 
 class JavaVMImpl;
-class SwigFactory;
+
+namespace JavaScriptOptions {
+
+struct ScriptOptionsParser;
+
+}
 
 class JavaVMach: public SWIGVM {
     public:
-        JavaVMach(bool checkOnly, SwigFactory& swigFactory);
+        /*
+         * scriptOptionsParser: JavaVMach takes ownership of ScriptOptionsParser pointer.
+         */
+        JavaVMach(bool checkOnly, std::unique_ptr<JavaScriptOptions::ScriptOptionsParser> scriptOptionsParser);
         virtual ~JavaVMach() {}
         virtual void shutdown();
         virtual bool run();

diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.cc b/exaudfclient/base/javacontainer/javacontainer_builder.cc
@@ -0,0 +1,32 @@
+#include "base/javacontainer/javacontainer_builder.h"
+#include "base/javacontainer/script_options/parser_ctpg.h"
+#include "base/javacontainer/script_options/parser_legacy.h"
+#include "base/swig_factory/swig_factory_impl.h"
+
+#ifdef ENABLE_JAVA_VM
+
+namespace SWIGVMContainers {
+
+JavaContainerBuilder::JavaContainerBuilder()
+: m_useCtpgParser(false) {}
+
+JavaContainerBuilder& JavaContainerBuilder::useCtpgParser(const bool useCtpgParser) {
+    m_useCtpgParser = useCtpgParser;
+    return *this;
+}
+
+JavaVMach* JavaContainerBuilder::build() {
+    std::unique_ptr<JavaScriptOptions::ScriptOptionsParser> parser;
+    if (m_useCtpgParser) {
+        parser = std::make_unique<JavaScriptOptions::ScriptOptionLinesParserCTPG>(std::make_unique<SwigFactoryImpl>());
+    } else {
+        parser = std::make_unique<JavaScriptOptions::ScriptOptionLinesParserLegacy>(std::make_unique<SwigFactoryImpl>());
+    }
+    return new JavaVMach(false, std::move(parser));
+}
+
+
+} //namespace SWIGVMContainers
+
+
+#endif //ENABLE_JAVA_VM
diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.h b/exaudfclient/base/javacontainer/javacontainer_builder.h
@@ -0,0 +1,35 @@
+#ifndef JAVACONTAINER_BUILDER_H
+#define JAVACONTAINER_BUILDER_H
+
+#include <memory>
+#include "base/javacontainer/javacontainer.h"
+
+#ifdef ENABLE_JAVA_VM
+
+namespace SWIGVMContainers {
+
+namespace JavaScriptOptions {
+
+struct ScriptOptionsParser;
+
+}
+
+class JavaContainerBuilder {
+    public:
+        JavaContainerBuilder();
+
+        JavaContainerBuilder& useCtpgParser(const bool value);
+
+        JavaVMach* build();
+
+    private:
+        bool m_useCtpgParser;
+};
+
+} //namespace SWIGVMContainers
+
+
+#endif //ENABLE_JAVA_VM
+
+
+#endif //JAVACONTAINER_BUILDER_H
diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc
@@ -12,14 +12,14 @@
 #include "base/javacontainer/javacontainer.h"
 #include "base/javacontainer/javacontainer_impl.h"
 #include "base/javacontainer/script_options/extractor.h"
-#include "base/javacontainer/script_options/parser_legacy.h"
-#include "base/swig_factory/swig_factory.h"
+#include "base/javacontainer/script_options/parser.h"
 
 
 using namespace SWIGVMContainers;
 using namespace std;
 
-JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory)
+JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI,
+                        std::unique_ptr<JavaScriptOptions::ScriptOptionsParser> scriptOptionsParser)
 : m_checkOnly(checkOnly)
 , m_exaJavaPath("")
 , m_localClasspath("/tmp") // **IMPORTANT**: /tmp needs to be in the classpath, otherwise ExaCompiler crashe with com.exasol.ExaCompilationException: /DATE_STRING.java:3: error: error while writing DATE_STRING: could not create parent directories
@@ -28,26 +28,12 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory)
 , m_jvm(NULL)
 , m_env(NULL)
 , m_needsCompilation(true)
-, m_swigFactory(swigFactory)
 {
 
     stringstream ss;
     m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path
 
-    JavaScriptOptions::ScriptOptionLinesParserLegacy scriptOptionsParser;
-
-    JavaScriptOptions::Extractor extractor(scriptOptionsParser, swigFactory);
-
-    DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode));  // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used
-
-    DBG_FUNC_CALL(cerr,setClasspath());
-
-    m_jvmOptions = std::move(extractor.moveJvmOptions());
-
-    for (set<string>::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end();
-         ++it) {
-        addJarToClasspath(*it);
-    }
+    parseScriptOptions(std::move(scriptOptionsParser));
 
     m_needsCompilation = checkNeedsCompilation();
     if (m_needsCompilation) {
@@ -64,6 +50,21 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory)
     }
 }
 
+void JavaVMImpl::parseScriptOptions(std::unique_ptr<JavaScriptOptions::ScriptOptionsParser> scriptOptionsParser) {
+    JavaScriptOptions::Extractor extractor(*scriptOptionsParser);
+
+    DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode));
+
+    DBG_FUNC_CALL(cerr,setClasspath());
+
+    m_jvmOptions = std::move(extractor.moveJvmOptions());
+
+    for (set<string>::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end();
+         ++it) {
+        addJarToClasspath(*it);
+    }
+}
+
 void JavaVMImpl::shutdown() {
     if (m_checkOnly)
         throwException("F-UDF.CL.SL.JAVA-1159: Java VM in check only mode");