#969: Use new CTPG parser in java vm

exasol · Oct 2, 2024 · 2b0ba44 · 2b0ba44
1 parent a9addbb
commit 2b0ba44
Show file tree

Hide file tree

Showing 21 changed files with 400 additions and 50 deletions.
diff --git a/exaudfclient/base/javacontainer/javacontainer.cc b/exaudfclient/base/javacontainer/javacontainer.cc
@@ -4,9 +4,9 @@
 using namespace SWIGVMContainers;
 using namespace std;
 
-JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory) {
+JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory, bool useCTPGParser) {
     try {
-        m_impl = new JavaVMImpl(checkOnly, false, swigFactory);
+        m_impl = new JavaVMImpl(checkOnly, false, swigFactory, useCTPGParser);
     } catch (std::exception& err) {
         lock_guard<mutex> lock(exception_msg_mtx);
         exception_msg = "F-UDF-CL-SL-JAVA-1000: "+std::string(err.what());

diff --git a/exaudfclient/base/javacontainer/javacontainer.h b/exaudfclient/base/javacontainer/javacontainer.h
@@ -14,7 +14,7 @@ class SwigFactory;
 
 class JavaVMach: public SWIGVM {
     public:
-        JavaVMach(bool checkOnly, SwigFactory& swigFactory);
+        JavaVMach(bool checkOnly, SwigFactory& swigFactory, bool useCTPGParser);
         virtual ~JavaVMach() {}
         virtual void shutdown();
         virtual bool run();

diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc
@@ -13,13 +13,14 @@
 #include "base/javacontainer/javacontainer_impl.h"
 #include "base/javacontainer/script_options/extractor.h"
 #include "base/javacontainer/script_options/parser_legacy.h"
+#include "base/javacontainer/script_options/parser_ctpg.h"
 #include "base/swig_factory/swig_factory.h"
 
 
 using namespace SWIGVMContainers;
 using namespace std;
 
-JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory)
+JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, bool useCTPGParser)
 : m_checkOnly(checkOnly)
 , m_exaJavaPath("")
 , m_localClasspath("/tmp") // **IMPORTANT**: /tmp needs to be in the classpath, otherwise ExaCompiler crashe with com.exasol.ExaCompilationException: /DATE_STRING.java:3: error: error while writing DATE_STRING: could not create parent directories
@@ -34,19 +35,12 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory)
     stringstream ss;
     m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path
 
-    JavaScriptOptions::ScriptOptionLinesParserLegacy scriptOptionsParser;
-    JavaScriptOptions::Extractor extractor(scriptOptionsParser, swigFactory,
-                                           [&](const std::string &msg){throwException(msg);});
-
-    DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode));  // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used
-
-    DBG_FUNC_CALL(cerr,setClasspath());
-
-    m_jvmOptions = std::move(extractor.moveJvmOptions());
-
-    for (set<string>::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end();
-         ++it) {
-        addJarToClasspath(*it);
+    if (useCTPGParser) {
+        JavaScriptOptions::ScriptOptionLinesParserCTPG parser;
+        parseScriptOptions(parser, swigFactory);
+    } else {
+        JavaScriptOptions::ScriptOptionLinesParserLegacy parser;
+        parseScriptOptions(parser, swigFactory);
     }
 
     m_needsCompilation = checkNeedsCompilation();
@@ -64,6 +58,22 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory)
     }
 }
 
+void JavaVMImpl::parseScriptOptions(JavaScriptOptions::ScriptOptionsParser & scriptOptionsParser, SwigFactory& swigFactory) {
+    JavaScriptOptions::Extractor extractor(scriptOptionsParser, swigFactory,
+                                           [&](const std::string &msg){throwException(msg);});
+
+    DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode));  // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used
+
+    DBG_FUNC_CALL(cerr,setClasspath());
+
+    m_jvmOptions = std::move(extractor.moveJvmOptions());
+
+    for (set<string>::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end();
+         ++it) {
+        addJarToClasspath(*it);
+    }
+}
+
 void JavaVMImpl::shutdown() {
     if (m_checkOnly)
         throwException("F-UDF.CL.SL.JAVA-1159: Java VM in check only mode");

diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.h b/exaudfclient/base/javacontainer/javacontainer_impl.h
@@ -14,10 +14,14 @@ class JavaVMTest;
 
 namespace SWIGVMContainers {
 
+namespace JavaScriptOptions {
+    struct ScriptOptionsParser;
+}
+
 class JavaVMImpl {
     public:
         friend class ::JavaVMTest;
-        JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory);
+        JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, bool useCTPGParser);
         ~JavaVMImpl() {}
         void shutdown();
         bool run();
@@ -37,6 +41,7 @@ class JavaVMImpl {
         void throwException(const std::string& ex);
         void setJvmOptions();
         void addJarToClasspath(const std::string& path);
+        void parseScriptOptions(JavaScriptOptions::ScriptOptionsParser & scriptOptionsParser, SwigFactory& swigFactory);
         bool m_checkOnly;
         std::string m_exaJavaPath;
         std::string m_localClasspath;

diff --git a/exaudfclient/base/javacontainer/script_options/BUILD b/exaudfclient/base/javacontainer/script_options/BUILD
@@ -3,9 +3,11 @@ package(default_visibility = ["//visibility:public"])
 
 cc_library(
     name = "java_script_option_lines",
-    hdrs = [":extractor.h", ":parser_legacy.h"],
+    hdrs = [":extractor.h", ":parser_legacy.h", ":parser_ctpg.h"],
     srcs = [":parser.h", ":converter.h", ":converter.cc", ":parser_legacy.cc", ":extractor.cc",
-            ":keywords.h", ":checksum.h", ":checksum.cc"],
-    deps = ["//base/script_options_parser/legacy:script_option_lines_parser_legacy", "//base:debug_message_h",
+            ":keywords.h", ":keywords.cc", ":checksum.h", ":checksum.cc", ":parser_ctpg.cc",
+            ":parser_ctpg_script_importer.cc", ":parser_ctpg_script_importer.h"],
+    deps = ["//base/script_options_parser/legacy:script_option_lines_parser_legacy",
+            "//base/script_options_parser/ctpg:script_option_lines_parser_ctpg", "//base:debug_message_h",
             "//base/exaudflib:header", "//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory_if"],
 )
diff --git a/exaudfclient/base/javacontainer/script_options/extractor.cc b/exaudfclient/base/javacontainer/script_options/extractor.cc
@@ -29,7 +29,7 @@ void Extractor::extract(std::string & scriptCode) {
     EXTR_DBG_FUNC_CALL(m_parser.parseForExternalJars( [&](const std::string& value){
             EXTR_DBG_FUNC_CALL(m_converter.convertExternalJar(value));
         }, m_throwException));
-    scriptCode = std::move(m_parser.getScriptCode());
+    scriptCode = std::move(m_parser.getScriptCode(m_throwException));
 }
 
 } //namespace JavaScriptOptions

diff --git a/exaudfclient/base/javacontainer/script_options/keywords.cc b/exaudfclient/base/javacontainer/script_options/keywords.cc
@@ -0,0 +1,33 @@
+#include "base/javacontainer/script_options/keywords.h"
+#include <string_view>
+
+namespace SWIGVMContainers {
+
+namespace JavaScriptOptions {
+
+Keywords::Keywords(bool withScriptOptionsPrefix)
+: m_jarKeyword()
+, m_scriptClassKeyword()
+, m_importKeyword()
+, m_jvmKeyword() {
+    const std::string_view jar{"%jar"};
+    const std::string_view scriptClass{"%scriptclass"};
+    const std::string_view import{"%import"};
+    const std::string_view jvm{"%jvmoption"};
+    if (withScriptOptionsPrefix) {
+        m_jarKeyword = jar;
+        m_scriptClassKeyword = scriptClass;
+        m_importKeyword = import;
+        m_jvmKeyword = jvm;
+    } else {
+        m_jarKeyword.assign(jar.substr(1));
+        m_scriptClassKeyword.assign(scriptClass.substr(1));
+        m_importKeyword.assign(import.substr(1));
+        m_jvmKeyword.assign(jvm.substr(1));
+    }
+}
+
+} //namespace JavaScriptOptions
+
+} //namespace SWIGVMContainers
+
diff --git a/exaudfclient/base/javacontainer/script_options/keywords.h b/exaudfclient/base/javacontainer/script_options/keywords.h
@@ -9,20 +9,16 @@ namespace JavaScriptOptions {
 
 class Keywords {
     public:
-        Keywords()
-            : m_jarKeyword("%jar")
-            , m_scriptClassKeyword("%scriptclass")
-            , m_importKeyword("%import")
-            , m_jvmOptionKeyword("%jvmoption") {}
-        const std::string & jarKeyword() { return m_jarKeyword; }
+        Keywords(bool withScriptOptionsPrefix);
         const std::string & scriptClassKeyword() { return m_scriptClassKeyword; }
         const std::string & importKeyword() { return m_importKeyword; }
-        const std::string & jvmOptionKeyword() { return m_jvmOptionKeyword; }
+        const std::string & jvmKeyword() { return m_jvmKeyword; }
+        const std::string & jarKeyword() { return m_jarKeyword; }
     private:
         std::string m_jarKeyword;
         std::string m_scriptClassKeyword;
         std::string m_importKeyword;
-        std::string m_jvmOptionKeyword;
+        std::string m_jvmKeyword;
 };
 
 } //namespace JavaScriptOptions

diff --git a/exaudfclient/base/javacontainer/script_options/parser.h b/exaudfclient/base/javacontainer/script_options/parser.h
@@ -51,7 +51,7 @@ struct ScriptOptionsParser {
     /*
      Returns the (eventually modified) script code.
     */
-    virtual std::string && getScriptCode() = 0;
+    virtual std::string && getScriptCode(std::function<void(const std::string&)> throwException) = 0;
 };
 
 } //namespace JavaScriptOptions

diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc
@@ -1,9 +1,128 @@
 #include "base/javacontainer/script_options/parser_ctpg.h"
+#include "base/javacontainer/script_options/parser_ctpg_script_importer.h"
+#include <sstream>
+
+namespace ctpg_parser = ExecutionGraph::OptionsLineParser::CTPG;
 
 namespace SWIGVMContainers {
 
 namespace JavaScriptOptions {
 
+ScriptOptionLinesParserCTPG::ScriptOptionLinesParserCTPG()
+: m_scriptCode()
+, m_keywords(false)
+, m_needParsing(true) {}
+
+void ScriptOptionLinesParserCTPG::prepareScriptCode(const std::string & scriptCode) {
+    m_scriptCode = scriptCode;
+}
+
+void ScriptOptionLinesParserCTPG::parseForScriptClass(std::function<void(const std::string &option)> callback,
+                         std::function<void(const std::string&)> throwException) {
+    parseForSingleOption(m_keywords.scriptClassKeyword(), callback,
+                            [&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1623 " + msg);});
+}
+
+void ScriptOptionLinesParserCTPG::parseForJvmOptions(std::function<void(const std::string &option)> callback,
+                        std::function<void(const std::string&)> throwException) {
+    parseForMultipleOption(m_keywords.jvmKeyword(), callback,
+                            [&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1624 " + msg);});
+}
+
+void ScriptOptionLinesParserCTPG::parseForExternalJars(std::function<void(const std::string &option)> callback,
+                          std::function<void(const std::string&)> throwException) {
+    parseForMultipleOption(m_keywords.jarKeyword(), callback,
+                            [&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1625 " + msg);});
+}
+
+void ScriptOptionLinesParserCTPG::extractImportScripts(SwigFactory & swigFactory,
+                          std::function<void(const std::string&)> throwException) {
+    parse([&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1626 " + msg);});
+
+    const auto optionIt = m_foundOptions.find(m_keywords.importKeyword());
+    if (optionIt != m_foundOptions.end()) {
+        CTPG::ScriptImporter scriptImporter(swigFactory, m_keywords);
+        scriptImporter.importScript(m_scriptCode, m_foundOptions, throwException);
+        //The imported scripts will change the location of the other options in m_foundOptions
+        //Also there might be new JVM / External Jar options
+        //=> We need to clear the option map and reset the parser.
+        m_foundOptions.clear();
+        m_needParsing = true;
+    }
+}
+
+std::string && ScriptOptionLinesParserCTPG::getScriptCode(std::function<void(const std::string&)> throwException) {
+    parse([&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1627 " + msg);});
+    //Remove all options from script code in reverse order
+    struct option_location {
+        size_t pos;
+        size_t len;
+    };
+    struct comp {
+        bool operator()(option_location a, option_location b) const {
+            return a.pos > b.pos;
+        }
+    };
+    std::set<option_location, comp> option_locations;
+    for (const auto & option: m_foundOptions) {
+        for (const auto & option_loc: option.second) {
+            option_location loc = { .pos = option_loc.idx_in_source, .len = option_loc.size};
+            option_locations.insert(loc);
+        }
+    }
+    for (const auto option_loc: option_locations) {
+        m_scriptCode.erase(option_loc.pos, option_loc.len);
+    }
+    return std::move(m_scriptCode);
+}
+
+void ScriptOptionLinesParserCTPG::parse(std::function<void(const std::string&)> throwException) {
+    if (m_needParsing) {
+        if(!m_foundOptions.empty()) {
+            throwException("F-UDF-CL-SL-JAVA-1620 Internal error. Parser result is not empty.");
+        }
+        ExecutionGraph::OptionsLineParser::CTPG::parseOptions(m_scriptCode, m_foundOptions,
+                                            [&](const char* msg){throwException(std::string("F-UDF-CL-SL-JAVA-1621 ") + msg);});
+        m_needParsing = false;
+
+        //Check for unknown options
+        for (const auto & option: m_foundOptions) {
+            if (m_keywords.jarKeyword() != option.first &&
+                m_keywords.scriptClassKeyword() != option.first  &&
+                m_keywords.importKeyword() != option.first  &&
+                m_keywords.jvmKeyword() != option.first) {
+                    std::stringstream ss;
+                    ss << "F-UDF-CL-SL-JAVA-1622 " << "Unexpected option: " << option.first;
+                    throwException(ss.str());
+                }
+        }
+    }
+}
+
+void ScriptOptionLinesParserCTPG::parseForSingleOption(const std::string key, std::function<void(const std::string &option)> callback,
+                            std::function<void(const std::string&)> throwException) {
+    parse(throwException);
+    const auto optionIt = m_foundOptions.find(key);
+    if (optionIt != m_foundOptions.end()) {
+        if (optionIt->second.size() != 1) {
+            std::stringstream ss;
+            ss << "F-UDF-CL-SL-JAVA-1628 found " << optionIt->second.size() <<  key << " options" << std::endl;
+            throwException(ss.str());
+        }
+        callback(optionIt->second[0].value);
+    }
+}
+
+void ScriptOptionLinesParserCTPG::parseForMultipleOption(const std::string key, std::function<void(const std::string &option)> callback,
+                            std::function<void(const std::string&)> throwException) {
+    parse(throwException);
+    const auto optionIt = m_foundOptions.find(key);
+    if (optionIt != m_foundOptions.end()) {
+        for (const auto & option : optionIt->second) {
+            callback(option.value);
+        }
+    }
+}
 
 } //namespace JavaScriptOptions
 

diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.h b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h
@@ -1,10 +1,53 @@
 #ifndef SCRIPTOPTIONLINEPARSERCTPGY_H
 #define SCRIPTOPTIONLINEPARSERCTPGY_H 1
 
+#include "base/javacontainer/script_options/parser.h"
+#include "base/javacontainer/script_options/keywords.h"
+#include "base/script_options_parser/ctpg/script_option_lines_ctpg.h"
+
+
 namespace SWIGVMContainers {
 
 namespace JavaScriptOptions {
 
+class ScriptOptionLinesParserCTPG : public ScriptOptionsParser {
+
+    public:
+        ScriptOptionLinesParserCTPG();
+
+        void prepareScriptCode(const std::string & scriptCode) override;
+
+        void parseForScriptClass(std::function<void(const std::string &option)> callback,
+                                 std::function<void(const std::string&)> throwException) override;
+
+        void parseForJvmOptions(std::function<void(const std::string &option)> callback,
+                                std::function<void(const std::string&)> throwException) override;
+
+        void parseForExternalJars(std::function<void(const std::string &option)> callback,
+                                  std::function<void(const std::string&)> throwException) override;
+
+        void extractImportScripts(SwigFactory & swigFactory,
+                                  std::function<void(const std::string&)> throwException) override;
+
+        std::string && getScriptCode(std::function<void(const std::string&)> throwException) override;
+
+    private:
+        void parse(std::function<void(const std::string&)> throwException);
+
+        void parseForSingleOption(const std::string key, std::function<void(const std::string &option)> callback,
+                                    std::function<void(const std::string&)> throwException);
+        void parseForMultipleOption(const std::string key, std::function<void(const std::string &option)> callback,
+                                    std::function<void(const std::string&)> throwException);
+
+        void importScripts(SwigFactory & swigFactory,
+                                  std::function<void(const std::string&)> throwException);
+
+    private:
+        std::string m_scriptCode;
+        Keywords m_keywords;
+        ExecutionGraph::OptionsLineParser::CTPG::options_map_t m_foundOptions;
+        bool m_needParsing;
+};
 
 } //namespace JavaScriptOptions