From 77e9955ab7db9e598f35d51aa69786d9178c2c1b Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 2 Oct 2024 07:23:00 -0300 Subject: [PATCH 01/43] #969: Use new CTPG parser in java vm --- .../base/javacontainer/javacontainer.cc | 4 +- .../base/javacontainer/javacontainer.h | 2 +- .../base/javacontainer/javacontainer_impl.cc | 37 +++-- .../base/javacontainer/javacontainer_impl.h | 7 +- .../base/javacontainer/script_options/BUILD | 8 +- .../javacontainer/script_options/keywords.cc | 33 +++++ .../javacontainer/script_options/keywords.h | 12 +- .../script_options/parser_ctpg.cc | 138 ++++++++++++++++++ .../script_options/parser_ctpg.h | 36 +++++ .../parser_ctpg_script_importer.cc | 87 +++++++++++ .../parser_ctpg_script_importer.h | 42 ++++++ .../script_options/parser_legacy.cc | 8 +- .../script_options/parser_legacy.h | 4 +- exaudfclient/base/javacontainer/test/BUILD | 20 ++- .../test/cpp/javacontainer_test.cc | 5 +- .../javacontainer/test/cpp/javavm_test.cc | 6 +- .../test/cpp/swig_factory_test.cc | 4 +- .../ctpg/script_option_lines_ctpg.h | 2 - exaudfclient/exaudfclient.cc | 2 +- 19 files changed, 411 insertions(+), 46 deletions(-) create mode 100644 exaudfclient/base/javacontainer/script_options/keywords.cc create mode 100644 exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc create mode 100644 exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h diff --git a/exaudfclient/base/javacontainer/javacontainer.cc b/exaudfclient/base/javacontainer/javacontainer.cc index 797506cb..df571f02 100644 --- a/exaudfclient/base/javacontainer/javacontainer.cc +++ b/exaudfclient/base/javacontainer/javacontainer.cc @@ -4,9 +4,9 @@ using namespace SWIGVMContainers; using namespace std; -JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory) { +JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory, bool useCTPGParser) { try { - m_impl = new JavaVMImpl(checkOnly, false, swigFactory); + m_impl = new JavaVMImpl(checkOnly, false, swigFactory, useCTPGParser); } catch (std::exception& err) { lock_guard lock(exception_msg_mtx); exception_msg = "F-UDF-CL-SL-JAVA-1000: "+std::string(err.what()); diff --git a/exaudfclient/base/javacontainer/javacontainer.h b/exaudfclient/base/javacontainer/javacontainer.h index dc7ba418..46fd0670 100644 --- a/exaudfclient/base/javacontainer/javacontainer.h +++ b/exaudfclient/base/javacontainer/javacontainer.h @@ -14,7 +14,7 @@ class SwigFactory; class JavaVMach: public SWIGVM { public: - JavaVMach(bool checkOnly, SwigFactory& swigFactory); + JavaVMach(bool checkOnly, SwigFactory& swigFactory, bool useCTPGParser); virtual ~JavaVMach() {} virtual void shutdown(); virtual bool run(); diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc index 96aa0898..76ad8165 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.cc +++ b/exaudfclient/base/javacontainer/javacontainer_impl.cc @@ -13,13 +13,14 @@ #include "base/javacontainer/javacontainer_impl.h" #include "base/javacontainer/script_options/extractor.h" #include "base/javacontainer/script_options/parser_legacy.h" +#include "base/javacontainer/script_options/parser_ctpg.h" #include "base/swig_factory/swig_factory.h" using namespace SWIGVMContainers; using namespace std; -JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory) +JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, bool useCTPGParser) : m_checkOnly(checkOnly) , m_exaJavaPath("") , m_localClasspath("/tmp") // **IMPORTANT**: /tmp needs to be in the classpath, otherwise ExaCompiler crashe with com.exasol.ExaCompilationException: /DATE_STRING.java:3: error: error while writing DATE_STRING: could not create parent directories @@ -34,19 +35,12 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory) stringstream ss; m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path - JavaScriptOptions::ScriptOptionLinesParserLegacy scriptOptionsParser; - - JavaScriptOptions::Extractor extractor(scriptOptionsParser, swigFactory); - - DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used - - DBG_FUNC_CALL(cerr,setClasspath()); - - m_jvmOptions = std::move(extractor.moveJvmOptions()); - - for (set::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end(); - ++it) { - addJarToClasspath(*it); + if (useCTPGParser) { + JavaScriptOptions::ScriptOptionLinesParserCTPG parser; + parseScriptOptions(parser, swigFactory); + } else { + JavaScriptOptions::ScriptOptionLinesParserLegacy parser; + parseScriptOptions(parser, swigFactory); } m_needsCompilation = checkNeedsCompilation(); @@ -64,6 +58,21 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory) } } +void JavaVMImpl::parseScriptOptions(JavaScriptOptions::ScriptOptionsParser & scriptOptionsParser, SwigFactory& swigFactory) { + JavaScriptOptions::Extractor extractor(scriptOptionsParser, swigFactory); + + DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used + + DBG_FUNC_CALL(cerr,setClasspath()); + + m_jvmOptions = std::move(extractor.moveJvmOptions()); + + for (set::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end(); + ++it) { + addJarToClasspath(*it); + } +} + void JavaVMImpl::shutdown() { if (m_checkOnly) throwException("F-UDF.CL.SL.JAVA-1159: Java VM in check only mode"); diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.h b/exaudfclient/base/javacontainer/javacontainer_impl.h index f4f093be..d423f32f 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.h +++ b/exaudfclient/base/javacontainer/javacontainer_impl.h @@ -14,10 +14,14 @@ class JavaVMTest; namespace SWIGVMContainers { +namespace JavaScriptOptions { + struct ScriptOptionsParser; +} + class JavaVMImpl { public: friend class ::JavaVMTest; - JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory); + JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, bool useCTPGParser); ~JavaVMImpl() {} void shutdown(); bool run(); @@ -37,6 +41,7 @@ class JavaVMImpl { void throwException(const std::string& ex); void setJvmOptions(); void addJarToClasspath(const std::string& path); + void parseScriptOptions(JavaScriptOptions::ScriptOptionsParser & scriptOptionsParser, SwigFactory& swigFactory); bool m_checkOnly; std::string m_exaJavaPath; std::string m_localClasspath; diff --git a/exaudfclient/base/javacontainer/script_options/BUILD b/exaudfclient/base/javacontainer/script_options/BUILD index 585f8a31..80af6e98 100644 --- a/exaudfclient/base/javacontainer/script_options/BUILD +++ b/exaudfclient/base/javacontainer/script_options/BUILD @@ -3,10 +3,12 @@ package(default_visibility = ["//visibility:public"]) cc_library( name = "java_script_option_lines", - hdrs = [":extractor.h", ":parser_legacy.h"], + hdrs = [":extractor.h", ":parser_legacy.h", ":parser_ctpg.h"], srcs = [":parser.h", ":converter.h", ":converter.cc", ":parser_legacy.cc", ":extractor.cc", - ":keywords.h", ":checksum.h", ":checksum.cc"], - deps = ["//base/script_options_parser/legacy:script_option_lines_parser_legacy", "//base/utils:utils", + ":keywords.h", ":keywords.cc", ":checksum.h", ":checksum.cc", ":parser_ctpg.cc", + ":parser_ctpg_script_importer.cc", ":parser_ctpg_script_importer.h"], + deps = ["//base/script_options_parser/legacy:script_option_lines_parser_legacy", + "//base/script_options_parser/ctpg:script_option_lines_parser_ctpg", "//base/utils:utils", "//base/exaudflib:header", "//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory_if", "//base/script_options_parser:exception"], ) diff --git a/exaudfclient/base/javacontainer/script_options/keywords.cc b/exaudfclient/base/javacontainer/script_options/keywords.cc new file mode 100644 index 00000000..9d89bf38 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/keywords.cc @@ -0,0 +1,33 @@ +#include "base/javacontainer/script_options/keywords.h" +#include + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +Keywords::Keywords(bool withScriptOptionsPrefix) +: m_jarKeyword() +, m_scriptClassKeyword() +, m_importKeyword() +, m_jvmKeyword() { + const std::string_view jar{"%jar"}; + const std::string_view scriptClass{"%scriptclass"}; + const std::string_view import{"%import"}; + const std::string_view jvm{"%jvmoption"}; + if (withScriptOptionsPrefix) { + m_jarKeyword = jar; + m_scriptClassKeyword = scriptClass; + m_importKeyword = import; + m_jvmKeyword = jvm; + } else { + m_jarKeyword.assign(jar.substr(1)); + m_scriptClassKeyword.assign(scriptClass.substr(1)); + m_importKeyword.assign(import.substr(1)); + m_jvmKeyword.assign(jvm.substr(1)); + } +} + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + diff --git a/exaudfclient/base/javacontainer/script_options/keywords.h b/exaudfclient/base/javacontainer/script_options/keywords.h index 840509a9..314bf5b6 100644 --- a/exaudfclient/base/javacontainer/script_options/keywords.h +++ b/exaudfclient/base/javacontainer/script_options/keywords.h @@ -9,20 +9,16 @@ namespace JavaScriptOptions { class Keywords { public: - Keywords() - : m_jarKeyword("%jar") - , m_scriptClassKeyword("%scriptclass") - , m_importKeyword("%import") - , m_jvmOptionKeyword("%jvmoption") {} - const std::string & jarKeyword() { return m_jarKeyword; } + Keywords(bool withScriptOptionsPrefix); const std::string & scriptClassKeyword() { return m_scriptClassKeyword; } const std::string & importKeyword() { return m_importKeyword; } - const std::string & jvmOptionKeyword() { return m_jvmOptionKeyword; } + const std::string & jvmKeyword() { return m_jvmKeyword; } + const std::string & jarKeyword() { return m_jarKeyword; } private: std::string m_jarKeyword; std::string m_scriptClassKeyword; std::string m_importKeyword; - std::string m_jvmOptionKeyword; + std::string m_jvmKeyword; }; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc index cd73abb7..f4ef7983 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc @@ -1,9 +1,147 @@ #include "base/javacontainer/script_options/parser_ctpg.h" +#include "base/javacontainer/script_options/parser_ctpg_script_importer.h" +#include "base/utils/exceptions.h" +#include "base/script_options_parser/exception.h" +#include + + +namespace ctpg_parser = ExecutionGraph::OptionsLineParser::CTPG; namespace SWIGVMContainers { namespace JavaScriptOptions { +ScriptOptionLinesParserCTPG::ScriptOptionLinesParserCTPG() +: m_scriptCode() +, m_keywords(false) +, m_needParsing(true) {} + +void ScriptOptionLinesParserCTPG::prepareScriptCode(const std::string & scriptCode) { + m_scriptCode = scriptCode; +} + +void ScriptOptionLinesParserCTPG::parseForScriptClass(std::function callback) { + try { + parseForSingleOption(m_keywords.scriptClassKeyword(), callback); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1623"); + } +} + +void ScriptOptionLinesParserCTPG::parseForJvmOptions(std::function callback) { + try { + parseForMultipleOption(m_keywords.jvmKeyword(), callback); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1624"); + } +} + +void ScriptOptionLinesParserCTPG::parseForExternalJars(std::function callback) { + try { + parseForMultipleOption(m_keywords.jarKeyword(), callback); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1625"); + } +} + +void ScriptOptionLinesParserCTPG::extractImportScripts(SwigFactory & swigFactory) { + + try { + parse(); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1626"); + } + + const auto optionIt = m_foundOptions.find(m_keywords.importKeyword()); + if (optionIt != m_foundOptions.end()) { + CTPG::ScriptImporter scriptImporter(swigFactory, m_keywords); + scriptImporter.importScript(m_scriptCode, m_foundOptions); + //The imported scripts will change the location of the other options in m_foundOptions + //Also there might be new JVM / External Jar options + //=> We need to clear the option map and reset the parser. + m_foundOptions.clear(); + m_needParsing = true; + } +} + +std::string && ScriptOptionLinesParserCTPG::getScriptCode() { + try { + parse(); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1627"); + } + //Remove all options from script code in reverse order + struct option_location { + size_t pos; + size_t len; + }; + struct comp { + bool operator()(option_location a, option_location b) const { + return a.pos > b.pos; + } + }; + std::set option_locations; + for (const auto & option: m_foundOptions) { + for (const auto & option_loc: option.second) { + option_location loc = { .pos = option_loc.idx_in_source, .len = option_loc.size}; + option_locations.insert(loc); + } + } + for (const auto option_loc: option_locations) { + m_scriptCode.erase(option_loc.pos, option_loc.len); + } + return std::move(m_scriptCode); +} + +void ScriptOptionLinesParserCTPG::parse() { + if (m_needParsing) { + if(!m_foundOptions.empty()) { + throw std::logic_error("F-UDF-CL-SL-JAVA-1620 Internal error. Parser result is not empty."); + } + try { + ExecutionGraph::OptionsLineParser::CTPG::parseOptions(m_scriptCode, m_foundOptions); + } catch(const ExecutionGraph::OptionParserException& ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1621"); + } + + m_needParsing = false; + + //Check for unknown options + for (const auto & option: m_foundOptions) { + if (m_keywords.jarKeyword() != option.first && + m_keywords.scriptClassKeyword() != option.first && + m_keywords.importKeyword() != option.first && + m_keywords.jvmKeyword() != option.first) { + std::stringstream ss; + ss << "F-UDF-CL-SL-JAVA-1622 " << "Unexpected option: " << option.first; + throw std::invalid_argument(ss.str()); + } + } + } +} + +void ScriptOptionLinesParserCTPG::parseForSingleOption(const std::string key, std::function callback) { + parse(); + const auto optionIt = m_foundOptions.find(key); + if (optionIt != m_foundOptions.end()) { + if (optionIt->second.size() != 1) { + std::stringstream ss; + ss << "F-UDF-CL-SL-JAVA-1628 found " << optionIt->second.size() << key << " options" << std::endl; + throw std::invalid_argument(ss.str()); + } + callback(optionIt->second[0].value); + } +} + +void ScriptOptionLinesParserCTPG::parseForMultipleOption(const std::string key, std::function callback) { + parse(); + const auto optionIt = m_foundOptions.find(key); + if (optionIt != m_foundOptions.end()) { + for (const auto & option : optionIt->second) { + callback(option.value); + } + } +} } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.h b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h index 3b71a5ee..e9dc917c 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg.h +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h @@ -1,10 +1,46 @@ #ifndef SCRIPTOPTIONLINEPARSERCTPGY_H #define SCRIPTOPTIONLINEPARSERCTPGY_H 1 +#include "base/javacontainer/script_options/parser.h" +#include "base/javacontainer/script_options/keywords.h" +#include "base/script_options_parser/ctpg/script_option_lines_ctpg.h" + + namespace SWIGVMContainers { namespace JavaScriptOptions { +class ScriptOptionLinesParserCTPG : public ScriptOptionsParser { + + public: + ScriptOptionLinesParserCTPG(); + + void prepareScriptCode(const std::string & scriptCode) override; + + void parseForScriptClass(std::function callback) override; + + void parseForJvmOptions(std::function callback) override; + + void parseForExternalJars(std::function callback) override; + + void extractImportScripts(SwigFactory & swigFactory) override; + + std::string && getScriptCode() override; + + private: + void parse(); + + void parseForSingleOption(const std::string key, std::function callback); + void parseForMultipleOption(const std::string key, std::function callback); + + void importScripts(SwigFactory & swigFactory); + + private: + std::string m_scriptCode; + Keywords m_keywords; + ExecutionGraph::OptionsLineParser::CTPG::options_map_t m_foundOptions; + bool m_needParsing; +}; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc new file mode 100644 index 00000000..a3dd297b --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc @@ -0,0 +1,87 @@ +#include "base/javacontainer/script_options/parser_ctpg_script_importer.h" +#include "base/swig_factory/swig_factory.h" +#include "base/utils/debug_message.h" +#include "base/utils/exceptions.h" +#include "base/script_options_parser/exception.h" + +#include +#include +#include + +namespace ctpg_parser = ExecutionGraph::OptionsLineParser::CTPG; + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +namespace CTPG { + +ScriptImporter::ScriptImporter(SwigFactory & swigFactory, Keywords & keywords) +: m_importedScriptChecksums() +, m_swigFactory(swigFactory) +, m_metaData() +, m_keywords(keywords) {} + +void ScriptImporter::importScript(std::string & scriptCode, ctpg_parser::options_map_t & options) { + const auto optionIt = options.find(std::string(m_keywords.importKeyword())); + if (optionIt != options.end()) { + m_importedScriptChecksums.addScript(scriptCode.c_str()); + //Sort options from first in script to last in script + std::sort(optionIt->second.begin(), optionIt->second.end(), + [](const ctpg_parser::ScriptOption& first, const ctpg_parser::ScriptOption& second) + { + return first.idx_in_source < second.idx_in_source; + }); + struct ReplacedScripts { + ReplacedScripts(ReplacedScripts&&) = default; + std::string script; + size_t origPos; + size_t origLen; + }; + std::vector replacedScripts; + replacedScripts.reserve(optionIt->second.size()); + //In order to continue compatibility with legacy implementation we must collect import scripts in forward direction + //but then replace in reverse direction (in order to keep consistency of positions) + for (const auto & option: optionIt->second) { + const char *importScriptCode = findImportScript(option.value); + std::string importScriptCodeStr; + if (m_importedScriptChecksums.addScript(importScriptCode)) { + // Script has not been imported yet + // If this imported script contains %import statements + // they will be resolved in the next recursion. + ctpg_parser::options_map_t newOptions; + try { + ExecutionGraph::OptionsLineParser::CTPG::parseOptions(importScriptCode, newOptions); + } catch(const ExecutionGraph::OptionParserException & ex) { + Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1630"); + } + importScriptCodeStr.assign(importScriptCode); + importScript(importScriptCodeStr, newOptions); + } + ReplacedScripts replacedScript = {.script = std::move(importScriptCodeStr), .origPos = option.idx_in_source, .origLen = option.size }; + replacedScripts.push_back(std::move(replacedScript)); + } + for (auto optionIt = replacedScripts.rbegin(); optionIt != replacedScripts.rend(); optionIt++) { + scriptCode.replace(optionIt->origPos, optionIt->origLen, optionIt->script); + } + } +} + +const char* ScriptImporter::findImportScript(const std::string & scriptKey) { + if (!m_metaData) { + m_metaData.reset(m_swigFactory.makeSwigMetadata()); + if (!m_metaData) + throw std::runtime_error("F-UDF-CL-SL-JAVA-1631: Failure while importing scripts"); + } + const char *importScriptCode = m_metaData->moduleContent(scriptKey.c_str()); + const char *exception = m_metaData->checkException(); + if (exception) + throw std::runtime_error("F-UDF-CL-SL-JAVA-1632: " + std::string(exception)); + return importScriptCode; +} + +} //namespace CTPG + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h new file mode 100644 index 00000000..cf3c7c9c --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h @@ -0,0 +1,42 @@ +#ifndef SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H +#define SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H 1 + +#include "base/javacontainer/script_options/checksum.h" +#include "base/javacontainer/script_options/keywords.h" +#include "base/exaudflib/swig/swig_meta_data.h" +#include "base/script_options_parser/ctpg/script_option_lines_ctpg.h" +#include + + +namespace SWIGVMContainers { + + struct SwigFactory; + +namespace JavaScriptOptions { + +namespace CTPG { + +class ScriptImporter { + + public: + ScriptImporter(SwigFactory & swigFactory, Keywords & keywords); + + void importScript(std::string & scriptCode, + ExecutionGraph::OptionsLineParser::CTPG::options_map_t & options); + + private: + const char* findImportScript(const std::string & scriptKey); + private: + Checksum m_importedScriptChecksums; + SwigFactory & m_swigFactory; + std::unique_ptr m_metaData; + Keywords & m_keywords; +}; + +} //namespace CTPG + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + +#endif //SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc index 4462e73b..d46b355d 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc @@ -17,7 +17,7 @@ ScriptOptionLinesParserLegacy::ScriptOptionLinesParserLegacy() : m_whitespace(" \t\f\v") , m_lineend(";") , m_scriptCode() -, m_keywords() {} +, m_keywords(true) {} void ScriptOptionLinesParserLegacy::prepareScriptCode(const std::string & scriptCode) { m_scriptCode = scriptCode; @@ -142,7 +142,7 @@ void ScriptOptionLinesParserLegacy::parseForScriptClass(std::function callback) { try { - parseForMultipleOptions(m_keywords.jvmOptionKeyword(), + parseForMultipleOptions(m_keywords.jvmKeyword(), [&](const std::string& value, size_t pos){callback(value);}); } catch(const ExecutionGraph::OptionParserException& ex) { Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1612"); @@ -162,7 +162,7 @@ std::string && ScriptOptionLinesParserLegacy::getScriptCode() { return std::move(m_scriptCode); } -void ScriptOptionLinesParserLegacy::parseForSingleOption(const std::string keyword, +void ScriptOptionLinesParserLegacy::parseForSingleOption(const std::string & keyword, std::function callback) { size_t pos; try { @@ -176,7 +176,7 @@ void ScriptOptionLinesParserLegacy::parseForSingleOption(const std::string keywo } } -void ScriptOptionLinesParserLegacy::parseForMultipleOptions(const std::string keyword, +void ScriptOptionLinesParserLegacy::parseForMultipleOptions(const std::string & keyword, std::function callback) { size_t pos; while (true) { diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.h b/exaudfclient/base/javacontainer/script_options/parser_legacy.h index 655f6e0f..4005732e 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.h +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.h @@ -28,9 +28,9 @@ class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { std::string && getScriptCode() override; private: - void parseForSingleOption(const std::string key, + void parseForSingleOption(const std::string& key, std::function callback); - void parseForMultipleOptions(const std::string key, + void parseForMultipleOptions(const std::string& key, std::function callback); private: diff --git a/exaudfclient/base/javacontainer/test/BUILD b/exaudfclient/base/javacontainer/test/BUILD index 32874b9f..6cba7658 100644 --- a/exaudfclient/base/javacontainer/test/BUILD +++ b/exaudfclient/base/javacontainer/test/BUILD @@ -10,14 +10,26 @@ java_test( deps = [":ExaStackTraceCleaner"] ) +JAVACONTAINER_TEST_SRCS = ["cpp/javacontainer_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", + "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"] cc_test( - name = "javacontainer-test", - srcs = ["cpp/javacontainer_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", - "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"], + name = "javacontainer-test-legacy-parser", + srcs = JAVACONTAINER_TEST_SRCS, deps = [ "//base/javacontainer:javacontainer", "@googletest//:gtest_main", ], data = ["test.jar", "other_test.jar"] -) \ No newline at end of file +) + +cc_test( + name = "javacontainer-test-ctpg-parser", + srcs = JAVACONTAINER_TEST_SRCS, + deps = [ + "//base/javacontainer:javacontainer", + "@googletest//:gtest_main", + ], + defines = ["USE_CTPG_PARSER"], + data = ["test.jar", "other_test.jar"] +) diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc index ff772f68..4b98c4f5 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc @@ -369,7 +369,10 @@ TEST(JavaContainer, import_script_script_class_option_ignored) { EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = "package com.exasol;\r\n" - "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" +#ifndef USE_CTPG_PARSER //The parsers behave differently: The legacy parser incorrectly keeps imported scriptclass options + "%scriptclass com.exasol.udf_profiling.UdfProfiler;" +#endif + "\n" "class OtherClass {\n" "static void doSomething() {\n\n" " }\n" diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc index 9ac82a59..f4ec7f39 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc @@ -20,7 +20,11 @@ JavaVMTest::JavaVMTest(std::string scriptCode, SwigFactoryTestImpl & swigFactory void JavaVMTest::run(std::string scriptCode, SwigFactoryTestImpl & swigFactory) { char* script_code = ::strdup(scriptCode.c_str()); SWIGVMContainers::SWIGVM_params->script_code = script_code; - SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, swigFactory); + bool useCTPGParser = false; +#ifdef USE_CTPG_PARSER + useCTPGParser = true; +#endif + SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, swigFactory, useCTPGParser); javaVMInternalStatus.m_exaJavaPath = javaVMImpl.m_exaJavaPath; javaVMInternalStatus.m_localClasspath = javaVMImpl.m_localClasspath; javaVMInternalStatus.m_scriptCode = javaVMImpl.m_scriptCode; diff --git a/exaudfclient/base/javacontainer/test/cpp/swig_factory_test.cc b/exaudfclient/base/javacontainer/test/cpp/swig_factory_test.cc index 7dbb6ee7..aa46fa7d 100644 --- a/exaudfclient/base/javacontainer/test/cpp/swig_factory_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/swig_factory_test.cc @@ -14,8 +14,8 @@ class SWIGMetadataTest : public SWIGVMContainers::SWIGMetadataIf { public: SWIGMetadataTest(const std::map & moduleContent, const std::string & exceptionMsg) - : m_moduleContent(moduleContent) - , m_exceptionMsg(exceptionMsg) {} + : m_exceptionMsg(exceptionMsg) + , m_moduleContent(moduleContent) {} virtual const char* databaseName() { throw NotImplemented("databaseName"); return nullptr;} virtual const char* databaseVersion() { throw NotImplemented("databaseVersion"); return nullptr;} virtual const char* scriptName() { throw NotImplemented("scriptName"); return nullptr;} diff --git a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.h b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.h index 21c29034..8f893a26 100644 --- a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.h +++ b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.h @@ -15,8 +15,6 @@ namespace OptionsLineParser namespace CTPG { -class ParserResult; - struct ScriptOption { std::string value; size_t idx_in_source; diff --git a/exaudfclient/exaudfclient.cc b/exaudfclient/exaudfclient.cc index 00b644f6..c1952d3e 100644 --- a/exaudfclient/exaudfclient.cc +++ b/exaudfclient/exaudfclient.cc @@ -163,7 +163,7 @@ int main(int argc, char **argv) { } else if (strcmp(argv[2], "lang=java")==0) { #ifdef ENABLE_JAVA_VM - vmMaker = [&](){return new SWIGVMContainers::JavaVMach(false, swigFactory);}; + vmMaker = [&](){return new SWIGVMContainers::JavaVMach(false, swigFactory, false);}; #else throw SWIGVM::exception("this exaudfclient has been compilied without Java support"); #endif From 780ff35f2b05fff37f2aaf9b080dd3b38d0f8429 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:08:01 -0300 Subject: [PATCH 02/43] Added CTPG specific tests --- exaudfclient/base/javacontainer/test/BUILD | 2 +- .../test/cpp/javacontainer_ctpg_test.cc | 108 ++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc diff --git a/exaudfclient/base/javacontainer/test/BUILD b/exaudfclient/base/javacontainer/test/BUILD index 6cba7658..66701152 100644 --- a/exaudfclient/base/javacontainer/test/BUILD +++ b/exaudfclient/base/javacontainer/test/BUILD @@ -25,7 +25,7 @@ cc_test( cc_test( name = "javacontainer-test-ctpg-parser", - srcs = JAVACONTAINER_TEST_SRCS, + srcs = JAVACONTAINER_TEST_SRCS + ["cpp/javacontainer_ctpg_test.cc"], deps = [ "//base/javacontainer:javacontainer", "@googletest//:gtest_main", diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc new file mode 100644 index 00000000..a3db7c2d --- /dev/null +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc @@ -0,0 +1,108 @@ + +#include "include/gtest/gtest.h" +#include "gmock/gmock.h" +#include "base/javacontainer/test/cpp/javavm_test.h" +#include "base/javacontainer/test/cpp/swig_factory_test.h" +#include + +class JavaContainerEscapeSequenceTest : public ::testing::TestWithParam> {}; + +TEST_P(JavaContainerEscapeSequenceTest, quoted_jvm_option) { +const std::pair option_value = GetParam(); + const std::string script_code = + "%jvmoption " + option_value.first + ";\n\n" + "class JVMOPTION_TEST_WITH_SPACE {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" + " ctx.emit(\"Success!\");\n" + " }\n" + "}\n"; + JavaVMTest vm(script_code); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); + const std::string expected_script_code = + "package com.exasol;\r\n\n\n" + "class JVMOPTION_TEST_WITH_SPACE {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" + "\tctx.emit(\"Success!\");\n" + " }\n}\n"; + EXPECT_EQ(expected_script_code, vm.getJavaVMInternalStatus().m_scriptCode); + EXPECT_EQ("/exaudf/base/javacontainer/exaudf_deploy.jar", vm.getJavaVMInternalStatus().m_exaJarPath); + EXPECT_EQ("/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar", vm.getJavaVMInternalStatus().m_classpath); + EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); + + const std::vector expectedJVMOptions = { option_value.second, "-Xms128m", "-Xmx128m", "-Xss512k", + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Djava.class.path=/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar", + "-XX:+UseSerialGC" }; + EXPECT_EQ(expectedJVMOptions, vm.getJavaVMInternalStatus().m_jvmOptions); +} + +const std::vector> escape_sequences = + { + std::make_pair("-Dhttp.agent=ABC\\nDEF", "-Dhttp.agent=ABC\nDEF"), + std::make_pair("-Dhttp.agent=ABC\\rDEF", "-Dhttp.agent=ABC\rDEF"), + std::make_pair("-Dhttp.agent=ABC\\;DEF", "-Dhttp.agent=ABC;DEF"), + std::make_pair("-Dhttp.agent=ABC\\aDEF", "-Dhttp.agent=ABC\\aDEF"), //any other escape sequence must stay as is + std::make_pair("\\n-Dhttp.agent=ABCDEF", "\n-Dhttp.agent=ABCDEF"), + std::make_pair("\\r-Dhttp.agent=ABCDEF", "\r-Dhttp.agent=ABCDEF"), + std::make_pair("\\;-Dhttp.agent=ABCDEF", ";-Dhttp.agent=ABCDEF"), + std::make_pair("-Dhttp.agent=ABCDEF\\n", "-Dhttp.agent=ABCDEF\n"), + std::make_pair("-Dhttp.agent=ABCDEF\\r", "-Dhttp.agent=ABCDEF\r"), + std::make_pair("-Dhttp.agent=ABCDEF\\;", "-Dhttp.agent=ABCDEF;"), + std::make_pair("\\ -Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), + std::make_pair("\\t-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), + std::make_pair("\\f-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), + std::make_pair("\\v-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF") + }; + +INSTANTIATE_TEST_SUITE_P( + JavaContainer, + JavaContainerEscapeSequenceTest, + ::testing::ValuesIn(escape_sequences) +); + +TEST(JavaContainer, import_script_with_escaped_options) { + const std::string script_code = + "%import other_script;\n\n" + "%jvmoption -Dsomeoption=\"ABC\";\n\n" + "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" + "%jar base/javacontainer/test/test.jar;" + "class JVMOPTION_TEST {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" + " ctx.emit(\"Success!\");\n" + " }\n" + "}\n"; + SwigFactoryTestImpl swigFactory; + + const std::string other_script_code = + "%jvmoption -Dsomeotheroption=\"DE\\nF\";\n\n" + "%jar base/javacontainer/test/other_test.jar;" + "class OtherClass {\n" + "static void doSomething() {\n\n" + " }\n" + "}\n"; + swigFactory.addModule("other_script", other_script_code); + JavaVMTest vm(script_code, swigFactory); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); + const std::string expected_script_code = + "package com.exasol;\r\n\n\n" + "class OtherClass {\n" + "static void doSomething() {\n\n" + " }\n" + "}\n\n\n\n\n\n" + "class JVMOPTION_TEST {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" + "\tctx.emit(\"Success!\");\n" + " }\n}\n"; + EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/other_test.jar:base/javacontainer/test/test.jar"); + EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); + const std::vector expectedJVMOptions = { "-Dexasol.scriptclass=com.exasol.udf_profiling.UdfProfiler", + "-Dsomeotheroption=\"DE\nF\"", "-Dsomeoption=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Djava.class.path=/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/other_test.jar:base/javacontainer/test/test.jar", + "-XX:+UseSerialGC" }; + EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); +} From 6739678eb8b44b1b52b286851bc43b28785d1a41 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 07:18:09 -0300 Subject: [PATCH 03/43] Findings from review --- exaudfclient/base/javacontainer/javacontainer_impl.cc | 2 +- exaudfclient/base/javacontainer/script_options/extractor.cc | 2 +- .../base/javacontainer/script_options/parser_ctpg.cc | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc index 76ad8165..1453af4a 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.cc +++ b/exaudfclient/base/javacontainer/javacontainer_impl.cc @@ -61,7 +61,7 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, boo void JavaVMImpl::parseScriptOptions(JavaScriptOptions::ScriptOptionsParser & scriptOptionsParser, SwigFactory& swigFactory) { JavaScriptOptions::Extractor extractor(scriptOptionsParser, swigFactory); - DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used + DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); DBG_FUNC_CALL(cerr,setClasspath()); diff --git a/exaudfclient/base/javacontainer/script_options/extractor.cc b/exaudfclient/base/javacontainer/script_options/extractor.cc index ed33388b..bc7934e4 100644 --- a/exaudfclient/base/javacontainer/script_options/extractor.cc +++ b/exaudfclient/base/javacontainer/script_options/extractor.cc @@ -18,7 +18,7 @@ Extractor::Extractor(ScriptOptionsParser & parser, void Extractor::extract(std::string & scriptCode) { m_parser.prepareScriptCode(scriptCode); EXTR_DBG_FUNC_CALL(m_parser.parseForScriptClass( [&](const std::string& value){ - EXTR_DBG_FUNC_CALL(m_converter.convertScriptClassName(value)); + EXTR_DBG_FUNC_CALL(m_converter.convertScriptClassName(value)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used })); EXTR_DBG_FUNC_CALL(m_parser.extractImportScripts(m_swigFactory)); EXTR_DBG_FUNC_CALL(m_parser.parseForJvmOptions( [&](const std::string& value){ diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc index f4ef7983..a545d43c 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc @@ -96,7 +96,9 @@ std::string && ScriptOptionLinesParserCTPG::getScriptCode() { void ScriptOptionLinesParserCTPG::parse() { if (m_needParsing) { if(!m_foundOptions.empty()) { - throw std::logic_error("F-UDF-CL-SL-JAVA-1620 Internal error. Parser result is not empty."); + throw std::logic_error( + "F-UDF-CL-SL-JAVA-1620 Internal error. Parser result is not empty. " + "Please open a bug ticket at https://github.com/exasol/script-languages-release/issues/new."); } try { ExecutionGraph::OptionsLineParser::CTPG::parseOptions(m_scriptCode, m_foundOptions); @@ -126,7 +128,7 @@ void ScriptOptionLinesParserCTPG::parseForSingleOption(const std::string key, st if (optionIt != m_foundOptions.end()) { if (optionIt->second.size() != 1) { std::stringstream ss; - ss << "F-UDF-CL-SL-JAVA-1628 found " << optionIt->second.size() << key << " options" << std::endl; + ss << "F-UDF-CL-SL-JAVA-1628 found " << optionIt->second.size() << " instances for script option key '" << key << "' but expected at most one." << std::endl; throw std::invalid_argument(ss.str()); } callback(optionIt->second[0].value); From 3c66a0bdaa2fa0cf3b28b09748e22ef28f10b426 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 13:13:07 -0300 Subject: [PATCH 04/43] Limit max recursion for script importer and added a test. --- exaudfclient/base/javacontainer/BUILD | 2 +- .../base/javacontainer/script_options/BUILD | 2 +- .../parser_ctpg_script_importer.cc | 17 ++- .../parser_ctpg_script_importer.h | 10 +- .../javacontainer/script_options/test/BUILD | 10 ++ .../test/ctpg_script_importer_test.cc | 57 ++++++++++ .../script_options/test/swig_factory_test.cc | 105 ++++++++++++++++++ .../script_options/test/swig_factory_test.h | 20 ++++ 8 files changed, 215 insertions(+), 8 deletions(-) create mode 100644 exaudfclient/base/javacontainer/script_options/test/BUILD create mode 100644 exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc create mode 100644 exaudfclient/base/javacontainer/script_options/test/swig_factory_test.cc create mode 100644 exaudfclient/base/javacontainer/script_options/test/swig_factory_test.h diff --git a/exaudfclient/base/javacontainer/BUILD b/exaudfclient/base/javacontainer/BUILD index fa42cd31..a544a342 100644 --- a/exaudfclient/base/javacontainer/BUILD +++ b/exaudfclient/base/javacontainer/BUILD @@ -108,7 +108,7 @@ cc_library( name = "javacontainer", srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", ":dummy"], hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], - deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//base/exaudflib:header", + deps = ["@java//:java", ":exascript_java", "//base/exaudflib:header", "//base/utils:utils","//base/javacontainer/script_options:java_script_option_lines", "//base/swig_factory:swig_factory_if"], # copts= ["-O0","-fno-lto"], diff --git a/exaudfclient/base/javacontainer/script_options/BUILD b/exaudfclient/base/javacontainer/script_options/BUILD index 80af6e98..6874aa4a 100644 --- a/exaudfclient/base/javacontainer/script_options/BUILD +++ b/exaudfclient/base/javacontainer/script_options/BUILD @@ -7,7 +7,7 @@ cc_library( srcs = [":parser.h", ":converter.h", ":converter.cc", ":parser_legacy.cc", ":extractor.cc", ":keywords.h", ":keywords.cc", ":checksum.h", ":checksum.cc", ":parser_ctpg.cc", ":parser_ctpg_script_importer.cc", ":parser_ctpg_script_importer.h"], - deps = ["//base/script_options_parser/legacy:script_option_lines_parser_legacy", + deps = ["@ssl//:ssl", "//base/script_options_parser/legacy:script_option_lines_parser_legacy", "//base/script_options_parser/ctpg:script_option_lines_parser_ctpg", "//base/utils:utils", "//base/exaudflib:header", "//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory_if", "//base/script_options_parser:exception"], diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc index a3dd297b..a86690b6 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc @@ -22,8 +22,19 @@ ScriptImporter::ScriptImporter(SwigFactory & swigFactory, Keywords & keywords) , m_metaData() , m_keywords(keywords) {} -void ScriptImporter::importScript(std::string & scriptCode, ctpg_parser::options_map_t & options) { +void ScriptImporter::importScript(std::string & scriptCode, + ctpg_parser::options_map_t & options) { + importScript(scriptCode, options, 0); +} + +void ScriptImporter::importScript(std::string & scriptCode, + ctpg_parser::options_map_t & options, + const size_t recursionDepth) { const auto optionIt = options.find(std::string(m_keywords.importKeyword())); + + if (recursionDepth >= cMaxRecursionDepth) { + throw std::runtime_error("F-UDF-CL-SL-JAVA-1633: Maximal recursion depth for importing scripts reached."); + } if (optionIt != options.end()) { m_importedScriptChecksums.addScript(scriptCode.c_str()); //Sort options from first in script to last in script @@ -45,7 +56,7 @@ void ScriptImporter::importScript(std::string & scriptCode, ctpg_parser::options for (const auto & option: optionIt->second) { const char *importScriptCode = findImportScript(option.value); std::string importScriptCodeStr; - if (m_importedScriptChecksums.addScript(importScriptCode)) { + if (m_importedScriptChecksums.addScript(importScriptCode) ) { // Script has not been imported yet // If this imported script contains %import statements // they will be resolved in the next recursion. @@ -56,7 +67,7 @@ void ScriptImporter::importScript(std::string & scriptCode, ctpg_parser::options Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1630"); } importScriptCodeStr.assign(importScriptCode); - importScript(importScriptCodeStr, newOptions); + importScript(importScriptCodeStr, newOptions, recursionDepth + 1); } ReplacedScripts replacedScript = {.script = std::move(importScriptCodeStr), .origPos = option.idx_in_source, .origLen = option.size }; replacedScripts.push_back(std::move(replacedScript)); diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h index cf3c7c9c..f0492e34 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.h @@ -21,16 +21,20 @@ class ScriptImporter { public: ScriptImporter(SwigFactory & swigFactory, Keywords & keywords); - void importScript(std::string & scriptCode, - ExecutionGraph::OptionsLineParser::CTPG::options_map_t & options); + void importScript(std::string & scriptCode, ExecutionGraph::OptionsLineParser::CTPG::options_map_t & options); private: - const char* findImportScript(const std::string & scriptKey); + void importScript(std::string & scriptCode, + ExecutionGraph::OptionsLineParser::CTPG::options_map_t & options, + const size_t recursionDepth); + const char* findImportScript(const std::string & scriptKey); private: Checksum m_importedScriptChecksums; SwigFactory & m_swigFactory; std::unique_ptr m_metaData; Keywords & m_keywords; + //The empirical maximal value for recursion depth is ~26000. So we choose 20000 to have a certain buffer. + const size_t cMaxRecursionDepth = 20000; }; } //namespace CTPG diff --git a/exaudfclient/base/javacontainer/script_options/test/BUILD b/exaudfclient/base/javacontainer/script_options/test/BUILD new file mode 100644 index 00000000..5c7f0cf5 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/test/BUILD @@ -0,0 +1,10 @@ + +cc_test( + name = "java-script-options-tests", + srcs = ["ctpg_script_importer_test.cc", "swig_factory_test.cc", "swig_factory_test.h"], + deps = [ + "//base/javacontainer/script_options:java_script_option_lines", + "@googletest//:gtest_main", + ], +) + diff --git a/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc b/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc new file mode 100644 index 00000000..3494377d --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc @@ -0,0 +1,57 @@ + +#include "include/gtest/gtest.h" +#include "gmock/gmock.h" +#include "base/javacontainer/script_options/parser_ctpg.h" + +#include "base/javacontainer/script_options/test/swig_factory_test.h" +#include + +using namespace SWIGVMContainers::JavaScriptOptions; + + +static const char* sc_scriptName = "script"; + + +void checkIndex(size_t currentIdx, const char* scriptKey) { + std::stringstream ss; + ss << sc_scriptName << currentIdx; + if (ss.str() != scriptKey) { + throw std::logic_error(std::string("Script Key does not match: '") + ss.str() + " != '" + scriptKey + "'"); + } +} + +const char* buildNewScriptCode(size_t currentIdx) { + std::stringstream ss; + ss << "%import " << sc_scriptName << currentIdx << ";something"; + static std::string ret; + ret = ss.str(); + return ret.c_str(); +} + +TEST(ScriptImporterTest, max_recursion_depth) { + + size_t currentIdx = 0; + SwigFactoryTestImpl swigFactoryTest([&](const char* scriptKey) { + checkIndex(currentIdx, scriptKey); + return buildNewScriptCode(++currentIdx); + }); + ScriptOptionLinesParserCTPG parser; + + const std::string code = buildNewScriptCode(currentIdx); + parser.prepareScriptCode(code); + EXPECT_THROW({ + try + { + parser.extractImportScripts(swigFactoryTest); + } + catch( const std::runtime_error& e ) + { + //We need to deceive "find_duplicate_error_codes.sh" here + const std::string expectedError = + std::string("F-UDF-CL-SL-JAVA-") + "1633: Maximal recursion depth for importing scripts reached."; + EXPECT_STREQ( expectedError.c_str(), e.what()); + throw; + } + }, std::runtime_error ); +} + diff --git a/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.cc b/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.cc new file mode 100644 index 00000000..8744d421 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.cc @@ -0,0 +1,105 @@ +#include "base/javacontainer/script_options/test/swig_factory_test.h" +#include "base/exaudflib/swig/swig_meta_data.h" + +#include + +class NotImplemented : public std::logic_error +{ +public: + NotImplemented() : std::logic_error("Function not yet implemented") { }; + NotImplemented(const std::string funcName) : std::logic_error("Function " + funcName + " not yet implemented") { }; +}; + +class SWIGMetadataTest : public SWIGVMContainers::SWIGMetadataIf { + +public: + SWIGMetadataTest(std::function callback): m_callback(callback) {} + virtual const char* databaseName() { throw NotImplemented("databaseName"); return nullptr;} + virtual const char* databaseVersion() { throw NotImplemented("databaseVersion"); return nullptr;} + virtual const char* scriptName() { throw NotImplemented("scriptName"); return nullptr;} + virtual const char* scriptSchema() { throw NotImplemented("scriptSchema"); return nullptr;} + virtual const char* currentUser() { throw NotImplemented("currentUser"); return nullptr;} + virtual const char* scopeUser() { throw NotImplemented("scopeUser"); return nullptr;} + virtual const char* currentSchema() { throw NotImplemented("currentSchema"); return nullptr;} + virtual const char* scriptCode() { throw NotImplemented("scriptCode"); return nullptr;} + virtual const unsigned long long sessionID() { throw NotImplemented("sessionID"); return 0;} + virtual const char *sessionID_S() { throw NotImplemented("sessionID_S"); return nullptr;} + virtual const unsigned long statementID() { throw NotImplemented("statementID"); return 0;} + virtual const unsigned int nodeCount() { throw NotImplemented("nodeCount"); return 0;} + virtual const unsigned int nodeID() { throw NotImplemented("nodeID"); return 0;} + virtual const unsigned long long vmID() { throw NotImplemented("vmID"); return 0;} + virtual const unsigned long long memoryLimit() { throw NotImplemented("memoryLimit"); return 0;} + virtual const SWIGVMContainers::VMTYPE vmType() { + throw NotImplemented("vmType"); + return SWIGVMContainers::VM_UNSUPPORTED; + } + virtual const char *vmID_S() { throw NotImplemented("vmID_S"); return nullptr;} + virtual const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name) { + throw NotImplemented("connectionInformation"); return nullptr; + } + virtual const char* moduleContent(const char* name) { + return m_callback(name); + } + virtual const unsigned int inputColumnCount() { throw NotImplemented("inputColumnCount"); return 0;} + virtual const char *inputColumnName(unsigned int col) { + throw NotImplemented("inputColumnName"); + return nullptr; + } + virtual const SWIGVMContainers::SWIGVM_datatype_e inputColumnType(unsigned int col) { + throw NotImplemented("inputColumnType"); + return SWIGVMContainers::UNSUPPORTED; + } + virtual const char *inputColumnTypeName(unsigned int col) { + throw NotImplemented("inputColumnTypeName"); return nullptr; + } + virtual const unsigned int inputColumnSize(unsigned int col) { + throw NotImplemented("inputColumnSize"); return 0; + } + virtual const unsigned int inputColumnPrecision(unsigned int col) { + throw NotImplemented("inputColumnPrecision"); return 0; + } + virtual const unsigned int inputColumnScale(unsigned int col) { + throw NotImplemented("inputColumnScale"); return 0; + } + virtual const SWIGVMContainers::SWIGVM_itertype_e inputType() { + throw NotImplemented("inputType"); + return SWIGVMContainers::EXACTLY_ONCE; + } + virtual const unsigned int outputColumnCount() { throw NotImplemented("outputColumnCount"); return 0;} + virtual const char *outputColumnName(unsigned int col) { throw NotImplemented("outputColumnName"); return nullptr;} + virtual const SWIGVMContainers::SWIGVM_datatype_e outputColumnType(unsigned int col) { + throw NotImplemented("outputColumnType"); + return SWIGVMContainers::UNSUPPORTED; + } + virtual const char *outputColumnTypeName(unsigned int col) { + throw NotImplemented("outputColumnTypeName"); return nullptr; + } + virtual const unsigned int outputColumnSize(unsigned int col) { + throw NotImplemented("outputColumnSize"); return 0; + } + virtual const unsigned int outputColumnPrecision(unsigned int col) { + throw NotImplemented("outputColumnPrecision"); return 0; + } + virtual const unsigned int outputColumnScale(unsigned int col) { + throw NotImplemented("outputColumnScale"); return 0; + } + virtual const SWIGVMContainers::SWIGVM_itertype_e outputType() { + throw NotImplemented("outputType"); + return SWIGVMContainers::EXACTLY_ONCE; + } + virtual const bool isEmittedColumn(unsigned int col) { throw NotImplemented("isEmittedColumn"); return false;} + virtual const char* checkException() { return nullptr;} + virtual const char* pluginLanguageName() { throw NotImplemented("pluginLanguageName"); return nullptr;} + virtual const char* pluginURI() { throw NotImplemented("pluginURI"); return nullptr;} + virtual const char* outputAddress() { throw NotImplemented("outputAddress"); return nullptr;} + +private: + std::function m_callback; +}; + +SwigFactoryTestImpl::SwigFactoryTestImpl(std::function callback) +: m_callback(callback) {} + +SWIGVMContainers::SWIGMetadataIf* SwigFactoryTestImpl::makeSwigMetadata() { + return new SWIGMetadataTest(m_callback); +} diff --git a/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.h b/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.h new file mode 100644 index 00000000..f5fa9555 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/test/swig_factory_test.h @@ -0,0 +1,20 @@ +#ifndef SWIG_FACTORY_TEST_H +#define SWIG_FACTORY_TEST_H 1 + +#include +#include +#include "base/swig_factory/swig_factory.h" +#include + +struct SwigFactoryTestImpl : public SWIGVMContainers::SwigFactory { + + SwigFactoryTestImpl(std::function callback); + + virtual SWIGVMContainers::SWIGMetadataIf* makeSwigMetadata() override; + +private: + std::function m_callback; +}; + + +#endif //namespace SWIG_FACTORY_TEST_H \ No newline at end of file From ba28b85dccc2a61b8cec56f4d7ac3171eceae06e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:52:32 -0300 Subject: [PATCH 05/43] Added valgrind/asan tests --- .github/workflows/check_bazel_tests.yml | 46 ++++++++++++++++++------- exaudfclient/.bazelrc | 8 +++++ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index def9fd7a..1ae180b6 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -6,7 +6,6 @@ on: - master pull_request: - jobs: build: runs-on: ubuntu-latest @@ -15,7 +14,37 @@ jobs: USE_BAZEL_VERSION: 7.2.1 steps: - uses: actions/checkout@v4 - + - matrix: + test: + - "//base/javacontainer/test:ExaStackTraceCleanerTest" + - "//base/javacontainer/test:javacontainer-test-legacy-parser" + - "//base/javacontainer/test:javacontainer-test-ctpg-parser" + - "//base/javacontainer/script_options/..." + - "//base/exaudflib/test/..." + - "//base/script_options_parser/ctpg/..." + - "//base/script_options_parser/legacy/..." + additional_args: [""] + include: + - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" + additional_args: ["--run_under='valgrind'"] + - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" + additional_args: ["--run_under='valgrind'"] + - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" + additional_args: ["--run_under='valgrind'"] + - test: "//base/script_options_parser/ctpg/..." + additional_args: ["--run_under='valgrind'"] + - test: "//base/script_options_parser/legacy/..." + additional_args: ["--run_under='valgrind'"] + - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" + additional_args: ["--config asan"] + - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" + additional_args: ["--config asan"] + - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" + additional_args: ["--config asan"] + - test: "//base/script_options_parser/ctpg/..." + additional_args: ["--config asan"] + - test: "//base/script_options_parser/legacy/..." + additional_args: ["--config asan"] - name: Search for duplicated error codes run: bash find_duplicate_error_codes.sh @@ -28,16 +57,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y openjdk-11-jdk libzmq3-dev - - name: Java Tests - run: | - bazel test //base/javacontainer/test/... - working-directory: ./exaudfclient/ - - name: ExaudfLib Tests + - name: Run tests run: | - bazel test //base/exaudflib/test/... + bazel test ${{ matrix.additional_args }} ${{ matrix.test }} working-directory: ./exaudfclient/ - - name: Script Options Parser Tests - run: | - bazel test //base/script_options_parser/... - working-directory: ./exaudfclient/ - diff --git a/exaudfclient/.bazelrc b/exaudfclient/.bazelrc index 248128c7..6af8be63 100644 --- a/exaudfclient/.bazelrc +++ b/exaudfclient/.bazelrc @@ -13,3 +13,11 @@ build:optimize --copt="-g0" --copt="-DNDEBUG" --copt=-fstack-protector-strong -- build:no-tty --curses=no --color=no build:debug-build --sandbox_debug --config=verbose build:no-symlinks --symlink_prefix=/ +build:asan --strip=never +build:asan --copt -fsanitize=address +build:asan --copt -DADDRESS_SANITIZER +build:asan --copt -O1 +build:asan --copt -g +build:asan --copt -fno-omit-frame-pointer +build:asan --linkopt -fsanitize=address +build:asan -c dbg \ No newline at end of file From fa1ccb9b49ab9b6308c5c8d311d102ea2172bea8 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:58:05 -0300 Subject: [PATCH 06/43] Fixed GH Workflow --- .github/workflows/check_bazel_tests.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 1ae180b6..ccc16464 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -9,12 +9,8 @@ on: jobs: build: runs-on: ubuntu-latest - - env: - USE_BAZEL_VERSION: 7.2.1 - steps: - - uses: actions/checkout@v4 - - matrix: + strategy: + matrix: test: - "//base/javacontainer/test:ExaStackTraceCleanerTest" - "//base/javacontainer/test:javacontainer-test-legacy-parser" @@ -46,6 +42,11 @@ jobs: - test: "//base/script_options_parser/legacy/..." additional_args: ["--config asan"] + + env: + USE_BAZEL_VERSION: 7.2.1 + steps: + - uses: actions/checkout@v4 - name: Search for duplicated error codes run: bash find_duplicate_error_codes.sh From dfdda46abc27d0ed668ca71befae6b422094aeac Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:02:35 -0300 Subject: [PATCH 07/43] Fixed GH Workflow --- .github/workflows/check_bazel_tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index ccc16464..21d95a9c 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -22,25 +22,25 @@ jobs: additional_args: [""] include: - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" - additional_args: ["--run_under='valgrind'"] + additional_args: "--run_under='valgrind'" - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" - additional_args: ["--run_under='valgrind'"] + additional_args: "--run_under='valgrind'" - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" - additional_args: ["--run_under='valgrind'"] + additional_args: "--run_under='valgrind'" - test: "//base/script_options_parser/ctpg/..." - additional_args: ["--run_under='valgrind'"] + additional_args: "--run_under='valgrind'" - test: "//base/script_options_parser/legacy/..." - additional_args: ["--run_under='valgrind'"] + additional_args: "--run_under='valgrind'" - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" - additional_args: ["--config asan"] + additional_args: "--config asan" - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" - additional_args: ["--config asan"] + additional_args: "--config asan" - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" - additional_args: ["--config asan"] + additional_args: "--config asan" - test: "//base/script_options_parser/ctpg/..." - additional_args: ["--config asan"] + additional_args: "--config asan" - test: "//base/script_options_parser/legacy/..." - additional_args: ["--config asan"] + additional_args: "--config asan" env: From 79553b27aafef751183f0e2725e2ffe2d0cd30c7 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:08:39 -0300 Subject: [PATCH 08/43] Fixed GH Workflow --- .github/workflows/check_bazel_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 21d95a9c..49542494 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -10,6 +10,7 @@ jobs: build: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: test: - "//base/javacontainer/test:ExaStackTraceCleanerTest" @@ -57,7 +58,7 @@ jobs: - name: Install JDK and ZMQ run: | sudo apt-get update - sudo apt-get install -y openjdk-11-jdk libzmq3-dev + sudo apt-get install -y openjdk-11-jdk libzmq3-dev valgrind - name: Run tests run: | bazel test ${{ matrix.additional_args }} ${{ matrix.test }} From b3c1c7cbc3339534edcca0145abc5b148a94dd6a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:19:21 -0300 Subject: [PATCH 09/43] Fixed GH Workflow --- .github/workflows/check_bazel_tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 49542494..854118b2 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -29,13 +29,11 @@ jobs: - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" additional_args: "--run_under='valgrind'" - test: "//base/script_options_parser/ctpg/..." - additional_args: "--run_under='valgrind'" + additional_args: "--run_under='valgrind' --test_timeout=3600" - test: "//base/script_options_parser/legacy/..." additional_args: "--run_under='valgrind'" - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" additional_args: "--config asan" - - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" - additional_args: "--config asan" - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" additional_args: "--config asan" - test: "//base/script_options_parser/ctpg/..." @@ -63,3 +61,8 @@ jobs: run: | bazel test ${{ matrix.additional_args }} ${{ matrix.test }} working-directory: ./exaudfclient/ + + - uses: actions/upload-artifact@v4 + with: + name: "test_logs_${{ matrix.test }}_${{ matrix.additional_args }}" + path: /home/runner/.cache/bazel/_bazel_runner/**/test.log From 25e21e95b290ecb40dd07a3c0737c4285bacf2db Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:29:09 -0300 Subject: [PATCH 10/43] Fixed GH Workflow --- .github/workflows/check_bazel_tests.yml | 51 +++++++++++++------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 854118b2..7d0de1f9 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -12,34 +12,37 @@ jobs: strategy: fail-fast: false matrix: - test: - - "//base/javacontainer/test:ExaStackTraceCleanerTest" - - "//base/javacontainer/test:javacontainer-test-legacy-parser" - - "//base/javacontainer/test:javacontainer-test-ctpg-parser" - - "//base/javacontainer/script_options/..." - - "//base/exaudflib/test/..." - - "//base/script_options_parser/ctpg/..." - - "//base/script_options_parser/legacy/..." - additional_args: [""] include: + - test: "//base/javacontainer/test:ExaStackTraceCleanerTest" + name: "ExaStackTraceCleanerTest" - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" - additional_args: "--run_under='valgrind'" - - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" - additional_args: "--run_under='valgrind'" - - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" - additional_args: "--run_under='valgrind'" - - test: "//base/script_options_parser/ctpg/..." - additional_args: "--run_under='valgrind' --test_timeout=3600" - - test: "//base/script_options_parser/legacy/..." - additional_args: "--run_under='valgrind'" - - test: "//base/javacontainer/test:javacontainer-test-legacy-parser" - additional_args: "--config asan" + name: "javacontainer-test-legacy-parser" - test: "//base/javacontainer/test:javacontainer-test-ctpg-parser" - additional_args: "--config asan" + name: "javacontainer-test-ctpg-parser" + - test: "//base/javacontainer/script_options/..." + name: "javacontainer-script_options" + - test: "//base/exaudflib/test/..." + name: "exaudflib" - test: "//base/script_options_parser/ctpg/..." - additional_args: "--config asan" + name: "script_options_parser_ctpg" - test: "//base/script_options_parser/legacy/..." - additional_args: "--config asan" + name: "script_options_parser_legacy" + - test: "--run_under='valgrind' //base/javacontainer/test:javacontainer-test-legacy-parser" + name: "javacontainer-test-legacy-parser-with-valgrind" + - test: "--run_under='valgrind' //base/javacontainer/test:javacontainer-test-ctpg-parser" + name: "javacontainer-test-ctpg-parser-with-valgrind" + - test: "--run_under='valgrind' --test_timeout=3600 //base/script_options_parser/ctpg/..." + name: "script_options_parser_ctpg_with_valgrind" + - test: "--run_under='valgrind' //base/script_options_parser/legacy/..." + name: "script_options_parser_legacy_with_valgrind" + - test: "--config=asan //base/javacontainer/test:javacontainer-test-legacy-parser" + name: "javacontainer-test-legacy-parser-with-asan" + - test: "--config=asan //base/javacontainer/test:javacontainer-test-ctpg-parser" + name: "javacontainer-test-ctpg-parser-with-asan" + - test: "--config=asan --test_timeout=3600 //base/script_options_parser/ctpg/..." + name: "script_options_parser_ctpg_with_asan" + - test: "--config=asan //base/script_options_parser/legacy/..." + name: "script_options_parser_legacy_with_asan" env: @@ -64,5 +67,5 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: "test_logs_${{ matrix.test }}_${{ matrix.additional_args }}" + name: "${{ matrix.name }}" path: /home/runner/.cache/bazel/_bazel_runner/**/test.log From 0ef2b94530c0a7ecc7c876fd418cecd76b3c8bc8 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:32:45 -0300 Subject: [PATCH 11/43] Fixed GH Workflow --- .github/workflows/check_bazel_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 7d0de1f9..85a71e17 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -68,4 +68,4 @@ jobs: - uses: actions/upload-artifact@v4 with: name: "${{ matrix.name }}" - path: /home/runner/.cache/bazel/_bazel_runner/**/test.log + path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/base/**/test.log From 9be6303efda37f81ddf414ae2a4541d8138806f3 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:40:16 -0300 Subject: [PATCH 12/43] Fixed GH Workflow --- .github/workflows/check_bazel_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 85a71e17..59898617 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -66,6 +66,7 @@ jobs: working-directory: ./exaudfclient/ - uses: actions/upload-artifact@v4 + if: always() with: name: "${{ matrix.name }}" - path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/base/**/test.log + path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/**/test.log From ee31d02e8bacbd09139c36e7cf0f8b0efbc0c21b Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:49:58 -0300 Subject: [PATCH 13/43] Fixed memory allocation/deallocation issue reported by ASAN in tests --- exaudfclient/base/javacontainer/test/cpp/javavm_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc index f4ec7f39..8056b406 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc @@ -32,6 +32,6 @@ void JavaVMTest::run(std::string scriptCode, SwigFactoryTestImpl & swigFactory) javaVMInternalStatus.m_classpath = javaVMImpl.m_classpath; javaVMInternalStatus.m_jvmOptions = javaVMImpl.m_jvmOptions; javaVMInternalStatus.m_needsCompilation = javaVMImpl.m_needsCompilation; - delete script_code; + ::free(script_code); } From f9663d1ecf58861635b3893b7d8ab4df4a69e90d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:52:07 -0300 Subject: [PATCH 14/43] Upload test logs only in error case --- .github/workflows/check_bazel_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 59898617..795aa3e3 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -66,7 +66,7 @@ jobs: working-directory: ./exaudfclient/ - uses: actions/upload-artifact@v4 - if: always() + if: failure() with: name: "${{ matrix.name }}" path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/**/test.log From c53ce9e41a241d05802483fc5d16e6695dfdda6c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:07:59 -0300 Subject: [PATCH 15/43] Deactivate slow tests under valgrind --- .github/workflows/check_bazel_tests.yml | 8 ++++---- .../ctpg/test/script_option_lines_test.cpp | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 795aa3e3..0fd17715 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -27,13 +27,13 @@ jobs: name: "script_options_parser_ctpg" - test: "//base/script_options_parser/legacy/..." name: "script_options_parser_legacy" - - test: "--run_under='valgrind' //base/javacontainer/test:javacontainer-test-legacy-parser" + - test: "--run_under='valgrind' --copt -DVALGRIND_ACTIVE //base/javacontainer/test:javacontainer-test-legacy-parser" name: "javacontainer-test-legacy-parser-with-valgrind" - - test: "--run_under='valgrind' //base/javacontainer/test:javacontainer-test-ctpg-parser" + - test: "--run_under='valgrind' --copt -DVALGRIND_ACTIVE //base/javacontainer/test:javacontainer-test-ctpg-parser" name: "javacontainer-test-ctpg-parser-with-valgrind" - - test: "--run_under='valgrind' --test_timeout=3600 //base/script_options_parser/ctpg/..." + - test: "--run_under='valgrind' --copt -DVALGRIND_ACTIVE //base/script_options_parser/ctpg/..." name: "script_options_parser_ctpg_with_valgrind" - - test: "--run_under='valgrind' //base/script_options_parser/legacy/..." + - test: "--run_under='valgrind' --copt -DVALGRIND_ACTIVE //base/script_options_parser/legacy/..." name: "script_options_parser_legacy_with_valgrind" - test: "--config=asan //base/javacontainer/test:javacontainer-test-legacy-parser" name: "javacontainer-test-legacy-parser-with-asan" diff --git a/exaudfclient/base/script_options_parser/ctpg/test/script_option_lines_test.cpp b/exaudfclient/base/script_options_parser/ctpg/test/script_option_lines_test.cpp index f7cd3540..76ca72b0 100644 --- a/exaudfclient/base/script_options_parser/ctpg/test/script_option_lines_test.cpp +++ b/exaudfclient/base/script_options_parser/ctpg/test/script_option_lines_test.cpp @@ -17,6 +17,8 @@ inline ScriptOption buildOption(const char* value, size_t idx, size_t len) { return option; } +#ifndef VALGRIND_ACTIVE + class ScriptOptionLinesWhitespaceTest : public ::testing::TestWithParam> {}; TEST_P(ScriptOptionLinesWhitespaceTest, WhitespaceExtractOptionLineTest) { @@ -58,6 +60,8 @@ INSTANTIATE_TEST_SUITE_P( ) ); +#endif //VALGRIND_ACTIVE + TEST(ScriptOptionLinesTest, ignore_anything_other_than_whitepsace) { const std::string code = "abc %option myoption;\n" From 9dba9d25d4c68c2333e07e9590ee1a37a617662c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:10:39 -0300 Subject: [PATCH 16/43] Inject a memory problem....let's see which checks find the problem.... --- .../script_options_parser/ctpg/script_option_lines_ctpg.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc index 7c9229b8..3b23877a 100644 --- a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc +++ b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc @@ -180,6 +180,11 @@ constexpr parser option_parser( ); void parse(std::string&& code, options_type& result) { + + //inject a memory problem + char buffer[100]; + buffer[100] = 0; + std::stringstream error_buffer; auto res = option_parser.parse( parse_options{}.set_skip_whitespace(false), From cd61bc0e4349fb0eb2dcfc6e82d462dd17c2a313 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Sun, 13 Oct 2024 09:45:22 -0300 Subject: [PATCH 17/43] Removed memory corruption --- .../script_options_parser/ctpg/script_option_lines_ctpg.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc index 3b23877a..b11dd209 100644 --- a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc +++ b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc @@ -181,10 +181,6 @@ constexpr parser option_parser( void parse(std::string&& code, options_type& result) { - //inject a memory problem - char buffer[100]; - buffer[100] = 0; - std::stringstream error_buffer; auto res = option_parser.parse( parse_options{}.set_skip_whitespace(false), From a145b9828cc10d5174ffdbc1b9b8383f5bd9218a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Sun, 13 Oct 2024 09:45:46 -0300 Subject: [PATCH 18/43] Added bazel config for valgrind --- exaudfclient/.bazelrc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/exaudfclient/.bazelrc b/exaudfclient/.bazelrc index 6af8be63..fe0d63ac 100644 --- a/exaudfclient/.bazelrc +++ b/exaudfclient/.bazelrc @@ -20,4 +20,9 @@ build:asan --copt -O1 build:asan --copt -g build:asan --copt -fno-omit-frame-pointer build:asan --linkopt -fsanitize=address -build:asan -c dbg \ No newline at end of file +build:asan -c dbg +build:valgrind --copt -O1 +build:valgrind -c dbg +build:valgrind --copt -g +build:valgrind --strip=never +build:valgrind --copt -DVALGRIND_ACTIVE \ No newline at end of file From a01b4fd8313348980a1ff44af8d5c83392ce9067 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Sun, 13 Oct 2024 09:46:09 -0300 Subject: [PATCH 19/43] Removed (unnecessary) timeout for asan test --- .github/workflows/check_bazel_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 0fd17715..49442cad 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -39,7 +39,7 @@ jobs: name: "javacontainer-test-legacy-parser-with-asan" - test: "--config=asan //base/javacontainer/test:javacontainer-test-ctpg-parser" name: "javacontainer-test-ctpg-parser-with-asan" - - test: "--config=asan --test_timeout=3600 //base/script_options_parser/ctpg/..." + - test: "--config=asan //base/script_options_parser/ctpg/..." name: "script_options_parser_ctpg_with_asan" - test: "--config=asan //base/script_options_parser/legacy/..." name: "script_options_parser_legacy_with_asan" From 46073214796ec9274a34c6a7c71d4cc19047da11 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Sun, 13 Oct 2024 11:23:23 -0300 Subject: [PATCH 20/43] Added builder for JavaVM --- exaudfclient/base/javacontainer/BUILD | 3 +- .../base/javacontainer/javacontainer.cc | 6 ++- .../base/javacontainer/javacontainer.h | 12 +++++- .../javacontainer/javacontainer_builder.cc | 32 +++++++++++++++ .../javacontainer/javacontainer_builder.h | 39 +++++++++++++++++++ .../base/javacontainer/javacontainer_impl.cc | 17 ++++---- .../base/javacontainer/javacontainer_impl.h | 10 ++++- .../javacontainer/script_options/parser.h | 1 + .../script_options/parser_ctpg.h | 2 + .../script_options/parser_legacy.h | 2 + .../javacontainer/test/cpp/javavm_test.cc | 13 +++++-- exaudfclient/exaudfclient.cc | 4 +- 12 files changed, 119 insertions(+), 22 deletions(-) create mode 100644 exaudfclient/base/javacontainer/javacontainer_builder.cc create mode 100644 exaudfclient/base/javacontainer/javacontainer_builder.h diff --git a/exaudfclient/base/javacontainer/BUILD b/exaudfclient/base/javacontainer/BUILD index a544a342..5c837f1d 100644 --- a/exaudfclient/base/javacontainer/BUILD +++ b/exaudfclient/base/javacontainer/BUILD @@ -106,7 +106,8 @@ cc_library( cc_library( name = "javacontainer", - srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", ":dummy"], + srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", + ":javacontainer_builder.h", ":javacontainer_builder.cc", ":dummy"], hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], deps = ["@java//:java", ":exascript_java", "//base/exaudflib:header", "//base/utils:utils","//base/javacontainer/script_options:java_script_option_lines", diff --git a/exaudfclient/base/javacontainer/javacontainer.cc b/exaudfclient/base/javacontainer/javacontainer.cc index df571f02..aa655b6a 100644 --- a/exaudfclient/base/javacontainer/javacontainer.cc +++ b/exaudfclient/base/javacontainer/javacontainer.cc @@ -1,12 +1,14 @@ #include "base/javacontainer/javacontainer.h" #include "base/javacontainer/javacontainer_impl.h" +#include "base/javacontainer/script_options/parser.h" using namespace SWIGVMContainers; using namespace std; -JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory, bool useCTPGParser) { +JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory, + std::unique_ptr scriptOptionsParser) { try { - m_impl = new JavaVMImpl(checkOnly, false, swigFactory, useCTPGParser); + m_impl = new JavaVMImpl(checkOnly, false, swigFactory, std::move(scriptOptionsParser)); } catch (std::exception& err) { lock_guard lock(exception_msg_mtx); exception_msg = "F-UDF-CL-SL-JAVA-1000: "+std::string(err.what()); diff --git a/exaudfclient/base/javacontainer/javacontainer.h b/exaudfclient/base/javacontainer/javacontainer.h index 46fd0670..a03aa268 100644 --- a/exaudfclient/base/javacontainer/javacontainer.h +++ b/exaudfclient/base/javacontainer/javacontainer.h @@ -4,6 +4,7 @@ #include "base/exaudflib/vm/swig_vm.h" #include +#include #ifdef ENABLE_JAVA_VM @@ -12,9 +13,18 @@ namespace SWIGVMContainers { class JavaVMImpl; class SwigFactory; +namespace JavaScriptOptions { + +struct ScriptOptionsParser; + +} + class JavaVMach: public SWIGVM { public: - JavaVMach(bool checkOnly, SwigFactory& swigFactory, bool useCTPGParser); + /* + * scriptOptionsParser: JavaVMach takes ownership of ScriptOptionsParser pointer. + */ + JavaVMach(bool checkOnly, SwigFactory& swigFactory, std::unique_ptr scriptOptionsParser); virtual ~JavaVMach() {} virtual void shutdown(); virtual bool run(); diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.cc b/exaudfclient/base/javacontainer/javacontainer_builder.cc new file mode 100644 index 00000000..6b10fedf --- /dev/null +++ b/exaudfclient/base/javacontainer/javacontainer_builder.cc @@ -0,0 +1,32 @@ +#include "base/javacontainer/javacontainer_builder.h" +#include "base/javacontainer/script_options/parser_ctpg.h" +#include "base/javacontainer/script_options/parser_legacy.h" +#include "base/javacontainer/javacontainer.h" + +#ifdef ENABLE_JAVA_VM + +namespace SWIGVMContainers { + +JavaContainerBuilder::JavaContainerBuilder(SwigFactory& swigFactory) +: m_parser() +, m_swigFactory(swigFactory) {} + +JavaContainerBuilder& JavaContainerBuilder::useCtpgParser(const bool useCtpgParser) { + m_parser = std::make_unique(); + return *this; +} + +JavaVMach* JavaContainerBuilder::build() { + if (m_parser) { + return new JavaVMach(false, m_swigFactory, std::move(m_parser)); + } else { + m_parser = std::make_unique(); + return new JavaVMach(false, m_swigFactory, std::move(m_parser)); + } +} + + +} //namespace SWIGVMContainers + + +#endif //ENABLE_JAVA_VM diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.h b/exaudfclient/base/javacontainer/javacontainer_builder.h new file mode 100644 index 00000000..614ce283 --- /dev/null +++ b/exaudfclient/base/javacontainer/javacontainer_builder.h @@ -0,0 +1,39 @@ +#ifndef JAVACONTAINER_BUILDER_H +#define JAVACONTAINER_BUILDER_H + +#include + +#ifdef ENABLE_JAVA_VM + +namespace SWIGVMContainers { + +class JavaVMach; +struct SwigFactory; + +namespace JavaScriptOptions { + +struct ScriptOptionsParser; + +} + +class JavaContainerBuilder { + public: + JavaContainerBuilder(SwigFactory& swigFactory); + + JavaContainerBuilder& useCtpgParser(const bool value); + + JavaVMach* build(); + + private: + std::unique_ptr m_parser; + SwigFactory& m_swigFactory; + +}; + +} //namespace SWIGVMContainers + + +#endif //ENABLE_JAVA_VM + + +#endif //JAVACONTAINER_BUILDER_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc index 1453af4a..2fac4e34 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.cc +++ b/exaudfclient/base/javacontainer/javacontainer_impl.cc @@ -20,7 +20,8 @@ using namespace SWIGVMContainers; using namespace std; -JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, bool useCTPGParser) +JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, + std::unique_ptr scriptOptionsParser) : m_checkOnly(checkOnly) , m_exaJavaPath("") , m_localClasspath("/tmp") // **IMPORTANT**: /tmp needs to be in the classpath, otherwise ExaCompiler crashe with com.exasol.ExaCompilationException: /DATE_STRING.java:3: error: error while writing DATE_STRING: could not create parent directories @@ -35,13 +36,8 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, boo stringstream ss; m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path - if (useCTPGParser) { - JavaScriptOptions::ScriptOptionLinesParserCTPG parser; - parseScriptOptions(parser, swigFactory); - } else { - JavaScriptOptions::ScriptOptionLinesParserLegacy parser; - parseScriptOptions(parser, swigFactory); - } + JavaScriptOptions::ScriptOptionLinesParserCTPG parser; + parseScriptOptions(std::move(scriptOptionsParser), swigFactory); m_needsCompilation = checkNeedsCompilation(); if (m_needsCompilation) { @@ -58,8 +54,9 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, boo } } -void JavaVMImpl::parseScriptOptions(JavaScriptOptions::ScriptOptionsParser & scriptOptionsParser, SwigFactory& swigFactory) { - JavaScriptOptions::Extractor extractor(scriptOptionsParser, swigFactory); +void JavaVMImpl::parseScriptOptions(std::unique_ptr scriptOptionsParser, + SwigFactory& swigFactory) { + JavaScriptOptions::Extractor extractor(*scriptOptionsParser, swigFactory); DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.h b/exaudfclient/base/javacontainer/javacontainer_impl.h index d423f32f..b0ad812a 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.h +++ b/exaudfclient/base/javacontainer/javacontainer_impl.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "base/exaudflib/vm/swig_vm.h" #include @@ -21,7 +22,11 @@ namespace JavaScriptOptions { class JavaVMImpl { public: friend class ::JavaVMTest; - JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, bool useCTPGParser); + /* + * scriptOptionsParser: JavaVMach takes ownership of ScriptOptionsParser pointer. + */ + JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, + std::unique_ptr scriptOptionsParser); ~JavaVMImpl() {} void shutdown(); bool run(); @@ -41,7 +46,8 @@ class JavaVMImpl { void throwException(const std::string& ex); void setJvmOptions(); void addJarToClasspath(const std::string& path); - void parseScriptOptions(JavaScriptOptions::ScriptOptionsParser & scriptOptionsParser, SwigFactory& swigFactory); + void parseScriptOptions(std::unique_ptr scriptOptionsParser, + SwigFactory& swigFactory); bool m_checkOnly; std::string m_exaJavaPath; std::string m_localClasspath; diff --git a/exaudfclient/base/javacontainer/script_options/parser.h b/exaudfclient/base/javacontainer/script_options/parser.h index 68dd634f..e82eac05 100644 --- a/exaudfclient/base/javacontainer/script_options/parser.h +++ b/exaudfclient/base/javacontainer/script_options/parser.h @@ -13,6 +13,7 @@ struct SwigFactory; namespace JavaScriptOptions { struct ScriptOptionsParser { + virtual ~ScriptOptionsParser() {}; /* Passes the script code for parsing to the parser. The parser might modify the script code, because it will remove known options. diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.h b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h index e9dc917c..3551ad52 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg.h +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h @@ -15,6 +15,8 @@ class ScriptOptionLinesParserCTPG : public ScriptOptionsParser { public: ScriptOptionLinesParserCTPG(); + virtual ~ScriptOptionLinesParserCTPG() {}; + void prepareScriptCode(const std::string & scriptCode) override; void parseForScriptClass(std::function callback) override; diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.h b/exaudfclient/base/javacontainer/script_options/parser_legacy.h index 4005732e..93d4c134 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.h +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.h @@ -15,6 +15,8 @@ class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { public: ScriptOptionLinesParserLegacy(); + virtual ~ScriptOptionLinesParserLegacy() {}; + void prepareScriptCode(const std::string & scriptCode) override; void parseForScriptClass(std::function callback) override; diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc index 8056b406..6ec41888 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc @@ -1,6 +1,8 @@ #include "base/javacontainer/test/cpp/javavm_test.h" #include "base/javacontainer/test/cpp/swig_factory_test.h" #include "base/javacontainer/javacontainer_impl.h" +#include "base/javacontainer/script_options/parser_ctpg.h" +#include "base/javacontainer/script_options/parser_legacy.h" #include @@ -20,11 +22,14 @@ JavaVMTest::JavaVMTest(std::string scriptCode, SwigFactoryTestImpl & swigFactory void JavaVMTest::run(std::string scriptCode, SwigFactoryTestImpl & swigFactory) { char* script_code = ::strdup(scriptCode.c_str()); SWIGVMContainers::SWIGVM_params->script_code = script_code; - bool useCTPGParser = false; -#ifdef USE_CTPG_PARSER - useCTPGParser = true; +#ifndef USE_CTPG_PARSER + std::unique_ptr parser = + std::make_unique(); +#else + std::unique_ptr parser = + std::make_unique(); #endif - SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, swigFactory, useCTPGParser); + SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, swigFactory, std::move(parser)); javaVMInternalStatus.m_exaJavaPath = javaVMImpl.m_exaJavaPath; javaVMInternalStatus.m_localClasspath = javaVMImpl.m_localClasspath; javaVMInternalStatus.m_scriptCode = javaVMImpl.m_scriptCode; diff --git a/exaudfclient/exaudfclient.cc b/exaudfclient/exaudfclient.cc index c1952d3e..22014439 100644 --- a/exaudfclient/exaudfclient.cc +++ b/exaudfclient/exaudfclient.cc @@ -38,7 +38,7 @@ #ifdef ENABLE_JAVA_VM -#include "base/javacontainer/javacontainer.h" +#include "base/javacontainer/javacontainer_builder.h" #endif //ENABLE_JAVA_VM #ifdef ENABLE_PYTHON_VM @@ -163,7 +163,7 @@ int main(int argc, char **argv) { } else if (strcmp(argv[2], "lang=java")==0) { #ifdef ENABLE_JAVA_VM - vmMaker = [&](){return new SWIGVMContainers::JavaVMach(false, swigFactory, false);}; + vmMaker = [&](){return SWIGVMContainers::JavaContainerBuilder().build()}; #else throw SWIGVM::exception("this exaudfclient has been compilied without Java support"); #endif From 3be1631df138ca9911a1b19b7272adf0aea4303c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:22:01 -0300 Subject: [PATCH 21/43] Added description to ctpg_script_importer_test.cc --- .../script_options/test/ctpg_script_importer_test.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc b/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc index 3494377d..7df4ff78 100644 --- a/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc +++ b/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc @@ -29,6 +29,14 @@ const char* buildNewScriptCode(size_t currentIdx) { } TEST(ScriptImporterTest, max_recursion_depth) { + /** + This test checks that running an infinite recursion of the script import will result in the expected exception. + For that, the test creates new "import scripts" on the fly: + Whenever the parser finds a new 'import script' option, + it calls SWIGVMContainers::SWIGMetadataIf::moduleContent(). + The mocked implementation redirects this request to `buildNewScriptCode()` which creates a + new dummy import script with another '%import ...` option. + */ size_t currentIdx = 0; SwigFactoryTestImpl swigFactoryTest([&](const char* scriptKey) { From 60915e55efa47b030a06461cac2c52876e7f7f3a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:23:46 -0300 Subject: [PATCH 22/43] Optimization in script_option_lines_ctpg.cc Parse only lines which contain a '%'. --- .../ctpg/script_option_lines_ctpg.cc | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc index b11dd209..74d799de 100644 --- a/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc +++ b/exaudfclient/base/script_options_parser/ctpg/script_option_lines_ctpg.cc @@ -205,30 +205,36 @@ void parseOptions(const std::string& code, options_map_t & result) { size_t current_pos = 0; do { + const size_t new_option_start_pos = code.find_first_of("%", current_pos); + if (new_option_start_pos == std::string::npos) + break; + current_pos = code.find_last_of("\r\n", new_option_start_pos); + if (std::string::npos == current_pos) + current_pos = 0; + else + current_pos++; const size_t new_pos = code.find_first_of("\r\n", current_pos); std::string line = code.substr(current_pos, new_pos); - if (!line.empty() && !std::all_of(line.begin(),line.end(), [](const char c) {return std::isspace(c);})) { - options_type parser_result; - ParserInternals::parse(std::move(line), parser_result); - for (const auto & option: parser_result) + options_type parser_result; + ParserInternals::parse(std::move(line), parser_result); + for (const auto & option: parser_result) + { + ScriptOption entry = { + .value = option.value, + .idx_in_source = current_pos + option.start.column - 1, + .size = option.end.column - option.start.column + 1 + }; + auto it_in_result = result.find(option.key); + if (it_in_result == result.end()) + { + options_t new_options; + new_options.push_back(entry); + result.insert(std::make_pair(option.key, new_options)); + } + else { - ScriptOption entry = { - .value = option.value, - .idx_in_source = current_pos + option.start.column - 1, - .size = option.end.column - option.start.column + 1 - }; - auto it_in_result = result.find(option.key); - if (it_in_result == result.end()) - { - options_t new_options; - new_options.push_back(entry); - result.insert(std::make_pair(option.key, new_options)); - } - else - { - it_in_result->second.push_back(entry); - } + it_in_result->second.push_back(entry); } } if (new_pos == std::string::npos) { From 9dd5a317a2fe45c6e96da9268d94b1ec4e2b2323 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:24:25 -0300 Subject: [PATCH 23/43] Added performance tests --- exaudfclient/base/javacontainer/test/BUILD | 25 +++++++++++++++++ .../test/cpp/javacontainer_perf_test.cc | 27 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 exaudfclient/base/javacontainer/test/cpp/javacontainer_perf_test.cc diff --git a/exaudfclient/base/javacontainer/test/BUILD b/exaudfclient/base/javacontainer/test/BUILD index 66701152..36fb3620 100644 --- a/exaudfclient/base/javacontainer/test/BUILD +++ b/exaudfclient/base/javacontainer/test/BUILD @@ -13,6 +13,9 @@ java_test( JAVACONTAINER_TEST_SRCS = ["cpp/javacontainer_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"] +JAVACONTAINER_PERF_TEST_SRCS = ["cpp/javacontainer_perf_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", + "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"] + cc_test( name = "javacontainer-test-legacy-parser", srcs = JAVACONTAINER_TEST_SRCS, @@ -33,3 +36,25 @@ cc_test( defines = ["USE_CTPG_PARSER"], data = ["test.jar", "other_test.jar"] ) + + +cc_test( + name = "javacontainer-perf-test-legacy-parser", + srcs = JAVACONTAINER_PERF_TEST_SRCS, + deps = [ + "//base/javacontainer:javacontainer", + "@googletest//:gtest_main", + ], + data = ["test.jar", "other_test.jar"] +) + +cc_test( + name = "javacontainer-perf-test-ctpg-parser", + srcs = JAVACONTAINER_PERF_TEST_SRCS, + deps = [ + "//base/javacontainer:javacontainer", + "@googletest//:gtest_main", + ], + defines = ["USE_CTPG_PARSER"], + data = ["test.jar", "other_test.jar"] +) diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_perf_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_perf_test.cc new file mode 100644 index 00000000..8321b285 --- /dev/null +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_perf_test.cc @@ -0,0 +1,27 @@ +#include "include/gtest/gtest.h" +#include "gmock/gmock.h" +#include "base/javacontainer/test/cpp/javavm_test.h" +#include "base/javacontainer/test/cpp/swig_factory_test.h" +#include + +const uint32_t NumInlineJavaLines = 500000; +const uint32_t NumInlineJavaWordsPerLine = 100; + + +TEST(JavaContainerPerformance, large_inline_java_udf_test) { + std::string script_code = + "%jvmoption option1=abc;\n" + "%jvmoption option2=def;\n" + "class JVMOPTION_TEST_WITH_SPACE {\n" + "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n"; + + for (uint32_t idxLine(0); idxLine < NumInlineJavaLines; idxLine++) { + for (uint32_t idxWord(0); idxWord < NumInlineJavaWordsPerLine; idxWord++) + script_code.append("somecode "); + script_code.append("\n"); + } + script_code.append(" }\n}\n"); + JavaVMTest vm(script_code); +} + + From 85ca346066292826af6aa49dd157a43145751f3a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:58:21 -0300 Subject: [PATCH 24/43] Activated valgrind memory leak check --- .github/workflows/check_bazel_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 49442cad..283355e2 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -27,13 +27,13 @@ jobs: name: "script_options_parser_ctpg" - test: "//base/script_options_parser/legacy/..." name: "script_options_parser_legacy" - - test: "--run_under='valgrind' --copt -DVALGRIND_ACTIVE //base/javacontainer/test:javacontainer-test-legacy-parser" + - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/javacontainer/test:javacontainer-test-legacy-parser" name: "javacontainer-test-legacy-parser-with-valgrind" - - test: "--run_under='valgrind' --copt -DVALGRIND_ACTIVE //base/javacontainer/test:javacontainer-test-ctpg-parser" + - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/javacontainer/test:javacontainer-test-ctpg-parser" name: "javacontainer-test-ctpg-parser-with-valgrind" - - test: "--run_under='valgrind' --copt -DVALGRIND_ACTIVE //base/script_options_parser/ctpg/..." + - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/script_options_parser/ctpg/..." name: "script_options_parser_ctpg_with_valgrind" - - test: "--run_under='valgrind' --copt -DVALGRIND_ACTIVE //base/script_options_parser/legacy/..." + - test: "--run_under='valgrind --leak-check=yes' --config=valgrind //base/script_options_parser/legacy/..." name: "script_options_parser_legacy_with_valgrind" - test: "--config=asan //base/javacontainer/test:javacontainer-test-legacy-parser" name: "javacontainer-test-legacy-parser-with-asan" From 56c94fc45f80e05b7b4cfa94ac0955000c2a1e31 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:59:23 -0300 Subject: [PATCH 25/43] Install R for test_package_management_scripts.yaml --- .github/workflows/test_package_management_scripts.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_package_management_scripts.yaml b/.github/workflows/test_package_management_scripts.yaml index 8757331a..32ec2a3f 100644 --- a/.github/workflows/test_package_management_scripts.yaml +++ b/.github/workflows/test_package_management_scripts.yaml @@ -17,6 +17,8 @@ jobs: with: python-version: "3.10" + - uses: r-lib/actions/setup-r@v2 + - name: Init submodules run: git submodule update --init --recursive From 967c71febe25b3275c7b6613baf41e1d3bfc750d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:14:41 -0300 Subject: [PATCH 26/43] Added comment in parser_ctpg_script_importer.cc --- .../javacontainer/script_options/parser_ctpg_script_importer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc index a86690b6..8f0504e8 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc @@ -72,6 +72,7 @@ void ScriptImporter::importScript(std::string & scriptCode, ReplacedScripts replacedScript = {.script = std::move(importScriptCodeStr), .origPos = option.idx_in_source, .origLen = option.size }; replacedScripts.push_back(std::move(replacedScript)); } + //No replace the imported script bodies from end to start. Doing it in forward order would invalidate the offsets of later import scripts. for (auto optionIt = replacedScripts.rbegin(); optionIt != replacedScripts.rend(); optionIt++) { scriptCode.replace(optionIt->origPos, optionIt->origLen, optionIt->script); } From 8dc58348cd382fbf5ed49bd4d56c0e1628be63e3 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:32:05 -0300 Subject: [PATCH 27/43] Move SwigFactory creation from udf client into JavaContainerBuilder class. --- exaudfclient/BUILD | 4 +- exaudfclient/base/javacontainer/BUILD | 2 +- .../base/javacontainer/javacontainer.cc | 5 +- .../base/javacontainer/javacontainer.h | 3 +- .../javacontainer/javacontainer_builder.cc | 14 ++--- .../javacontainer/javacontainer_builder.h | 5 +- .../base/javacontainer/javacontainer_impl.cc | 17 +++--- .../base/javacontainer/javacontainer_impl.h | 10 ++-- .../javacontainer/script_options/extractor.cc | 8 ++- .../javacontainer/script_options/extractor.h | 7 +-- .../javacontainer/script_options/parser.h | 4 +- .../script_options/parser_ctpg.cc | 10 ++-- .../script_options/parser_ctpg.h | 9 ++-- .../script_options/parser_legacy.cc | 9 ++-- .../script_options/parser_legacy.h | 8 ++- .../test/ctpg_script_importer_test.cc | 7 +-- .../test/cpp/javacontainer_ctpg_test.cc | 7 +-- .../test/cpp/javacontainer_test.cc | 53 ++++++++++--------- .../javacontainer/test/cpp/javavm_test.cc | 20 ++++--- .../base/javacontainer/test/cpp/javavm_test.h | 5 +- 20 files changed, 99 insertions(+), 108 deletions(-) diff --git a/exaudfclient/BUILD b/exaudfclient/BUILD index 656ed774..1c609375 100644 --- a/exaudfclient/BUILD +++ b/exaudfclient/BUILD @@ -78,7 +78,7 @@ cc_binary( srcs = ["exaudfclient.cc", "//base:load_dynamic"], linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace deps = ["//base/exaudflib:header", "//base/utils:utils"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ - ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"], + ["//base/exaudflib:exaudflib-deps"], defines = VM_ENABLED_DEFINES, data = ["//base:libexaudflib_complete.so"] ) @@ -101,7 +101,7 @@ cc_binary( srcs = ["exaudfclient.cc", "//base:load_dynamic"], linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace deps = ["//base/exaudflib:header", "//base/utils:utils"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ - ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"] + + ["//base/exaudflib:exaudflib-deps"] + [ "@zmq//:zmq", "@protobuf//:protobuf"], defines = VM_ENABLED_DEFINES, data = ["//base:libexaudflib_complete.so"], diff --git a/exaudfclient/base/javacontainer/BUILD b/exaudfclient/base/javacontainer/BUILD index 5c837f1d..d63bbf5b 100644 --- a/exaudfclient/base/javacontainer/BUILD +++ b/exaudfclient/base/javacontainer/BUILD @@ -111,7 +111,7 @@ cc_library( hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], deps = ["@java//:java", ":exascript_java", "//base/exaudflib:header", "//base/utils:utils","//base/javacontainer/script_options:java_script_option_lines", - "//base/swig_factory:swig_factory_if"], + "//base/swig_factory:swig_factory"], # copts= ["-O0","-fno-lto"], alwayslink=True, ) diff --git a/exaudfclient/base/javacontainer/javacontainer.cc b/exaudfclient/base/javacontainer/javacontainer.cc index aa655b6a..f8de356c 100644 --- a/exaudfclient/base/javacontainer/javacontainer.cc +++ b/exaudfclient/base/javacontainer/javacontainer.cc @@ -5,10 +5,9 @@ using namespace SWIGVMContainers; using namespace std; -JavaVMach::JavaVMach(bool checkOnly, SwigFactory& swigFactory, - std::unique_ptr scriptOptionsParser) { +JavaVMach::JavaVMach(bool checkOnly, std::unique_ptr scriptOptionsParser) { try { - m_impl = new JavaVMImpl(checkOnly, false, swigFactory, std::move(scriptOptionsParser)); + m_impl = new JavaVMImpl(checkOnly, false, std::move(scriptOptionsParser)); } catch (std::exception& err) { lock_guard lock(exception_msg_mtx); exception_msg = "F-UDF-CL-SL-JAVA-1000: "+std::string(err.what()); diff --git a/exaudfclient/base/javacontainer/javacontainer.h b/exaudfclient/base/javacontainer/javacontainer.h index a03aa268..c8c1b91f 100644 --- a/exaudfclient/base/javacontainer/javacontainer.h +++ b/exaudfclient/base/javacontainer/javacontainer.h @@ -11,7 +11,6 @@ namespace SWIGVMContainers { class JavaVMImpl; -class SwigFactory; namespace JavaScriptOptions { @@ -24,7 +23,7 @@ class JavaVMach: public SWIGVM { /* * scriptOptionsParser: JavaVMach takes ownership of ScriptOptionsParser pointer. */ - JavaVMach(bool checkOnly, SwigFactory& swigFactory, std::unique_ptr scriptOptionsParser); + JavaVMach(bool checkOnly, std::unique_ptr scriptOptionsParser); virtual ~JavaVMach() {} virtual void shutdown(); virtual bool run(); diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.cc b/exaudfclient/base/javacontainer/javacontainer_builder.cc index 6b10fedf..c1535e57 100644 --- a/exaudfclient/base/javacontainer/javacontainer_builder.cc +++ b/exaudfclient/base/javacontainer/javacontainer_builder.cc @@ -2,26 +2,26 @@ #include "base/javacontainer/script_options/parser_ctpg.h" #include "base/javacontainer/script_options/parser_legacy.h" #include "base/javacontainer/javacontainer.h" +#include "base/swig_factory/swig_factory_impl.h" #ifdef ENABLE_JAVA_VM namespace SWIGVMContainers { -JavaContainerBuilder::JavaContainerBuilder(SwigFactory& swigFactory) -: m_parser() -, m_swigFactory(swigFactory) {} +JavaContainerBuilder::JavaContainerBuilder() +: m_parser() {} JavaContainerBuilder& JavaContainerBuilder::useCtpgParser(const bool useCtpgParser) { - m_parser = std::make_unique(); + m_parser = std::make_unique(std::make_unique()); return *this; } JavaVMach* JavaContainerBuilder::build() { if (m_parser) { - return new JavaVMach(false, m_swigFactory, std::move(m_parser)); + return new JavaVMach(false, std::move(m_parser)); } else { - m_parser = std::make_unique(); - return new JavaVMach(false, m_swigFactory, std::move(m_parser)); + m_parser = std::make_unique(std::make_unique()); + return new JavaVMach(false, std::move(m_parser)); } } diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.h b/exaudfclient/base/javacontainer/javacontainer_builder.h index 614ce283..e258abd6 100644 --- a/exaudfclient/base/javacontainer/javacontainer_builder.h +++ b/exaudfclient/base/javacontainer/javacontainer_builder.h @@ -8,7 +8,6 @@ namespace SWIGVMContainers { class JavaVMach; -struct SwigFactory; namespace JavaScriptOptions { @@ -18,7 +17,7 @@ struct ScriptOptionsParser; class JavaContainerBuilder { public: - JavaContainerBuilder(SwigFactory& swigFactory); + JavaContainerBuilder(); JavaContainerBuilder& useCtpgParser(const bool value); @@ -26,8 +25,6 @@ class JavaContainerBuilder { private: std::unique_ptr m_parser; - SwigFactory& m_swigFactory; - }; } //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc index 2fac4e34..8a9cb2a0 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.cc +++ b/exaudfclient/base/javacontainer/javacontainer_impl.cc @@ -12,16 +12,14 @@ #include "base/javacontainer/javacontainer.h" #include "base/javacontainer/javacontainer_impl.h" #include "base/javacontainer/script_options/extractor.h" -#include "base/javacontainer/script_options/parser_legacy.h" -#include "base/javacontainer/script_options/parser_ctpg.h" -#include "base/swig_factory/swig_factory.h" +#include "base/javacontainer/script_options/parser.h" using namespace SWIGVMContainers; using namespace std; -JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, - std::unique_ptr scriptOptionsParser) +JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, + std::unique_ptr scriptOptionsParser) : m_checkOnly(checkOnly) , m_exaJavaPath("") , m_localClasspath("/tmp") // **IMPORTANT**: /tmp needs to be in the classpath, otherwise ExaCompiler crashe with com.exasol.ExaCompilationException: /DATE_STRING.java:3: error: error while writing DATE_STRING: could not create parent directories @@ -30,14 +28,12 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, , m_jvm(NULL) , m_env(NULL) , m_needsCompilation(true) -, m_swigFactory(swigFactory) { stringstream ss; m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path - JavaScriptOptions::ScriptOptionLinesParserCTPG parser; - parseScriptOptions(std::move(scriptOptionsParser), swigFactory); + parseScriptOptions(std::move(scriptOptionsParser)); m_needsCompilation = checkNeedsCompilation(); if (m_needsCompilation) { @@ -54,9 +50,8 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, } } -void JavaVMImpl::parseScriptOptions(std::unique_ptr scriptOptionsParser, - SwigFactory& swigFactory) { - JavaScriptOptions::Extractor extractor(*scriptOptionsParser, swigFactory); +void JavaVMImpl::parseScriptOptions(std::unique_ptr scriptOptionsParser) { + JavaScriptOptions::Extractor extractor(*scriptOptionsParser); DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.h b/exaudfclient/base/javacontainer/javacontainer_impl.h index b0ad812a..973b7ede 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.h +++ b/exaudfclient/base/javacontainer/javacontainer_impl.h @@ -23,10 +23,10 @@ class JavaVMImpl { public: friend class ::JavaVMTest; /* - * scriptOptionsParser: JavaVMach takes ownership of ScriptOptionsParser pointer. + * scriptOptionsParser: JavaVMImpl takes ownership of ScriptOptionsParser pointer. */ - JavaVMImpl(bool checkOnly, bool noJNI, SwigFactory& swigFactory, - std::unique_ptr scriptOptionsParser); + JavaVMImpl(bool checkOnly, bool noJNI, + std::unique_ptr scriptOptionsParser); ~JavaVMImpl() {} void shutdown(); bool run(); @@ -46,8 +46,7 @@ class JavaVMImpl { void throwException(const std::string& ex); void setJvmOptions(); void addJarToClasspath(const std::string& path); - void parseScriptOptions(std::unique_ptr scriptOptionsParser, - SwigFactory& swigFactory); + void parseScriptOptions(std::unique_ptr scriptOptionsParser); bool m_checkOnly; std::string m_exaJavaPath; std::string m_localClasspath; @@ -59,7 +58,6 @@ class JavaVMImpl { JavaVM *m_jvm; JNIEnv *m_env; bool m_needsCompilation; - SwigFactory& m_swigFactory; }; } //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/extractor.cc b/exaudfclient/base/javacontainer/script_options/extractor.cc index bc7934e4..9a75c1c1 100644 --- a/exaudfclient/base/javacontainer/script_options/extractor.cc +++ b/exaudfclient/base/javacontainer/script_options/extractor.cc @@ -10,17 +10,15 @@ namespace SWIGVMContainers { namespace JavaScriptOptions { -Extractor::Extractor(ScriptOptionsParser & parser, - SwigFactory& swigFactory) -: m_parser(parser) -, m_swigFactory(swigFactory) {} +Extractor::Extractor(ScriptOptionsParser & parser) +: m_parser(parser) {} void Extractor::extract(std::string & scriptCode) { m_parser.prepareScriptCode(scriptCode); EXTR_DBG_FUNC_CALL(m_parser.parseForScriptClass( [&](const std::string& value){ EXTR_DBG_FUNC_CALL(m_converter.convertScriptClassName(value)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used })); - EXTR_DBG_FUNC_CALL(m_parser.extractImportScripts(m_swigFactory)); + EXTR_DBG_FUNC_CALL(m_parser.extractImportScripts()); EXTR_DBG_FUNC_CALL(m_parser.parseForJvmOptions( [&](const std::string& value){ EXTR_DBG_FUNC_CALL(m_converter.convertJvmOption(value)); })); diff --git a/exaudfclient/base/javacontainer/script_options/extractor.h b/exaudfclient/base/javacontainer/script_options/extractor.h index d69ae354..03ce623b 100644 --- a/exaudfclient/base/javacontainer/script_options/extractor.h +++ b/exaudfclient/base/javacontainer/script_options/extractor.h @@ -10,8 +10,6 @@ namespace SWIGVMContainers { -struct SwigFactory; - namespace JavaScriptOptions { class ScriptOptionsParser; @@ -19,8 +17,7 @@ class ScriptOptionsParser; class Extractor { public: - Extractor(ScriptOptionsParser & parser, - SwigFactory& swigFactory); + Extractor(ScriptOptionsParser & parser); const std::set & getJarPaths() const { return m_converter.getJarPaths(); @@ -36,8 +33,6 @@ class Extractor { Converter m_converter; ScriptOptionsParser & m_parser; - - SwigFactory& m_swigFactory; }; diff --git a/exaudfclient/base/javacontainer/script_options/parser.h b/exaudfclient/base/javacontainer/script_options/parser.h index e82eac05..abb497fc 100644 --- a/exaudfclient/base/javacontainer/script_options/parser.h +++ b/exaudfclient/base/javacontainer/script_options/parser.h @@ -8,8 +8,6 @@ namespace SWIGVMContainers { -struct SwigFactory; - namespace JavaScriptOptions { struct ScriptOptionsParser { @@ -43,7 +41,7 @@ struct ScriptOptionsParser { Searches for the "%import" options and embeds the respective imported script code at the same location as the option in the script code. */ - virtual void extractImportScripts(SwigFactory & swigFactory) = 0; + virtual void extractImportScripts() = 0; /* Returns the (eventually modified) script code. diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc index a545d43c..3f576242 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc @@ -2,6 +2,7 @@ #include "base/javacontainer/script_options/parser_ctpg_script_importer.h" #include "base/utils/exceptions.h" #include "base/script_options_parser/exception.h" +#include "base/swig_factory/swig_factory.h" #include @@ -11,10 +12,11 @@ namespace SWIGVMContainers { namespace JavaScriptOptions { -ScriptOptionLinesParserCTPG::ScriptOptionLinesParserCTPG() +ScriptOptionLinesParserCTPG::ScriptOptionLinesParserCTPG(std::unique_ptr swigFactory) : m_scriptCode() , m_keywords(false) -, m_needParsing(true) {} +, m_needParsing(true) +, m_swigFactory(std::move(swigFactory)) {} void ScriptOptionLinesParserCTPG::prepareScriptCode(const std::string & scriptCode) { m_scriptCode = scriptCode; @@ -44,7 +46,7 @@ void ScriptOptionLinesParserCTPG::parseForExternalJars(std::function namespace SWIGVMContainers { +struct SwigFactory; + namespace JavaScriptOptions { class ScriptOptionLinesParserCTPG : public ScriptOptionsParser { public: - ScriptOptionLinesParserCTPG(); + ScriptOptionLinesParserCTPG(std::unique_ptr swigFactory); virtual ~ScriptOptionLinesParserCTPG() {}; @@ -25,7 +27,7 @@ class ScriptOptionLinesParserCTPG : public ScriptOptionsParser { void parseForExternalJars(std::function callback) override; - void extractImportScripts(SwigFactory & swigFactory) override; + void extractImportScripts() override; std::string && getScriptCode() override; @@ -42,6 +44,7 @@ class ScriptOptionLinesParserCTPG : public ScriptOptionsParser { Keywords m_keywords; ExecutionGraph::OptionsLineParser::CTPG::options_map_t m_foundOptions; bool m_needParsing; + std::unique_ptr m_swigFactory; }; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc index d46b355d..4a07ead4 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc @@ -13,17 +13,18 @@ namespace SWIGVMContainers { namespace JavaScriptOptions { -ScriptOptionLinesParserLegacy::ScriptOptionLinesParserLegacy() +ScriptOptionLinesParserLegacy::ScriptOptionLinesParserLegacy(std::unique_ptr swigFactory) : m_whitespace(" \t\f\v") , m_lineend(";") , m_scriptCode() -, m_keywords(true) {} +, m_keywords(true) +, m_swigFactory(std::move(swigFactory)) {} void ScriptOptionLinesParserLegacy::prepareScriptCode(const std::string & scriptCode) { m_scriptCode = scriptCode; } -void ScriptOptionLinesParserLegacy::extractImportScripts(SwigFactory & swigFactory) { +void ScriptOptionLinesParserLegacy::extractImportScripts() { std::unique_ptr metaData; // Attention: We must hash the parent script before modifying it (adding the // package definition). Otherwise we don't recognize if the script imports its self @@ -111,7 +112,7 @@ void ScriptOptionLinesParserLegacy::extractImportScripts(SwigFactory & swigFacto } if (!newScript.empty()) { if (!metaData) { - metaData.reset(swigFactory.makeSwigMetadata()); + metaData.reset(m_swigFactory->makeSwigMetadata()); if (!metaData) throw std::runtime_error("F-UDF-CL-SL-JAVA-1615: Failure while importing scripts"); } diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.h b/exaudfclient/base/javacontainer/script_options/parser_legacy.h index 93d4c134..447ca394 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.h +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.h @@ -5,15 +5,18 @@ #include "base/javacontainer/script_options/parser.h" #include "base/javacontainer/script_options/keywords.h" +#include namespace SWIGVMContainers { +struct SwigFactory; + namespace JavaScriptOptions { class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { public: - ScriptOptionLinesParserLegacy(); + ScriptOptionLinesParserLegacy(std::unique_ptr swigFactory); virtual ~ScriptOptionLinesParserLegacy() {}; @@ -25,7 +28,7 @@ class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { void parseForExternalJars(std::function callback) override; - void extractImportScripts(SwigFactory & swigFactory) override; + void extractImportScripts() override; std::string && getScriptCode() override; @@ -40,6 +43,7 @@ class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { const std::string m_lineend; std::string m_scriptCode; Keywords m_keywords; + std::unique_ptr m_swigFactory; }; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc b/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc index 7df4ff78..eb34a961 100644 --- a/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc +++ b/exaudfclient/base/javacontainer/script_options/test/ctpg_script_importer_test.cc @@ -39,18 +39,19 @@ TEST(ScriptImporterTest, max_recursion_depth) { */ size_t currentIdx = 0; - SwigFactoryTestImpl swigFactoryTest([&](const char* scriptKey) { + std::unique_ptr swigFactory = + std::make_unique([&](const char* scriptKey) { checkIndex(currentIdx, scriptKey); return buildNewScriptCode(++currentIdx); }); - ScriptOptionLinesParserCTPG parser; + ScriptOptionLinesParserCTPG parser(std::move(swigFactory)); const std::string code = buildNewScriptCode(currentIdx); parser.prepareScriptCode(code); EXPECT_THROW({ try { - parser.extractImportScripts(swigFactoryTest); + parser.extractImportScripts(); } catch( const std::runtime_error& e ) { diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc index a3db7c2d..e66168e5 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_ctpg_test.cc @@ -4,6 +4,7 @@ #include "base/javacontainer/test/cpp/javavm_test.h" #include "base/javacontainer/test/cpp/swig_factory_test.h" #include +#include class JavaContainerEscapeSequenceTest : public ::testing::TestWithParam> {}; @@ -72,7 +73,7 @@ TEST(JavaContainer, import_script_with_escaped_options) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + std::unique_ptr swigFactory = std::make_unique(); const std::string other_script_code = "%jvmoption -Dsomeotheroption=\"DE\\nF\";\n\n" @@ -81,8 +82,8 @@ TEST(JavaContainer, import_script_with_escaped_options) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc index 4b98c4f5..0c754206 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc @@ -5,6 +5,7 @@ #include "base/javacontainer/test/cpp/swig_factory_test.h" #include + TEST(JavaContainer, basic_jar) { const std::string script_code = "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" "%jar base/javacontainer/test/test.jar;"; @@ -123,15 +124,15 @@ TEST(JavaContainer, simple_import_script) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "class OtherClass {\n" "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -164,7 +165,7 @@ TEST(JavaContainer, import_script_with_recursion) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "%import other_script;\n\n" @@ -172,8 +173,8 @@ TEST(JavaContainer, import_script_with_recursion) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -205,7 +206,7 @@ TEST(JavaContainer, import_script_with_jvmoption) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "%jvmoption -Dhttp.agent=\"ABC\";\n\n" @@ -213,8 +214,8 @@ TEST(JavaContainer, import_script_with_jvmoption) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -247,7 +248,7 @@ TEST(JavaContainer, multiple_import_scripts) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_scipt_code_A = "%import other_script_B;\n\n" @@ -266,10 +267,10 @@ TEST(JavaContainer, multiple_import_scripts) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script_A", other_scipt_code_A); - swigFactory.addModule("other_script_B", other_scipt_code_B); - swigFactory.addModule("other_script_C", other_scipt_code_C); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script_A", other_scipt_code_A); + swigFactory->addModule("other_script_B", other_scipt_code_B); + swigFactory->addModule("other_script_C", other_scipt_code_C); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -312,7 +313,7 @@ TEST(JavaContainer, import_script_with_mixed_options) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "%jvmoption -Dsomeotheroption=\"DEF\";\n\n" @@ -321,8 +322,8 @@ TEST(JavaContainer, import_script_with_mixed_options) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -355,7 +356,7 @@ TEST(JavaContainer, import_script_script_class_option_ignored) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_script_code = "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" @@ -363,8 +364,8 @@ TEST(JavaContainer, import_script_script_class_option_ignored) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script", other_script_code); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script", other_script_code); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = @@ -401,7 +402,7 @@ TEST(JavaContainer, import_scripts_deep_recursion) { " ctx.emit(\"Success!\");\n" " }\n" "}\n"; - SwigFactoryTestImpl swigFactory; + auto swigFactory = std::make_unique(); const std::string other_scipt_code_A = "%import other_script_B;\n\n" @@ -427,11 +428,11 @@ TEST(JavaContainer, import_scripts_deep_recursion) { "static void doSomething() {\n\n" " }\n" "}\n"; - swigFactory.addModule("other_script_A", other_scipt_code_A); - swigFactory.addModule("other_script_B", other_scipt_code_B); - swigFactory.addModule("other_script_C", other_scipt_code_C); - swigFactory.addModule("other_script_D", other_scipt_code_D); - JavaVMTest vm(script_code, swigFactory); + swigFactory->addModule("other_script_A", other_scipt_code_A); + swigFactory->addModule("other_script_B", other_scipt_code_B); + swigFactory->addModule("other_script_C", other_scipt_code_C); + swigFactory->addModule("other_script_D", other_scipt_code_D); + JavaVMTest vm(script_code, std::move(swigFactory)); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc index 6ec41888..6a26dc35 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc @@ -5,31 +5,29 @@ #include "base/javacontainer/script_options/parser_legacy.h" #include - -SwigFactoryTestImpl & defaultSwigFactory() { - static SwigFactoryTestImpl swigFactory; - return swigFactory; +std::unique_ptr makeDefaultSwigFactory() { + return std::make_unique(); } JavaVMTest::JavaVMTest(std::string scriptCode) : javaVMInternalStatus() { - run(scriptCode, defaultSwigFactory()); + run(scriptCode, std::move(makeDefaultSwigFactory())); } -JavaVMTest::JavaVMTest(std::string scriptCode, SwigFactoryTestImpl & swigFactory) : javaVMInternalStatus() { - run(scriptCode, swigFactory); +JavaVMTest::JavaVMTest(std::string scriptCode, std::unique_ptr swigFactory) : javaVMInternalStatus() { + run(scriptCode, std::move(swigFactory)); } -void JavaVMTest::run(std::string scriptCode, SwigFactoryTestImpl & swigFactory) { +void JavaVMTest::run(std::string scriptCode, std::unique_ptr swigFactory) { char* script_code = ::strdup(scriptCode.c_str()); SWIGVMContainers::SWIGVM_params->script_code = script_code; #ifndef USE_CTPG_PARSER std::unique_ptr parser = - std::make_unique(); + std::make_unique(std::move(swigFactory)); #else std::unique_ptr parser = - std::make_unique(); + std::make_unique(std::move(swigFactory)); #endif - SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, swigFactory, std::move(parser)); + SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, std::move(parser)); javaVMInternalStatus.m_exaJavaPath = javaVMImpl.m_exaJavaPath; javaVMInternalStatus.m_localClasspath = javaVMImpl.m_localClasspath; javaVMInternalStatus.m_scriptCode = javaVMImpl.m_scriptCode; diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.h b/exaudfclient/base/javacontainer/test/cpp/javavm_test.h index d4e31844..a0e8e0b5 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.h +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.h @@ -4,6 +4,7 @@ #include #include #include +#include struct JavaVMInternalStatus { std::string m_exaJavaPath; @@ -21,12 +22,12 @@ class JavaVMTest { public: JavaVMTest(std::string scriptCode); - JavaVMTest(std::string scriptCode, SwigFactoryTestImpl & swigFactory); + JavaVMTest(std::string scriptCode, std::unique_ptr swigFactory); const JavaVMInternalStatus& getJavaVMInternalStatus() {return javaVMInternalStatus;} private: - void run(std::string scriptCode, SwigFactoryTestImpl & swigFactory); + void run(std::string scriptCode, std::unique_ptr swigFactory); private: JavaVMInternalStatus javaVMInternalStatus; }; From f1b1940d3ea562e0197943f39c7eb3d9a9758300 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:34:00 -0300 Subject: [PATCH 28/43] Fixed JavaContainerBuilder::useCtpgParser --- exaudfclient/base/javacontainer/javacontainer_builder.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.cc b/exaudfclient/base/javacontainer/javacontainer_builder.cc index c1535e57..9b27a326 100644 --- a/exaudfclient/base/javacontainer/javacontainer_builder.cc +++ b/exaudfclient/base/javacontainer/javacontainer_builder.cc @@ -12,7 +12,9 @@ JavaContainerBuilder::JavaContainerBuilder() : m_parser() {} JavaContainerBuilder& JavaContainerBuilder::useCtpgParser(const bool useCtpgParser) { - m_parser = std::make_unique(std::make_unique()); + if (useCtpgParser) { + m_parser = std::make_unique(std::make_unique()); + } return *this; } From a846279ea4356762b91b66e7a137fa56f8bc4bde Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:46:57 -0300 Subject: [PATCH 29/43] Added virtual destructor to SwigFactory. --- exaudfclient/base/swig_factory/swig_factory.h | 1 + 1 file changed, 1 insertion(+) diff --git a/exaudfclient/base/swig_factory/swig_factory.h b/exaudfclient/base/swig_factory/swig_factory.h index 06e82952..48f93c1d 100644 --- a/exaudfclient/base/swig_factory/swig_factory.h +++ b/exaudfclient/base/swig_factory/swig_factory.h @@ -6,6 +6,7 @@ namespace SWIGVMContainers { struct SWIGMetadataIf; struct SwigFactory { + virtual ~SwigFactory() {}; virtual SWIGMetadataIf* makeSwigMetadata() = 0; From ef099fd79d156ef9c8d3c0f2f6ff176c88487927 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:42:06 -0300 Subject: [PATCH 30/43] Fixed compiler errors --- .../base/javacontainer/javacontainer_builder.cc | 16 +++++++--------- .../base/javacontainer/javacontainer_builder.h | 5 ++--- exaudfclient/exaudfclient.cc | 5 +---- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.cc b/exaudfclient/base/javacontainer/javacontainer_builder.cc index 9b27a326..ebbaab74 100644 --- a/exaudfclient/base/javacontainer/javacontainer_builder.cc +++ b/exaudfclient/base/javacontainer/javacontainer_builder.cc @@ -1,7 +1,6 @@ #include "base/javacontainer/javacontainer_builder.h" #include "base/javacontainer/script_options/parser_ctpg.h" #include "base/javacontainer/script_options/parser_legacy.h" -#include "base/javacontainer/javacontainer.h" #include "base/swig_factory/swig_factory_impl.h" #ifdef ENABLE_JAVA_VM @@ -9,22 +8,21 @@ namespace SWIGVMContainers { JavaContainerBuilder::JavaContainerBuilder() -: m_parser() {} +: m_useCtpgParser(false) {} JavaContainerBuilder& JavaContainerBuilder::useCtpgParser(const bool useCtpgParser) { - if (useCtpgParser) { - m_parser = std::make_unique(std::make_unique()); - } + m_useCtpgParser = useCtpgParser; return *this; } JavaVMach* JavaContainerBuilder::build() { - if (m_parser) { - return new JavaVMach(false, std::move(m_parser)); + std::unique_ptr parser; + if (m_useCtpgParser) { + parser = std::make_unique(std::make_unique()); } else { - m_parser = std::make_unique(std::make_unique()); - return new JavaVMach(false, std::move(m_parser)); + parser = std::make_unique(std::make_unique()); } + return new JavaVMach(false, std::move(parser)); } diff --git a/exaudfclient/base/javacontainer/javacontainer_builder.h b/exaudfclient/base/javacontainer/javacontainer_builder.h index e258abd6..5893b6d2 100644 --- a/exaudfclient/base/javacontainer/javacontainer_builder.h +++ b/exaudfclient/base/javacontainer/javacontainer_builder.h @@ -2,13 +2,12 @@ #define JAVACONTAINER_BUILDER_H #include +#include "base/javacontainer/javacontainer.h" #ifdef ENABLE_JAVA_VM namespace SWIGVMContainers { -class JavaVMach; - namespace JavaScriptOptions { struct ScriptOptionsParser; @@ -24,7 +23,7 @@ class JavaContainerBuilder { JavaVMach* build(); private: - std::unique_ptr m_parser; + bool m_useCtpgParser; }; } //namespace SWIGVMContainers diff --git a/exaudfclient/exaudfclient.cc b/exaudfclient/exaudfclient.cc index 22014439..40756f18 100644 --- a/exaudfclient/exaudfclient.cc +++ b/exaudfclient/exaudfclient.cc @@ -49,8 +49,6 @@ #include "protegrityclient.h" #endif -#include "base/swig_factory/swig_factory_impl.h" - using namespace std; using namespace SWIGVMContainers; @@ -140,7 +138,6 @@ int main(int argc, char **argv) { ::setlocale(LC_ALL, "en_US.utf8"); std::functionvmMaker=[](){return nullptr;}; // the initial vm maker returns NULL - SwigFactoryImpl swigFactory; #ifdef UDF_PLUGIN_CLIENT vmMaker = [](){return new SWIGVMContainers::Protegrity(false);}; #else @@ -163,7 +160,7 @@ int main(int argc, char **argv) { } else if (strcmp(argv[2], "lang=java")==0) { #ifdef ENABLE_JAVA_VM - vmMaker = [&](){return SWIGVMContainers::JavaContainerBuilder().build()}; + vmMaker = [&](){return SWIGVMContainers::JavaContainerBuilder().build();}; #else throw SWIGVM::exception("this exaudfclient has been compilied without Java support"); #endif From e63355cd0635781aafba4982b4993b1afaa19bd2 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:42:17 -0300 Subject: [PATCH 31/43] Added a bazl build check --- .github/workflows/check_bazel_tests.yml | 29 +++++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 283355e2..5cef1370 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -7,7 +7,29 @@ on: pull_request: jobs: + env: + USE_BAZEL_VERSION: 7.2.1 build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Search for duplicated error codes + run: bash find_duplicate_error_codes.sh + + - name: Install bazel + run: | + curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 + chmod +x bazel + - name: Install JDK and ZMQ + run: | + sudo apt-get update + sudo apt-get install -y openjdk-11-jdk libzmq3-dev + - name: Build + run: | + bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --verbose_failures + working-directory: ./exaudfclient/ + + tests: runs-on: ubuntu-latest strategy: fail-fast: false @@ -43,15 +65,8 @@ jobs: name: "script_options_parser_ctpg_with_asan" - test: "--config=asan //base/script_options_parser/legacy/..." name: "script_options_parser_legacy_with_asan" - - - env: - USE_BAZEL_VERSION: 7.2.1 steps: - uses: actions/checkout@v4 - - name: Search for duplicated error codes - run: bash find_duplicate_error_codes.sh - - name: Install bazel run: | curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 From 5bd6219240a91bbb6ed1f35c326d861f2695b724 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:44:39 -0300 Subject: [PATCH 32/43] Fixed formatting in check_bazel_tests.yml --- .github/workflows/check_bazel_tests.yml | 66 ++++++++++++------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 5cef1370..e640b54f 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -12,22 +12,22 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Search for duplicated error codes - run: bash find_duplicate_error_codes.sh + - uses: actions/checkout@v4 + - name: Search for duplicated error codes + run: bash find_duplicate_error_codes.sh - - name: Install bazel - run: | - curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x bazel - - name: Install JDK and ZMQ - run: | - sudo apt-get update - sudo apt-get install -y openjdk-11-jdk libzmq3-dev - - name: Build - run: | - bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --verbose_failures - working-directory: ./exaudfclient/ + - name: Install bazel + run: | + curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 + chmod +x bazel + - name: Install JDK and ZMQ + run: | + sudo apt-get update + sudo apt-get install -y openjdk-11-jdk libzmq3-dev + - name: Build + run: | + bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --verbose_failures + working-directory: ./exaudfclient/ tests: runs-on: ubuntu-latest @@ -66,22 +66,22 @@ jobs: - test: "--config=asan //base/script_options_parser/legacy/..." name: "script_options_parser_legacy_with_asan" steps: - - uses: actions/checkout@v4 - - name: Install bazel - run: | - curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x bazel - - name: Install JDK and ZMQ - run: | - sudo apt-get update - sudo apt-get install -y openjdk-11-jdk libzmq3-dev valgrind - - name: Run tests - run: | - bazel test ${{ matrix.additional_args }} ${{ matrix.test }} - working-directory: ./exaudfclient/ + - uses: actions/checkout@v4 + - name: Install bazel + run: | + curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 + chmod +x bazel + - name: Install JDK and ZMQ + run: | + sudo apt-get update + sudo apt-get install -y openjdk-11-jdk libzmq3-dev valgrind + - name: Run tests + run: | + bazel test ${{ matrix.additional_args }} ${{ matrix.test }} + working-directory: ./exaudfclient/ - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: "${{ matrix.name }}" - path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/**/test.log + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: "${{ matrix.name }}" + path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/**/test.log From 66afcb0e16cc9b3b71951e010567b6c629557577 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:46:14 -0300 Subject: [PATCH 33/43] Fixed formatting in check_bazel_tests.yml --- .github/workflows/check_bazel_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index e640b54f..11ec2fc7 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -6,9 +6,9 @@ on: - master pull_request: +env: + USE_BAZEL_VERSION: 7.2.1 jobs: - env: - USE_BAZEL_VERSION: 7.2.1 build: runs-on: ubuntu-latest steps: From f6ebc7e11240199ba90ebdfc78834358ca6e12f2 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:48:19 -0300 Subject: [PATCH 34/43] Removed Java from bazel build (dependencies not installed) --- .github/workflows/check_bazel_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 11ec2fc7..24069b37 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -26,7 +26,7 @@ jobs: sudo apt-get install -y openjdk-11-jdk libzmq3-dev - name: Build run: | - bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --verbose_failures + bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config fast-binary --verbose_failures working-directory: ./exaudfclient/ tests: From 8f570d6b629eb7cbe497a479425ab1c37b2bc5f9 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:53:12 -0300 Subject: [PATCH 35/43] Added Python env variables in check_bazel_tests.yml --- .github/workflows/check_bazel_tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 24069b37..fa15caba 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -10,7 +10,7 @@ env: USE_BAZEL_VERSION: 7.2.1 jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Search for duplicated error codes @@ -23,11 +23,14 @@ jobs: - name: Install JDK and ZMQ run: | sudo apt-get update - sudo apt-get install -y openjdk-11-jdk libzmq3-dev + sudo apt-get install -y openjdk-11-jdk libzmq3-dev python3 - name: Build run: | bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config fast-binary --verbose_failures working-directory: ./exaudfclient/ + env: + - PYTHON3_PREFIX: /usr + - PYTHON3_VERSION: python3.10 tests: runs-on: ubuntu-latest From 89f29f2c09fb23cbe5a760747e97e0b0ba90026b Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:54:55 -0300 Subject: [PATCH 36/43] fixed Python env variables in check_bazel_tests.yml --- .github/workflows/check_bazel_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index fa15caba..dd89488f 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -29,8 +29,8 @@ jobs: bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config fast-binary --verbose_failures working-directory: ./exaudfclient/ env: - - PYTHON3_PREFIX: /usr - - PYTHON3_VERSION: python3.10 + PYTHON3_PREFIX: /usr + PYTHON3_VERSION: python3.10 tests: runs-on: ubuntu-latest From c6ca9983571fa2c94d1b64b418278877e457248e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 07:03:22 -0300 Subject: [PATCH 37/43] Added protobuf compiler to build job in check_bazel_tests.yml --- .github/workflows/check_bazel_tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index dd89488f..565826a5 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -23,7 +23,7 @@ jobs: - name: Install JDK and ZMQ run: | sudo apt-get update - sudo apt-get install -y openjdk-11-jdk libzmq3-dev python3 + sudo apt-get install -y openjdk-11-jdk libzmq3-dev python3 protobuf-compiler - name: Build run: | bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config fast-binary --verbose_failures @@ -31,6 +31,11 @@ jobs: env: PYTHON3_PREFIX: /usr PYTHON3_VERSION: python3.10 + ZMQ_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu/ + ZMQ_INCLUDE_PREFIX: /usr/include + PROTOBUF_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu + PROTOBUF_INCLUDE_PREFIX: /usr/include/ + PROTOBUF_BIN: /usr/bin/protoc tests: runs-on: ubuntu-latest From 53c9606cfef9a920801a9f869f6362c550bd7506 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 08:20:59 -0300 Subject: [PATCH 38/43] Created installUdfClientDeps.sh --- .github/workflows/check_bazel_tests.yml | 31 ++++---------- scripts/installUdfClientDeps.sh | 56 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 23 deletions(-) create mode 100644 scripts/installUdfClientDeps.sh diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 565826a5..0b80aaa6 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -8,6 +8,7 @@ on: env: USE_BAZEL_VERSION: 7.2.1 + UDF_CLIENT_ENV_FILE: /tmp/.udf_client_env jobs: build: runs-on: ubuntu-22.04 @@ -16,26 +17,14 @@ jobs: - name: Search for duplicated error codes run: bash find_duplicate_error_codes.sh - - name: Install bazel - run: | - curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x bazel - - name: Install JDK and ZMQ + - name: Install UDF Client Dependencies run: | - sudo apt-get update - sudo apt-get install -y openjdk-11-jdk libzmq3-dev python3 protobuf-compiler + bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" - name: Build run: | - bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config fast-binary --verbose_failures + source "$UDF_CLIENT_ENV_FILE" + bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --verbose_failures working-directory: ./exaudfclient/ - env: - PYTHON3_PREFIX: /usr - PYTHON3_VERSION: python3.10 - ZMQ_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu/ - ZMQ_INCLUDE_PREFIX: /usr/include - PROTOBUF_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu - PROTOBUF_INCLUDE_PREFIX: /usr/include/ - PROTOBUF_BIN: /usr/bin/protoc tests: runs-on: ubuntu-latest @@ -75,16 +64,12 @@ jobs: name: "script_options_parser_legacy_with_asan" steps: - uses: actions/checkout@v4 - - name: Install bazel - run: | - curl -L -o bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x bazel - - name: Install JDK and ZMQ + - name: Install UDF Client Dependencies run: | - sudo apt-get update - sudo apt-get install -y openjdk-11-jdk libzmq3-dev valgrind + bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" - name: Run tests run: | + source "$UDF_CLIENT_ENV_FILE" bazel test ${{ matrix.additional_args }} ${{ matrix.test }} working-directory: ./exaudfclient/ diff --git a/scripts/installUdfClientDeps.sh b/scripts/installUdfClientDeps.sh new file mode 100644 index 00000000..075b5c06 --- /dev/null +++ b/scripts/installUdfClientDeps.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail + +source /etc/os-release + +if [ "$VERSION_CODENAME" != "jammy" ]; then + echo "Script can only work correctly under Ubuntu 22.04" + exit 1 +fi + +if [ $# -ne 1 ]; +then + me=$(basename "$0") + echo "Usage: '$me '. For example: '$me .env'. will contain necessary environment variables to run 'bazel build'." + exit 1 +fi + +OUT_ENV_FILE="$1" + +export DEBIAN_FRONTEND=noninteractive + +apt update +apt install -y curl openjdk-11-jdk libzmq3-dev python3 protobuf-compiler build-essential python3-pip libpcre3-dev chrpath tar locales coreutils libssl-dev + +BAZEL_PACKAGE_VERSION="7.2.1" +BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" +BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" + + +curl -L --output "$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" && \ +apt install -y "./$BAZEL_PACKAGE_FILE" && \ +rm "$BAZEL_PACKAGE_FILE" && \ + +apt -y clean +apt -y autoremove + + +curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \ + tar zxf swig-2.0.4.tar.gz && \ + (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \ + rm -rf swig-2.0.4 swig-2.0.4.tar.gz + + +pip install numpy + + +cat >"$OUT_ENV_FILE" < Date: Tue, 15 Oct 2024 08:22:06 -0300 Subject: [PATCH 39/43] Run installUdfClientDeps.sh as sudo --- .github/workflows/check_bazel_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 0b80aaa6..ccc31fc9 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -19,7 +19,7 @@ jobs: - name: Install UDF Client Dependencies run: | - bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" + sudo bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" - name: Build run: | source "$UDF_CLIENT_ENV_FILE" @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - name: Install UDF Client Dependencies run: | - bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" + sudo bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" - name: Run tests run: | source "$UDF_CLIENT_ENV_FILE" From f09a77d03bf1cc89ad55ef6a15ba75145f40ffac Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 08:22:26 -0300 Subject: [PATCH 40/43] Update exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc Co-authored-by: Torsten Kilias --- .../javacontainer/script_options/parser_ctpg_script_importer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc index 8f0504e8..226e2359 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg_script_importer.cc @@ -72,7 +72,7 @@ void ScriptImporter::importScript(std::string & scriptCode, ReplacedScripts replacedScript = {.script = std::move(importScriptCodeStr), .origPos = option.idx_in_source, .origLen = option.size }; replacedScripts.push_back(std::move(replacedScript)); } - //No replace the imported script bodies from end to start. Doing it in forward order would invalidate the offsets of later import scripts. + //Now replace the imported script bodies from end to start. Doing it in forward order would invalidate the offsets of later import scripts. for (auto optionIt = replacedScripts.rbegin(); optionIt != replacedScripts.rend(); optionIt++) { scriptCode.replace(optionIt->origPos, optionIt->origLen, optionIt->script); } From 4287b0e20b7ac33096354f4dcc344ef079c8af0a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 08:31:00 -0300 Subject: [PATCH 41/43] Don't use installUdfClientDeps.sh for bazel test Background: Tests need to run under Ubuntu 24.04 or later (because of ASAN). installUdfClientDeps.sh is tested only on Ubuntu 22.04. --- .github/workflows/check_bazel_tests.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index ccc31fc9..ea35ed85 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -64,9 +64,19 @@ jobs: name: "script_options_parser_legacy_with_asan" steps: - uses: actions/checkout@v4 - - name: Install UDF Client Dependencies + - name: Install JDK and ZMQ run: | - sudo bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" + sudo apt-get update + sudo apt-get install -y openjdk-11-jdk libzmq3-dev valgrind + - name: Install bazel + run: | + BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" + BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" + curl -L --output /tmp/"$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" + apt install -y "/tmp/$BAZEL_PACKAGE_FILE" + rm "/tmp/$BAZEL_PACKAGE_FILE" + env: + BAZEL_PACKAGE_VERSION: "7.2.1" - name: Run tests run: | source "$UDF_CLIENT_ENV_FILE" From e39d4c7386bc0df428a3eb9be3019598262a294f Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 08:32:28 -0300 Subject: [PATCH 42/43] Fixed installation of bazel in check_bazel_tests.yml --- .github/workflows/check_bazel_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index ea35ed85..a592f25a 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -73,7 +73,7 @@ jobs: BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" curl -L --output /tmp/"$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" - apt install -y "/tmp/$BAZEL_PACKAGE_FILE" + sudo apt install -y "/tmp/$BAZEL_PACKAGE_FILE" rm "/tmp/$BAZEL_PACKAGE_FILE" env: BAZEL_PACKAGE_VERSION: "7.2.1" From c812821fc876ed71d34ef9160cb4e53a736f19ba Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 15 Oct 2024 08:34:03 -0300 Subject: [PATCH 43/43] Fixed execution of bazel tests in check_bazel_tests.yml --- .github/workflows/check_bazel_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index a592f25a..9d0e28dc 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -79,7 +79,6 @@ jobs: BAZEL_PACKAGE_VERSION: "7.2.1" - name: Run tests run: | - source "$UDF_CLIENT_ENV_FILE" bazel test ${{ matrix.additional_args }} ${{ matrix.test }} working-directory: ./exaudfclient/