From 07e916143501395127a79229ffc060428a5539c6 Mon Sep 17 00:00:00 2001 From: Bi0T1N <28802083+Bi0T1N@users.noreply.github.com> Date: Tue, 7 Feb 2023 22:00:59 +0100 Subject: [PATCH 1/3] Cherry-pick the python3 embedSdf script and tests (#884) These won't be used as part of the cmake build on this branch, but are useful for generating the same file from bazel. Co-authored-by: Bi0T1N Co-authored-by: Michael Carroll Signed-off-by: Michael Carroll --- sdf/CMakeLists.txt | 4 +- sdf/embedSdf.py | 151 +++++++++++++++++++++++++++++++++++++++++++++ src/SDF_TEST.cc | 56 +++++++++++++++++ 3 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 sdf/embedSdf.py diff --git a/sdf/CMakeLists.txt b/sdf/CMakeLists.txt index 127458d91..e4b6a78e0 100644 --- a/sdf/CMakeLists.txt +++ b/sdf/CMakeLists.txt @@ -20,8 +20,8 @@ execute_process( OUTPUT_FILE "${PROJECT_BINARY_DIR}/src/EmbeddedSdf.cc" ) -# Generate aggregated SDF description files for use by the sdformat.org -# website. If the description files change, the generated full*.sdf files need +# Generate aggregated SDF description files for use by the sdformat.org +# website. If the description files change, the generated full*.sdf files need # to be removed before running this target. if (GZ_PROGRAM) diff --git a/sdf/embedSdf.py b/sdf/embedSdf.py new file mode 100644 index 000000000..b89607fa6 --- /dev/null +++ b/sdf/embedSdf.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +import inspect +from pathlib import Path, PurePosixPath + +""""Script for generating a C++ file that contains the content from all SDF files""" + +# The list of supported SDF specification versions. This will let us drop +# versions without removing the directories. +SUPPORTED_SDF_VERSIONS = ['1.10', '1.9', '1.8', '1.7', '1.6', '1.5', '1.4', '1.3', '1.2'] + +# The list of supported SDF conversions. This list includes versions that +# a user can convert an existing SDF version to. +SUPPORTED_SDF_CONVERSIONS = ['1.10', '1.9', '1.8', '1.7', '1.6', '1.5', '1.4', '1.3'] + +# whitespace indentation for C++ code +INDENTATION = ' ' +# newline character +NEWLINE = '\n' + +def get_copyright_notice() -> str: + """ + Provides the copyrigt notice for the C++ file + + :returns: copyright notice + """ + res = inspect.cleandoc(""" + /* + * Copyright 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + """) + return res + 2*NEWLINE + + +def get_file_header_prolog() -> str: + """ + Provides the include statement, namespace and variable declaration of the C++ file + + :returns: prolog of the C++ file + """ + res = inspect.cleandoc(""" + #include "EmbeddedSdf.hh" + + namespace sdf + { + inline namespace SDF_VERSION_NAMESPACE + { + ///////////////////////////////////////////////// + const std::map &GetEmbeddedSdf() + { + static const std::map result { + """) + return res + NEWLINE + + +def embed_sdf_content(arg_path: str, arg_file_content: str) -> str: + """ + Generates a string pair with the folder and filename as well as the content of the file + + :param arg_path: Foldername and filename of the SDF + :param arg_file_content: Content of the provided file + :returns: raw string literal mapping pair for the std::map + """ + res = [] + res.append('// NOLINT') + res.append('{') + res.append(f'"{arg_path}",') + res.append('R"__sdf_literal__(') + res.append(f'{arg_file_content}') + res.append(')__sdf_literal__"') + res.append('},') + res.append('') + return NEWLINE.join(res) + + +def get_map_content(arg_pathlist: Path) -> str: + """ + Generates a string pair with the folder and filename as well as the content + of the file in ascending order + + :param arg_pathlist: Foldername and all filenames inside it + :returns: mapping pairs for the std::map + """ + map_str = '' + files = [] + for path in arg_pathlist: + # dir separator is hardcoded to '/' in C++ mapping + posix_path = PurePosixPath(path) + files.append(str(posix_path)) + # get ascending order + files.sort() + for file in files: + with Path(file).open() as f: + content = f.read() + map_str += embed_sdf_content(file, content) + return map_str + + +def get_file_header_epilog() -> str: + """ + Provides the return statement and the closing brackets of the C++ file + + :returns: epilog of the C++ file + """ + res = inspect.cleandoc(""" + }; + + return result; + } + + } + } // namespace sdf + + """) + return NEWLINE + res + + +if __name__ == "__main__": + copyright = get_copyright_notice() + prolog = get_file_header_prolog() + + map_str = "" + for sdf_version in SUPPORTED_SDF_VERSIONS: + pathlist = Path(sdf_version).glob('*.sdf') + map_str += get_map_content(pathlist) + + for sdf_conversion in SUPPORTED_SDF_CONVERSIONS: + pathlist = Path(sdf_conversion).glob('*.convert') + map_str += get_map_content(pathlist) + + # remove the last comma + map_str = map_str[:-2] + + epilog = get_file_header_epilog() + + output = copyright + prolog + map_str + epilog + + # output to stdin so that CMake can read it and create the appropriate file + print(output) diff --git a/src/SDF_TEST.cc b/src/SDF_TEST.cc index ca14549df..315343e12 100644 --- a/src/SDF_TEST.cc +++ b/src/SDF_TEST.cc @@ -21,6 +21,9 @@ #include #include +#include "test_config.hh" +#include "test_utils.hh" + #include "sdf/sdf.hh" class SDFUpdateFixture @@ -559,6 +562,59 @@ TEST(SDF, Version) EXPECT_STREQ(SDF_VERSION, sdf::SDF::Version().c_str()); } +///////////////////////////////////////////////// +TEST(SDF, EmbeddedSpec) +{ + std::string result; + + result = sdf::SDF::EmbeddedSpec("actor.sdf", false); + EXPECT_NE(result.find(""), std::string::npos); + EXPECT_NE(result.find(""), + std::string::npos); + result = sdf::SDF::EmbeddedSpec("actor.sdf", true); + EXPECT_NE(result.find(""), std::string::npos); + EXPECT_NE(result.find(""), + std::string::npos); + + result = sdf::SDF::EmbeddedSpec("root.sdf", false); + EXPECT_NE(result.find("SDFormat base element"), std::string::npos); + EXPECT_NE(result.find("name=\"version\" type=\"string\""), std::string::npos); + result = sdf::SDF::EmbeddedSpec("root.sdf", true); + EXPECT_NE(result.find("SDFormat base element"), std::string::npos); + EXPECT_NE(result.find("name=\"version\" type=\"string\""), std::string::npos); +} + +TEST(SDF, EmbeddedSpecNonExistent) +{ + std::string result; + + // Capture sdferr output + std::stringstream stderr_buffer; + sdf::testing::RedirectConsoleStream redir( + sdf::Console::Instance()->GetMsgStream(), &stderr_buffer); +#ifdef _WIN32 + sdf::Console::Instance()->SetQuiet(false); + sdf::testing::ScopeExit revertSetQuiet( + [] + { + sdf::Console::Instance()->SetQuiet(true); + }); +#endif + + result = sdf::SDF::EmbeddedSpec("unavailable.sdf", false); + EXPECT_NE(stderr_buffer.str().find("Unable to find SDF filename"), + std::string::npos); + EXPECT_NE(stderr_buffer.str().find("with version"), std::string::npos); + EXPECT_TRUE(result.empty()); + + // clear the contents of the buffer + stderr_buffer.str(""); + + result = sdf::SDF::EmbeddedSpec("unavailable.sdf", true); + EXPECT_TRUE(stderr_buffer.str().empty()); + EXPECT_TRUE(result.empty()); +} + ///////////////////////////////////////////////// TEST(SDF, FilePath) { From 09d69f4d341d7e6d59ec2a99c2bb7f4aebcf7832 Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Thu, 16 Feb 2023 11:11:19 -0600 Subject: [PATCH 2/3] Improvements to embedSdf script Signed-off-by: Michael Carroll --- sdf/CMakeLists.txt | 25 +++++-- sdf/embedSdf.py | 168 +++++++++++++++++++++++++++++---------------- 2 files changed, 130 insertions(+), 63 deletions(-) diff --git a/sdf/CMakeLists.txt b/sdf/CMakeLists.txt index e4b6a78e0..6e6fbe7f8 100644 --- a/sdf/CMakeLists.txt +++ b/sdf/CMakeLists.txt @@ -12,13 +12,28 @@ add_subdirectory(1.10) add_custom_target(schema) add_dependencies(schema schema1_10) +# Optionally use the python script (default in sdf14+) +if (EMBEDSDF_PY) + if (NOT Python3_Interpreter_FOUND) + gz_build_error("Python is required to generate the C++ file with the SDF content") + endif() + # Generate the EmbeddedSdf.cc file, which contains all the supported SDF # descriptions in a map of strings. The parser.cc file uses EmbeddedSdf.hh. -execute_process( - COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/sdf/embedSdf.rb - WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/sdf" - OUTPUT_FILE "${PROJECT_BINARY_DIR}/src/EmbeddedSdf.cc" -) + execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/sdf/embedSdf.py + --output-file "${PROJECT_BINARY_DIR}/src/EmbeddedSdf.cc" + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/sdf" + OUTPUT_FILE "${PROJECT_BINARY_DIR}/src/EmbeddedSdf.cc" + ) +else() + execute_process( + COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/sdf/embedSdf.py + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/sdf" + OUTPUT_FILE "${PROJECT_BINARY_DIR}/src/EmbeddedSdf.cc" + ) +endif() + # Generate aggregated SDF description files for use by the sdformat.org # website. If the description files change, the generated full*.sdf files need diff --git a/sdf/embedSdf.py b/sdf/embedSdf.py index b89607fa6..292958712 100644 --- a/sdf/embedSdf.py +++ b/sdf/embedSdf.py @@ -1,29 +1,49 @@ #!/usr/bin/env python3 +""" +Script for generating a C++ file that contains the content from all SDF files +""" + +from typing import List, Optional + +import argparse import inspect -from pathlib import Path, PurePosixPath +import sys +from pathlib import Path -""""Script for generating a C++ file that contains the content from all SDF files""" # The list of supported SDF specification versions. This will let us drop # versions without removing the directories. -SUPPORTED_SDF_VERSIONS = ['1.10', '1.9', '1.8', '1.7', '1.6', '1.5', '1.4', '1.3', '1.2'] +SUPPORTED_SDF_VERSIONS = [ + "1.10", + "1.9", + "1.8", + "1.7", + "1.6", + "1.5", + "1.4", + "1.3", + "1.2", +] # The list of supported SDF conversions. This list includes versions that # a user can convert an existing SDF version to. -SUPPORTED_SDF_CONVERSIONS = ['1.10', '1.9', '1.8', '1.7', '1.6', '1.5', '1.4', '1.3'] +SUPPORTED_SDF_CONVERSIONS = ["1.10", "1.9", "1.8", "1.7", "1.6", "1.5", "1.4", "1.3"] # whitespace indentation for C++ code -INDENTATION = ' ' +INDENTATION = " " + # newline character -NEWLINE = '\n' +NEWLINE = "\n" + def get_copyright_notice() -> str: """ - Provides the copyrigt notice for the C++ file + Provides the copyright notice for the C++ file :returns: copyright notice """ - res = inspect.cleandoc(""" + res = inspect.cleandoc( + """ /* * Copyright 2022 Open Source Robotics Foundation * @@ -40,8 +60,9 @@ def get_copyright_notice() -> str: * limitations under the License. * */ - """) - return res + 2*NEWLINE + """ + ) + return res + 2 * NEWLINE def get_file_header_prolog() -> str: @@ -50,7 +71,8 @@ def get_file_header_prolog() -> str: :returns: prolog of the C++ file """ - res = inspect.cleandoc(""" + res = inspect.cleandoc( + """ #include "EmbeddedSdf.hh" namespace sdf @@ -61,60 +83,39 @@ def get_file_header_prolog() -> str: const std::map &GetEmbeddedSdf() { static const std::map result { - """) + """ + ) return res + NEWLINE def embed_sdf_content(arg_path: str, arg_file_content: str) -> str: """ - Generates a string pair with the folder and filename as well as the content of the file + Generates a string pair with the folder and filename + as well as the content of the file :param arg_path: Foldername and filename of the SDF :param arg_file_content: Content of the provided file :returns: raw string literal mapping pair for the std::map """ res = [] - res.append('// NOLINT') - res.append('{') + res.append("// NOLINT") + res.append("{") res.append(f'"{arg_path}",') res.append('R"__sdf_literal__(') - res.append(f'{arg_file_content}') + res.append(f"{arg_file_content}") res.append(')__sdf_literal__"') - res.append('},') - res.append('') + res.append("}") return NEWLINE.join(res) -def get_map_content(arg_pathlist: Path) -> str: - """ - Generates a string pair with the folder and filename as well as the content - of the file in ascending order - - :param arg_pathlist: Foldername and all filenames inside it - :returns: mapping pairs for the std::map - """ - map_str = '' - files = [] - for path in arg_pathlist: - # dir separator is hardcoded to '/' in C++ mapping - posix_path = PurePosixPath(path) - files.append(str(posix_path)) - # get ascending order - files.sort() - for file in files: - with Path(file).open() as f: - content = f.read() - map_str += embed_sdf_content(file, content) - return map_str - - def get_file_header_epilog() -> str: """ Provides the return statement and the closing brackets of the C++ file :returns: epilog of the C++ file """ - res = inspect.cleandoc(""" + res = inspect.cleandoc( + """ }; return result; @@ -123,29 +124,80 @@ def get_file_header_epilog() -> str: } } // namespace sdf - """) + """ + ) return NEWLINE + res -if __name__ == "__main__": - copyright = get_copyright_notice() +def write_output(file_content: str, output_filename: str) -> None: + """ + Print the content of the EmbeddedSdf.cc to a file + """ + copyright_notice = get_copyright_notice() prolog = get_file_header_prolog() + epilog = get_file_header_epilog() + output_content = copyright_notice + prolog + file_content + epilog - map_str = "" - for sdf_version in SUPPORTED_SDF_VERSIONS: - pathlist = Path(sdf_version).glob('*.sdf') - map_str += get_map_content(pathlist) + with open(output_filename, "w", encoding="utf8") as output_file: + output_file.write(output_content) - for sdf_conversion in SUPPORTED_SDF_CONVERSIONS: - pathlist = Path(sdf_conversion).glob('*.convert') - map_str += get_map_content(pathlist) - # remove the last comma - map_str = map_str[:-2] +def collect_file_locations() -> List[Path]: + paths: List[Path] = [] - epilog = get_file_header_epilog() + for sdf_version in SUPPORTED_SDF_VERSIONS: + paths.extend(Path(sdf_version).glob("*.sdf")) + for sdf_conversion in SUPPORTED_SDF_CONVERSIONS: + paths.extend(Path(sdf_conversion).glob("*.convert")) + return paths + + +def generate_map_content(paths: List[Path], relative_to: Optional[str] = None) -> str: + ''' + Generate the EmbeddedSdf.cc content + ''' + content = [] + for path in paths: + with open(path, "r", encoding="utf8") as input_sdf: + file_content = input_sdf.read() + + # Strip relative path if requested + if relative_to is not None: + path = path.relative_to(relative_to) + content.append(embed_sdf_content(str(path), file_content)) + return ",".join(content) + + +def main(args=None) -> int: + ''' + Main entrypoint + ''' + if args is None: + args = sys.argv[1:] + + parser = argparse.ArgumentParser() + + parser.add_argument( + "--sdf-root", + default=None, + help="Directory containing sdf description files for each version", + ) + parser.add_argument( + "--input-files", nargs="*", help="List of input files to be embedded" + ) + parser.add_argument( + "--output-file", help="File to output embeddedsdf.cc content to" + ) + + args = parser.parse_args(args) + if not args.input_files or len(args.input_files) == 0: + paths = collect_file_locations() + else: + paths = [Path(f) for f in args.input_files] + content = generate_map_content(paths, args.sdf_root) + write_output(content, args.output_file) + return 0 - output = copyright + prolog + map_str + epilog - # output to stdin so that CMake can read it and create the appropriate file - print(output) +if __name__ == "__main__": + sys.exit(main()) From a82e65f562eb58985715115a2865eb6e1185c2e5 Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Thu, 16 Feb 2023 12:58:16 -0600 Subject: [PATCH 3/3] Update sdf/CMakeLists.txt Signed-off-by: Michael Carroll Co-authored-by: Addisu Z. Taddese --- sdf/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdf/CMakeLists.txt b/sdf/CMakeLists.txt index 6e6fbe7f8..983e7804b 100644 --- a/sdf/CMakeLists.txt +++ b/sdf/CMakeLists.txt @@ -28,7 +28,7 @@ if (EMBEDSDF_PY) ) else() execute_process( - COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/sdf/embedSdf.py + COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/sdf/embedSdf.rb WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/sdf" OUTPUT_FILE "${PROJECT_BINARY_DIR}/src/EmbeddedSdf.cc" )