From a98ae2eaaeff30f33a4b13f76af34d4daf55742a Mon Sep 17 00:00:00 2001 From: "Dobrowolski, PawelX" Date: Wed, 14 Feb 2024 16:40:18 +0100 Subject: [PATCH] lmdk: python tools for building and headers pack To build modules using pythons were created several tools making building easier. For future deployment there is a tool extracting headers descibed in manifest and creating zip file from them. modules are build now: python scripts/lmdk/build_module.py -l we can build multiple modules by adding them one after another. Signed-off-by: Dobrowolski, PawelX --- scripts/lmdk/build_module.py | 100 ++++++++++++++++++++++++++++++ scripts/lmdk/tools/__init__.py | 0 scripts/lmdk/tools/cmake_build.py | 55 ++++++++++++++++ scripts/lmdk/tools/header_pack.py | 44 +++++++++++++ scripts/lmdk/tools/parse_args.py | 0 scripts/lmdk/tools/utils.py | 59 ++++++++++++++++++ 6 files changed, 258 insertions(+) create mode 100644 scripts/lmdk/build_module.py create mode 100644 scripts/lmdk/tools/__init__.py create mode 100644 scripts/lmdk/tools/cmake_build.py create mode 100644 scripts/lmdk/tools/header_pack.py create mode 100644 scripts/lmdk/tools/parse_args.py create mode 100644 scripts/lmdk/tools/utils.py diff --git a/scripts/lmdk/build_module.py b/scripts/lmdk/build_module.py new file mode 100644 index 000000000000..f9a11d43aad8 --- /dev/null +++ b/scripts/lmdk/build_module.py @@ -0,0 +1,100 @@ +import argparse +import shlex +import subprocess +import pathlib +import errno +import platform as py_platform +import sys +import shutil +import os +import warnings +import fnmatch +import hashlib +import json +import gzip +import dataclasses +import concurrent.futures as concurrent +from tools import cmake_build, header_pack + +# anytree module is defined in Zephyr build requirements +from anytree import AnyNode, RenderTree, render + +# https://chrisyeh96.github.io/2017/08/08/definitive-guide-python-imports.html#case-3-importing-from-parent-directory +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +MIN_PYTHON_VERSION = 3, 8 +assert sys.version_info >= MIN_PYTHON_VERSION, \ + f"Python {MIN_PYTHON_VERSION} or above is required." + +# Constant value resolves SOF_TOP directory as: "this script directory/.." +SOF_TOP = pathlib.Path(__file__).parents[1].resolve() +west_top = pathlib.Path(SOF_TOP, "../..").resolve() +LMDK_BUILD_DIR = west_top / "sof" / "lmdk" +RIMAGE_BUILD_DIR = west_top / "build-rimage" + +if py_platform.system() == "Windows": + xtensa_tools_version_postfix = "-win32" +elif py_platform.system() == "Linux": + xtensa_tools_version_postfix = "-linux" +else: + xtensa_tools_version_postfix = "-unsupportedOS" + warnings.warn(f"Your operating system: {py_platform.system()} is not supported") + + +class stores_libraries_arguments(argparse.Action): + """Stores libraries arguments whether provided module name is supported.""" + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, "libraries", values) + +args = None +def parse_args(): + global args + global west_top + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, + epilog=("This script supports XtensaTools but only when installed in a specific\n" + + "directory structure, example:\n" + + "myXtensa/\n" + + "└── install/\n" + + " ├── builds/\n" + + " │   ├── RD-2012.5{}/\n".format(xtensa_tools_version_postfix) + + " │   │   └── Intel_HiFiEP/\n" + + " │   └── RG-2017.8{}/\n".format(xtensa_tools_version_postfix) + + " │   ├── LX4_langwell_audio_17_8/\n" + + " │   └── X4H3I16w2D48w3a_2017_8/\n" + + " └── tools/\n" + + " ├── RD-2012.5{}/\n".format(xtensa_tools_version_postfix) + + " │   └── XtensaTools/\n" + + " └── RG-2017.8{}/\n".format(xtensa_tools_version_postfix) + + " └── XtensaTools/\n" + + "$XTENSA_TOOLS_ROOT=/path/to/myXtensa ...\n"), add_help=False) + + parser.add_argument("-k", "--key", type=pathlib.Path, required=False, + help="Path to a non-default rimage signing key.") + + parser.add_argument("-l", "--libraries", nargs="*", action=stores_libraries_arguments, default=[], + help=""" Function for CMake building modules. + We can build more then one module just by adding more module names.""") + + args = parser.parse_args() + + # if args.all: + # args.platforms = list(platform_configs_all) + + # print help message if no arguments provided + if len(sys.argv) == 1: #or args.help: + parser.epilog += "\nTo build module you must provide name and key to sign '" + + parser.print_help() + sys.exit(0) + +def main(): + parse_args() + + if args.libraries: + cmake_build.build_libraries(LMDK_BUILD_DIR, RIMAGE_BUILD_DIR, args) + header_pack.create_headers_pack() + + +if __name__ == "__main__": + main() + diff --git a/scripts/lmdk/tools/__init__.py b/scripts/lmdk/tools/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/lmdk/tools/cmake_build.py b/scripts/lmdk/tools/cmake_build.py new file mode 100644 index 000000000000..866397be0f0b --- /dev/null +++ b/scripts/lmdk/tools/cmake_build.py @@ -0,0 +1,55 @@ +import argparse +import shlex +import subprocess +import pathlib +import errno +import platform as py_platform +import sys +import shutil +import os +import warnings +import fnmatch +import hashlib +import json +import gzip +import dataclasses +import concurrent.futures as concurrent +from .utils import rmtree_if_exists, execute_command +# anytree module is defined in Zephyr build requirements +from anytree import AnyNode, RenderTree, render + + +SOF_TOP = pathlib.Path(__file__).parents[3].resolve() + + +@dataclasses.dataclass +# pylint:disable=too-many-instance-attributes +class PlatformConfig: + "Product parameters" + vendor: str + PLAT_CONFIG: str + XTENSA_TOOLS_VERSION: str + XTENSA_CORE: str + DEFAULT_TOOLCHAIN_VARIANT: str = "xt-clang" + RIMAGE_KEY: pathlib.Path = pathlib.Path(SOF_TOP, "keys", "otc_private_key_3k.pem") + aliases: list = dataclasses.field(default_factory=list) + ipc4: bool = False + + +def build_libraries(LMDK_BUILD_DIR, RIMAGE_BUILD_DIR, args): + library_dir = LMDK_BUILD_DIR / "libraries" + # CMake build rimage module + for lib in args.libraries: + library_cmake = library_dir / lib / "CMakeLists.txt" + if library_cmake.is_file(): + print(f"\nBuilding loadable module: " + lib) + lib_path = pathlib.Path(library_dir, lib, "build") + rmtree_if_exists(lib_path) + lib_path.mkdir(parents=True, exist_ok=True) + rimage_bin = RIMAGE_BUILD_DIR / "rimage.exe" + if not rimage_bin.is_file(): + rimage_bin = RIMAGE_BUILD_DIR / "rimage" + execute_command(["cmake", "-B", "build", "-G", "Ninja", + "-DRIMAGE_COMMAND="+str(rimage_bin), "-DSIGNING_KEY="+str(PlatformConfig.RIMAGE_KEY)], + cwd=library_dir/lib) + execute_command(["cmake", "--build", "build", "-v"], cwd=library_dir/lib) \ No newline at end of file diff --git a/scripts/lmdk/tools/header_pack.py b/scripts/lmdk/tools/header_pack.py new file mode 100644 index 000000000000..3ec5986371d9 --- /dev/null +++ b/scripts/lmdk/tools/header_pack.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass +from distutils.dir_util import copy_tree +import json +import pathlib +import shutil + + +# Headers for needs of lmdk are defined in +# lmdk/include/headers_list.json +from typing import io + + +SOF_TOP = pathlib.Path(__file__).parents[3].resolve() +LMDK_HEADERS = SOF_TOP / "lmdk" / "include" / "headers_manifest.json" + + +def create_separate_headers(): + f = open(LMDK_HEADERS) + data = json.load(f) + src = '' + + for i in data: + src = '' + for p in i[:-1]: + src += p + src += "/" + pathlib.Path(SOF_TOP / "lmdk" / "include" / "sof" / src).mkdir(parents=True, exist_ok=True) + for p in i[-1:]: + src += p + shutil.copyfile(SOF_TOP / src, SOF_TOP / "lmdk" /"include" / "sof" / src) + f.close() + +# -> to do +# def validate_separate_headers(): +# return 0 + + +def create_headers_pack(): + create_separate_headers() + shutil.make_archive(SOF_TOP / "lmdk" /"include" / "header_pack", "zip", SOF_TOP / "lmdk" /"include" / "sof") + shutil.rmtree(SOF_TOP / "lmdk" /"include" / "sof", ignore_errors=True) + return 0 + + diff --git a/scripts/lmdk/tools/parse_args.py b/scripts/lmdk/tools/parse_args.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/lmdk/tools/utils.py b/scripts/lmdk/tools/utils.py new file mode 100644 index 000000000000..85ca6a9d40ab --- /dev/null +++ b/scripts/lmdk/tools/utils.py @@ -0,0 +1,59 @@ +import argparse +import shlex +import subprocess +import pathlib +import errno +import platform as py_platform +import sys +import shutil +import os +import warnings +import fnmatch +import hashlib +import json +import gzip +import dataclasses +import concurrent.futures as concurrent + +from west import configuration as west_config + +# anytree module is defined in Zephyr build requirements +from anytree import AnyNode, RenderTree, render +from packaging import version + + +def rmtree_if_exists(directory): + "This is different from ignore_errors=False because it deletes everything or nothing" + if os.path.exists(directory): + shutil.rmtree(directory) + + +def execute_command(*run_args, **run_kwargs): + """[summary] Provides wrapper for subprocess.run that prints + command executed when 'more verbose' verbosity level is set.""" + command_args = run_args[0] + + # If you really need the shell in some non-portable section then + # invoke subprocess.run() directly. + if run_kwargs.get('shell') or not isinstance(command_args, list): + raise RuntimeError("Do not rely on non-portable shell parsing") + + cwd = run_kwargs.get('cwd') + print_cwd = f"In dir: {cwd}" if cwd else f"in current dir: {os.getcwd()}" + print_args = shlex.join(command_args) + output = f"{print_cwd}; running command:\n {print_args}" + env_arg = run_kwargs.get('env') + env_change = set(env_arg.items()) - set(os.environ.items()) if env_arg else None + if env_change and run_kwargs.get('sof_log_env'): + output += "\n... with extra/modified environment:\n" + for k_v in env_change: + output += f"{k_v[0]}={k_v[1]}\n" + print(output, flush=True) + + run_kwargs = {k: run_kwargs[k] for k in run_kwargs if not k.startswith("sof_")} + + if not 'check' in run_kwargs: + run_kwargs['check'] = True + #pylint:disable=subprocess-run-check + + return subprocess.run(*run_args, **run_kwargs) \ No newline at end of file