lmdk: python tools for building and headers pack

To build modules using pythons were created several tools making
building easier.
For future deployment there is a tool extracting headers descibed in
manifest and creating zip file from them.

modules are build now:
	python scripts/lmdk/build_module.py -l <modules_names>

we can build multiple modules by adding them one after another.

Signed-off-by: Dobrowolski, PawelX <[email protected]>

scripts/lmdk/build_module.py

@@ -0,0 +1,100 @@
+import argparse
+import shlex
+import subprocess
+import pathlib
+import errno
+import platform as py_platform
+import sys
+import shutil
+import os
+import warnings
+import fnmatch
+import hashlib
+import json
+import gzip
+import dataclasses
+import concurrent.futures as concurrent
+from tools import cmake_build, header_pack
+
+# anytree module is defined in Zephyr build requirements
+from anytree import AnyNode, RenderTree, render
+
+# https://chrisyeh96.github.io/2017/08/08/definitive-guide-python-imports.html#case-3-importing-from-parent-directory
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
+
+MIN_PYTHON_VERSION = 3, 8
+assert sys.version_info >= MIN_PYTHON_VERSION, \
+	f"Python {MIN_PYTHON_VERSION} or above is required."
+
+# Constant value resolves SOF_TOP directory as: "this script directory/.."
+SOF_TOP = pathlib.Path(__file__).parents[1].resolve()
+west_top = pathlib.Path(SOF_TOP, "../..").resolve()
+LMDK_BUILD_DIR = west_top / "sof" / "lmdk"
+RIMAGE_BUILD_DIR = west_top / "build-rimage"
+
+if py_platform.system() == "Windows":
+	xtensa_tools_version_postfix = "-win32"
+elif py_platform.system() == "Linux":
+	xtensa_tools_version_postfix = "-linux"
+else:
+	xtensa_tools_version_postfix = "-unsupportedOS"
+	warnings.warn(f"Your operating system: {py_platform.system()} is not supported")
+
+
+class stores_libraries_arguments(argparse.Action):
+	"""Stores libraries arguments whether provided module name is supported."""
+	def __call__(self, parser, namespace, values, option_string=None):
+		setattr(namespace, "libraries", values)
+
+args = None
+def parse_args():
+	global args
+	global west_top
+	parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
+			epilog=("This script supports XtensaTools but only when installed in a specific\n" +
+				"directory structure, example:\n" +
+				"myXtensa/\n" +
+				"└── install/\n" +
+				"       ├── builds/\n" +
+				"       │   ├── RD-2012.5{}/\n".format(xtensa_tools_version_postfix) +
+				"       │   │   └── Intel_HiFiEP/\n" +
+				"       │   └── RG-2017.8{}/\n".format(xtensa_tools_version_postfix) +
+				"       │       ├── LX4_langwell_audio_17_8/\n" +
+				"       │       └── X4H3I16w2D48w3a_2017_8/\n" +
+				"       └── tools/\n" +
+				"               ├── RD-2012.5{}/\n".format(xtensa_tools_version_postfix) +
+				"               │   └── XtensaTools/\n" +
+				"               └── RG-2017.8{}/\n".format(xtensa_tools_version_postfix) +
+				"                   └── XtensaTools/\n" +
+			"$XTENSA_TOOLS_ROOT=/path/to/myXtensa ...\n"), add_help=False)
+
+	parser.add_argument("-k", "--key", type=pathlib.Path, required=False,
+						help="Path to a non-default rimage signing key.")
+
+	parser.add_argument("-l", "--libraries", nargs="*", action=stores_libraries_arguments, default=[],
+						help=""" Function for CMake building modules.
+						We can build more then one module just by adding more module names.""")
+
+	args = parser.parse_args()
+
+	# if args.all:
+	# 	args.platforms = list(platform_configs_all)
+
+	# print help message if no arguments provided
+	if len(sys.argv) == 1: #or args.help:
+		parser.epilog += "\nTo build module you must provide name and key to sign '"
+
+		parser.print_help()
+		sys.exit(0)
+
+def main():
+	parse_args()
+
+	if args.libraries:
+		cmake_build.build_libraries(LMDK_BUILD_DIR, RIMAGE_BUILD_DIR, args)
+		header_pack.create_headers_pack()
+
+
+if __name__ == "__main__":
+	main()
+

scripts/lmdk/tools/cmake_build.py

@@ -0,0 +1,55 @@
+import argparse
+import shlex
+import subprocess
+import pathlib
+import errno
+import platform as py_platform
+import sys
+import shutil
+import os
+import warnings
+import fnmatch
+import hashlib
+import json
+import gzip
+import dataclasses
+import concurrent.futures as concurrent
+from .utils import rmtree_if_exists, execute_command
+# anytree module is defined in Zephyr build requirements
+from anytree import AnyNode, RenderTree, render
+
+
+SOF_TOP = pathlib.Path(__file__).parents[3].resolve()
+
+
+@dataclasses.dataclass
+# pylint:disable=too-many-instance-attributes
+class PlatformConfig:
+    "Product parameters"
+    vendor: str
+    PLAT_CONFIG: str
+    XTENSA_TOOLS_VERSION: str
+    XTENSA_CORE: str
+    DEFAULT_TOOLCHAIN_VARIANT: str = "xt-clang"
+    RIMAGE_KEY: pathlib.Path = pathlib.Path(SOF_TOP, "keys", "otc_private_key_3k.pem")
+    aliases: list = dataclasses.field(default_factory=list)
+    ipc4: bool = False
+
+
+def build_libraries(LMDK_BUILD_DIR, RIMAGE_BUILD_DIR, args):
+    library_dir = LMDK_BUILD_DIR / "libraries"
+    # CMake build rimage module
+    for lib in args.libraries:
+        library_cmake = library_dir / lib / "CMakeLists.txt"
+        if library_cmake.is_file():
+            print(f"\nBuilding loadable module: " + lib)
+            lib_path = pathlib.Path(library_dir, lib, "build")
+            rmtree_if_exists(lib_path)
+            lib_path.mkdir(parents=True, exist_ok=True)
+            rimage_bin = RIMAGE_BUILD_DIR / "rimage.exe"
+            if not rimage_bin.is_file():
+                rimage_bin = RIMAGE_BUILD_DIR / "rimage"
+            execute_command(["cmake", "-B", "build", "-G", "Ninja",
+                             "-DRIMAGE_COMMAND="+str(rimage_bin), "-DSIGNING_KEY="+str(PlatformConfig.RIMAGE_KEY)],
+                             cwd=library_dir/lib)
+            execute_command(["cmake", "--build", "build", "-v"], cwd=library_dir/lib)

scripts/lmdk/tools/header_pack.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass
+from distutils.dir_util import copy_tree
+import json
+import pathlib
+import shutil
+
+
+# Headers for needs of lmdk are defined in
+#            lmdk/include/headers_list.json
+from typing import io
+
+
+SOF_TOP = pathlib.Path(__file__).parents[3].resolve()
+LMDK_HEADERS = SOF_TOP / "lmdk" / "include" / "headers_manifest.json"
+
+
+def create_separate_headers():
+    f = open(LMDK_HEADERS)
+    data = json.load(f)
+    src = ''
+
+    for i in data:
+        src = ''
+        for p in i[:-1]:
+            src += p
+            src += "/"
+        pathlib.Path(SOF_TOP / "lmdk" / "include" / "sof" / src).mkdir(parents=True, exist_ok=True)
+        for p in i[-1:]:
+            src += p
+        shutil.copyfile(SOF_TOP / src, SOF_TOP / "lmdk" /"include" / "sof" / src)
+    f.close()
+
+# -> to do
+# def validate_separate_headers():
+#     return 0
+
+
+def create_headers_pack():
+    create_separate_headers()
+    shutil.make_archive(SOF_TOP / "lmdk" /"include" / "header_pack", "zip", SOF_TOP / "lmdk" /"include" / "sof")
+    shutil.rmtree(SOF_TOP / "lmdk" /"include" / "sof", ignore_errors=True)
+    return 0
+
+

scripts/lmdk/tools/utils.py

@@ -0,0 +1,59 @@
+import argparse
+import shlex
+import subprocess
+import pathlib
+import errno
+import platform as py_platform
+import sys
+import shutil
+import os
+import warnings
+import fnmatch
+import hashlib
+import json
+import gzip
+import dataclasses
+import concurrent.futures as concurrent
+
+from west import configuration as west_config
+
+# anytree module is defined in Zephyr build requirements
+from anytree import AnyNode, RenderTree, render
+from packaging import version
+
+
+def rmtree_if_exists(directory):
+    "This is different from ignore_errors=False because it deletes everything or nothing"
+    if os.path.exists(directory):
+        shutil.rmtree(directory)
+
+
+def execute_command(*run_args, **run_kwargs):
+    """[summary] Provides wrapper for subprocess.run that prints
+    command executed when 'more verbose' verbosity level is set."""
+    command_args = run_args[0]
+
+    # If you really need the shell in some non-portable section then
+    # invoke subprocess.run() directly.
+    if run_kwargs.get('shell') or not isinstance(command_args, list):
+        raise RuntimeError("Do not rely on non-portable shell parsing")
+
+    cwd = run_kwargs.get('cwd')
+    print_cwd = f"In dir: {cwd}" if cwd else f"in current dir: {os.getcwd()}"
+    print_args = shlex.join(command_args)
+    output = f"{print_cwd}; running command:\n    {print_args}"
+    env_arg = run_kwargs.get('env')
+    env_change = set(env_arg.items()) - set(os.environ.items()) if env_arg else None
+    if env_change and run_kwargs.get('sof_log_env'):
+        output += "\n... with extra/modified environment:\n"
+        for k_v in env_change:
+            output += f"{k_v[0]}={k_v[1]}\n"
+    print(output, flush=True)
+
+    run_kwargs = {k: run_kwargs[k] for k in run_kwargs if not k.startswith("sof_")}
+
+    if not 'check' in run_kwargs:
+        run_kwargs['check'] = True
+    #pylint:disable=subprocess-run-check
+
+    return subprocess.run(*run_args, **run_kwargs)