From 9c078924c2e95fef785f17b238e15dd04a9b718f Mon Sep 17 00:00:00 2001 From: mikee47 Date: Mon, 8 Jul 2024 20:26:11 +0100 Subject: [PATCH] Improve clean-tools.py - Simplify - Use defined environment paths, very confusing otherwise - Avoid trying to clean inside .git - fails anyway, but not the intent - Found some more stuff to get rid of - Further testing and fixes for Windows --- Tools/ci/clean-tools.py | 169 +++++++++++++++++++++++----------------- 1 file changed, 96 insertions(+), 73 deletions(-) diff --git a/Tools/ci/clean-tools.py b/Tools/ci/clean-tools.py index 1654b37b50..75762ca914 100644 --- a/Tools/ci/clean-tools.py +++ b/Tools/ci/clean-tools.py @@ -5,53 +5,75 @@ import shutil import argparse +# Mandatory environment variables +IDF_PATH = os.environ['IDF_PATH'] +IDF_TOOLS_PATH = os.environ['IDF_TOOLS_PATH'] + # The commented-out ARC paths below are for regular libraries with RTTI. # These are used by default without the -fno-rtti switch so without them the cmake # compiler check fails. Otherwise they can go. TOOLS = r'esp32/tools/' IDF = r'esp-idf([^/]+)/' + ARC = r'([^/]+)\.a' -FILTERS = [ - # Leave versioned directory to avoid re-installation - rf'{TOOLS}.*esp-elf-gdb/.*/.*esp-elf-gdb/', - rf'{TOOLS}esp32ulp-elf/.*/esp32ulp-elf/', - rf'{TOOLS}openocd-esp32/.*/openocd-esp32/', - # Libraries not required by Sming - # rf'{TOOLS}.*/riscv32-esp-elf/lib/{ARC}', - rf'{TOOLS}.*/riscv32-esp-elf/lib/rv32i_.*', - rf'{TOOLS}.*/riscv32-esp-elf/lib/rv32i/', - rf'{TOOLS}.*/riscv32-esp-elf/lib/rv32imac_.*', - rf'{TOOLS}.*/riscv32-esp-elf/lib/rv32imafc_.*', - rf'{TOOLS}.*/riscv32-esp-elf/lib/rv32imafc/', - # rf'{TOOLS}.*/riscv32-esp-elf/lib/rv32imc_zicsr_zifencei/ilp32/{ARC}', - # rf'{TOOLS}.*/riscv32-esp-elf/lib/rv32imc/ilp32/{ARC}', - # rf'{TOOLS}.*/lib/esp32/{ARC}', - rf'{TOOLS}.*/lib/esp32-psram', - rf'{TOOLS}.*/lib/esp32/psram', - # rf'{TOOLS}.*/lib/esp32s2/{ARC}', - # rf'{TOOLS}.*/lib/esp32s3/{ARC}', - # rf'{TOOLS}.*/xtensa-esp32-elf/lib/{ARC}', - # rf'{TOOLS}.*/xtensa-esp32s2-elf/lib/{ARC}', - # rf'{TOOLS}.*/xtensa-esp32s3-elf/lib/{ARC}', - # rf'{TOOLS}.*/xtensa-esp-elf/lib/{ARC}', - # Components, examples and archives - rf'{IDF}docs/', - rf'{IDF}examples/', - rf'{IDF}components/cmock/', - rf'{IDF}components/unity/', - rf'{IDF}.*esp32c6.*', - rf'{IDF}.*esp32h2.*', - rf'{IDF}.*esp32p4.*', - rf'{IDF}.*/tests', -] - -# Python 3.8 doesn't have str.removeprefix -def removeprefix(path: str, prefix: str) -> str: - return path[len(prefix):] if path.startswith(prefix) else path + +# These filters are matched from the **start** of the path so there's an implicit .* at the end +FILTERS = { + IDF_TOOLS_PATH: [ + # Leave versioned directory to avoid re-installation + r'.*esp-elf-gdb/.*/.*esp-elf-gdb/', + r'esp32ulp-elf/.*/esp32ulp-elf/', + r'openocd-esp32/.*/openocd-esp32/', + # Libraries not required by Sming + # r'.*/riscv32-esp-elf/lib/{ARC}', + r'.*/riscv32-esp-elf/lib/rv32i_.*', + r'.*/riscv32-esp-elf/lib/rv32i/', + r'.*/riscv32-esp-elf/lib/rv32imac_.*', + r'.*/riscv32-esp-elf/lib/rv32imafc_.*', + r'.*/riscv32-esp-elf/lib/rv32imafc/', + # r'.*/riscv32-esp-elf/lib/rv32imc_zicsr_zifencei/ilp32/{ARC}', + # r'.*/riscv32-esp-elf/lib/rv32imc/ilp32/{ARC}', + # r'.*/lib/esp32/{ARC}', + r'.*/lib/esp32(-|/)psram', + # r'.*/lib/esp32s2/{ARC}', + # r'.*/lib/esp32s3/{ARC}', + # r'.*/xtensa-esp32-elf/lib/{ARC}', + # r'.*/xtensa-esp32s2-elf/lib/{ARC}', + # r'.*/xtensa-esp32s3-elf/lib/{ARC}', + # r'.*/xtensa-esp-elf/lib/{ARC}', + ], + IDF_PATH: [ + # Components, examples and archives + r'docs/', + r'.*/doc', + r'examples/', + r'components/asio/', + r'components/cmock/', + r'components/openthread/openthread/third_party', + r'components/unity/', + r'components/.*esp32c6.*', + r'components/.*esp32h2.*', + r'components/.*esp32p4.*', + r'components/.*/test', + r'components/.*/fuzz', + r'components/expat/expat/testdata', + r'components/libsodium', + r'components/nghttp/nghttp2/third-party/mruby', + r'components/nghttp/nghttp2/third-party', + r'components/tinyusb/', + r'components/.*/win32', + r'tools/esp_app_trace', + r'tools/test', + r'tools/ci', + ] +} + +def fix_path(path: str) -> str: + return path[2:].replace('\\', '/') if path[1] == ':' else path def scan_log(logfile: str, file_list: dict): - with open(logfile, 'r') as f: + with open(logfile, 'r', encoding='utf8') as f: for path in f: path = path.strip() try: @@ -59,10 +81,8 @@ def scan_log(logfile: str, file_list: dict): path, _, size = path.rpartition(' ') blocks = int(blocks) size = int(size) - if path[1] == ':': - path = path[2:].replace('\\', '/') - path = removeprefix(path, '/opt/') - if not path.startswith('esp'): + path = fix_path(path) + if '/esp' not in path: continue existing = file_list.get(path) if not existing or size > existing: @@ -80,20 +100,15 @@ def scan_logs(log_dir) -> dict: return file_list -def scan_tools_dir(tools_dir) -> dict: - file_list = {} +def scan_tree(start_path: str, file_list: dict): def scan(root_path): for entry in os.scandir(root_path): if entry.is_dir(): scan(entry.path) continue - path = entry.path - path = removeprefix(path, tools_dir + '/') - if not path.startswith('esp'): - continue - path = path.replace('\\', '/') + path = fix_path(entry.path) file_list[path] = entry.stat().st_size - scan(tools_dir) + scan(start_path) return file_list @@ -106,25 +121,33 @@ def mbstr(size) -> str: print(f'{len(file_list)} files, total size {mbstr(total_size)}') - total_size = 0 - for flt in FILTERS: - expr = re.compile(flt) - size = sum(size for path, size in file_list.items() if expr.match(path)) - total_size += size - print(f'{flt}: {mbstr(size)}') + def match(start_path: str, filters: list): + start_path = fix_path(start_path) + matched_size = 0 + for flt in filters: + flt = rf'{start_path}/{flt}' + expr = re.compile(flt, flags=re.IGNORECASE) + size = sum(size for path, size in file_list.items() if expr.match(path)) + matched_size += size + print(f'{flt}: {mbstr(size)}') + return matched_size + + total_size = sum(match(*item) for item in FILTERS.items()) print(f'Total size {mbstr(total_size)}') -def clean_tools_dir(tools_dir: str, do_clean: bool): - re_filter = re.compile('|'.join(FILTERS)) +def clean_tree(start_path: str, filters: list, do_clean: bool): + if not os.path.exists(start_path): + print(f'"{start_path}" not found, skipping.') + return + + re_filter = re.compile('|'.join(rf'{fix_path(start_path)}/{f}' for f in filters), flags=re.IGNORECASE) def clean_path(root_path): for entry in os.scandir(root_path): try: - path = entry.path - path = path.replace('\\', '/') - path = removeprefix(path, tools_dir.replace('\\', '/') + '/') + path = fix_path(entry.path) if entry.is_dir(): if re_filter.match(path + '/'): print(f"rmtree {entry.path}") @@ -139,7 +162,7 @@ def clean_path(root_path): except Exception as e: print(f'{repr(e)}') - clean_path(tools_dir) + clean_path(start_path) def main(): @@ -150,25 +173,25 @@ def main(): args = parser.parse_args() - tools_dir = os.path.dirname(os.environ['IDF_PATH']) - - print(f'Action: {args.action} "{tools_dir}"') + print(f'Action: {args.action}, IDF_PATH="{IDF_PATH}", IDF_TOOLS_PATH="{IDF_TOOLS_PATH}"') if args.action == 'scan': if args.logdir: file_list = scan_logs(args.logdir) else: - file_list = scan_tools_dir(tools_dir) + file_list = {} + scan_tree(IDF_PATH, file_list) + scan_tree(IDF_TOOLS_PATH, file_list) scan_list(file_list) - elif args.action == 'clean': - if not os.path.exists(tools_dir): - print(f'"{tools_dir}" not found, skipping.') - return - clean_tools_dir(tools_dir, args.delete) + return + + if args.action == 'clean': + for path, filters in FILTERS.items(): + clean_tree(path, filters, args.delete) if args.delete: - print("OK, items cleaned.") + print("** Cleaning finished.") else: - print("Dry run, nothing deleted.") + print("** Dry run, nothing deleted.") if __name__ == '__main__': main()