diff --git a/applications/keyword_detection/CMakeLists.txt b/applications/keyword_detection/CMakeLists.txt index cae79214..b493ec23 100644 --- a/applications/keyword_detection/CMakeLists.txt +++ b/applications/keyword_detection/CMakeLists.txt @@ -32,6 +32,7 @@ else() endif() set(ML_USE_CASE "kws") set(ML_MODEL "GenerateKWSModel") +set(ML_USE_CASE_RESOURCES_FILE "${CMAKE_CURRENT_LIST_DIR}/resources/use_case_resources.json") set(TFM_PLATFORM_UPGRADE_STRATEGY "SWAP_USING_SCRATCH") set(TFM_PLATFORM_CONFIRM_IMAGE ON) diff --git a/applications/keyword_detection/resources/use_case_resources.json b/applications/keyword_detection/resources/use_case_resources.json new file mode 100644 index 00000000..584d649a --- /dev/null +++ b/applications/keyword_detection/resources/use_case_resources.json @@ -0,0 +1,16 @@ +[ + { + "name": "kws", + "url_prefix": [ + "https://github.com/ARM-software/ML-zoo/raw/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8/" + ], + "resources": [ + {"name": "ifm0.npy", "url": "{url_prefix:0}testing_input/input/0.npy"}, + {"name": "ofm0.npy", "url": "{url_prefix:0}testing_output/Identity/0.npy"}, + { + "name": "kws_micronet_m.tflite", + "url": "{url_prefix:0}kws_micronet_m.tflite" + } + ] + } +] diff --git a/applications/speech_recognition/CMakeLists.txt b/applications/speech_recognition/CMakeLists.txt index f5d0aabb..89468209 100644 --- a/applications/speech_recognition/CMakeLists.txt +++ b/applications/speech_recognition/CMakeLists.txt @@ -32,6 +32,7 @@ else() endif() set(ML_USE_CASE "asr") set(ML_MODEL "GenerateASRModel") +set(ML_USE_CASE_RESOURCES_FILE "${CMAKE_CURRENT_LIST_DIR}/resources/use_case_resources.json") set(TFM_PLATFORM_UPGRADE_STRATEGY "SWAP_USING_SCRATCH") set(TFM_PLATFORM_CONFIRM_IMAGE ON) diff --git a/applications/speech_recognition/resources/use_case_resources.json b/applications/speech_recognition/resources/use_case_resources.json new file mode 100644 index 00000000..593d129f --- /dev/null +++ b/applications/speech_recognition/resources/use_case_resources.json @@ -0,0 +1,22 @@ +[ + { + "name": "asr", + "url_prefix": [ + "https://github.com/ARM-software/ML-zoo/raw/eb2170aac1317f00b128ab82bdb159cfcca36ea6/models/speech_recognition/tiny_wav2letter/tflite_pruned_int8/" + ], + "resources": [ + { + "name": "tiny_wav2letter_pruned_int8.tflite", + "url": "{url_prefix:0}tiny_wav2letter_pruned_int8.tflite" + }, + { + "name": "ifm0.npy", + "url": "{url_prefix:0}testing_input/input_1_int8/0.npy" + }, + { + "name": "ofm0.npy", + "url": "{url_prefix:0}testing_output/Identity_int8/0.npy" + } + ] + } +] diff --git a/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt b/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt index 84962634..7d0da291 100644 --- a/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt +++ b/components/ai/ml_embedded_evaluation_kit/CMakeLists.txt @@ -12,7 +12,7 @@ include(ApplyPatches) set(PATCH_FILES_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/integration/patches") set(PATCH_FILES - "${PATCH_FILES_DIRECTORY}/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch" + "${PATCH_FILES_DIRECTORY}/0001-use-case-resources-Enabled-user-provided-metadata.patch" "${PATCH_FILES_DIRECTORY}/0002-Use-CMSIS_device_header-instead-of-RTE_Components.h.patch" "${PATCH_FILES_DIRECTORY}/0003-Make-ETHOSU_ARCH-configurable-in-TensorFlow-CMake.patch" "${PATCH_FILES_DIRECTORY}/0004-Move-activation_buf_dram-to-.bss.NoInit-region.patch" diff --git a/components/ai/ml_embedded_evaluation_kit/integration/README.md b/components/ai/ml_embedded_evaluation_kit/integration/README.md index d1c5f2cf..4db63d7a 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/README.md +++ b/components/ai/ml_embedded_evaluation_kit/integration/README.md @@ -27,6 +27,9 @@ necessary ML libraries and running Vela compiler on the default models for the s In order for your application to access the API headers for the libraries, you need to link to the two resulting static libraries: `${ML_USE_CASE}_api`, `${ML_USE_CASE}_model`. These libraries provide the necessary include paths to the respective APIs from the ML Embedded Evaluation Kit. +In addition, you need to provide a JSON file containing the metadata for the model to use. +This is provided to the build system by setting the path to the JSON file with the +`ML_USE_CASE_RESOURCES_FILE` CMake variable. ## Documentation diff --git a/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake b/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake index 0a953cd7..8fad4fb3 100644 --- a/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake +++ b/components/ai/ml_embedded_evaluation_kit/integration/cmake/SetupMlEmbeddedEvaluationKitLibraries.cmake @@ -21,6 +21,7 @@ set(ETHOS_U_NPU_DRIVER_SRC_PATH "${ml_embedded_evaluation_kit_SOURCE_DIR}/depend # Extra arguments for setting up default resources (for vela optimizer) set(ML_RESOURCES_SET_UP_ARGS "--additional-ethos-u-config-name=${ETHOSU_TARGET_NPU_CONFIG}" + "--use-case-resources-file=${ML_USE_CASE_RESOURCES_FILE}" ) # Tensorflow settings @@ -35,7 +36,11 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/tensorflow-microlite) set(RESOURCES_OUTFILE "${ml_embedded_evaluation_kit_SOURCE_DIR}/resources_downloaded/resources_downloaded_metadata.json") if(NOT EXISTS "${RESOURCES_OUTFILE}") execute_process( - COMMAND ${CMAKE_COMMAND} -E env CC=gcc ${PYTHON} ${ml_embedded_evaluation_kit_SOURCE_DIR}/set_up_default_resources.py ${ML_RESOURCES_SET_UP_ARGS} + COMMAND + ${CMAKE_COMMAND} + -E env CC=gcc + ${PYTHON} ${ml_embedded_evaluation_kit_SOURCE_DIR}/set_up_default_resources.py + ${ML_RESOURCES_SET_UP_ARGS} RESULT_VARIABLE return_code ) if (NOT return_code EQUAL "0") diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch deleted file mode 100644 index 476cf15e..00000000 --- a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-resources-Replace-model-wav2letter-with-tiny_wav2let.patch +++ /dev/null @@ -1,66 +0,0 @@ -From 9ff3fa2ade014893d5406680db6dac8d79d9ef96 Mon Sep 17 00:00:00 2001 -From: Gabor Abonyi -Date: Mon, 22 Jan 2024 14:55:03 +0100 -Subject: [PATCH 1/4] resources: Replace model wav2letter with tiny_wav2letter - -`wav2letter` model needs singificant amount of memory (around 14Mb), -where as `tiny_wav2letter` needs around 4Mb. -Update the location of the resource where referenced for -the `asr` and `kws_asr` use cases. - -Signed-off-by: Gabor Abonyi ---- - scripts/py/use_case_resources.json | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - -diff --git a/scripts/py/use_case_resources.json b/scripts/py/use_case_resources.json -index 80fa28d..9e50fb6 100644 ---- a/scripts/py/use_case_resources.json -+++ b/scripts/py/use_case_resources.json -@@ -16,16 +16,16 @@ - { - "name": "asr", - "url_prefix": [ -- "https://github.com/ARM-software/ML-zoo/raw/1a92aa08c0de49a7304e0a7f3f59df6f4fd33ac8/models/speech_recognition/wav2letter/tflite_pruned_int8/" -+ "https://github.com/ARM-software/ML-zoo/raw/eb2170aac1317f00b128ab82bdb159cfcca36ea6/models/speech_recognition/tiny_wav2letter/tflite_pruned_int8/" - ], - "resources": [ - { -- "name": "wav2letter_pruned_int8.tflite", -- "url": "{url_prefix:0}wav2letter_pruned_int8.tflite" -+ "name": "tiny_wav2letter_pruned_int8.tflite", -+ "url": "{url_prefix:0}tiny_wav2letter_pruned_int8.tflite" - }, - { - "name": "ifm0.npy", -- "url": "{url_prefix:0}testing_input/input_2_int8/0.npy" -+ "url": "{url_prefix:0}testing_input/input_1_int8/0.npy" - }, - { - "name": "ofm0.npy", -@@ -96,18 +96,18 @@ - { - "name": "kws_asr", - "url_prefix": [ -- "https://github.com/ARM-software/ML-zoo/raw/1a92aa08c0de49a7304e0a7f3f59df6f4fd33ac8/models/speech_recognition/wav2letter/tflite_pruned_int8/", -+ "https://github.com/ARM-software/ML-zoo/raw/eb2170aac1317f00b128ab82bdb159cfcca36ea6/models/speech_recognition/tiny_wav2letter/tflite_pruned_int8/", - "https://github.com/ARM-software/ML-zoo/raw/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8/" - ], - "resources": [ - { -- "name": "wav2letter_pruned_int8.tflite", -- "url": "{url_prefix:0}wav2letter_pruned_int8.tflite" -+ "name": "tiny_wav2letter_pruned_int8.tflite", -+ "url": "{url_prefix:0}tiny_wav2letter_pruned_int8.tflite" - }, - { - "sub_folder": "asr", - "name": "ifm0.npy", -- "url": "{url_prefix:0}testing_input/input_2_int8/0.npy" -+ "url": "{url_prefix:0}testing_input/input_1_int8/0.npy" - }, - { - "sub_folder": "asr", --- -2.40.1 - diff --git a/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-use-case-resources-Enabled-user-provided-metadata.patch b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-use-case-resources-Enabled-user-provided-metadata.patch new file mode 100644 index 00000000..725a3e19 --- /dev/null +++ b/components/ai/ml_embedded_evaluation_kit/integration/patches/0001-use-case-resources-Enabled-user-provided-metadata.patch @@ -0,0 +1,101 @@ +From f3f30b50769f5c3cf3c32f1b715acefd6ba0b8bc Mon Sep 17 00:00:00 2001 +From: Hugues Kamba-Mpiana +Date: Mon, 4 Mar 2024 16:01:55 +0000 +Subject: [PATCH] use-case-resources: Enabled user provided metadata + +An optional argument has been added to the `set_up_default_resources.py` +Python script to allow passing of a user defined use case resources +metadata JSON file. +This shortens the build time by only downloading the resources the +end user is interested in. It also shortens the optimization part +which takes additional minutes as it is done for all models and for +all the specified NPU configurations. + + +Signed-off-by: Hugues Kamba-Mpiana +--- + set_up_default_resources.py | 23 ++++++++++++++++++----- + 1 file changed, 18 insertions(+), 5 deletions(-) + +diff --git a/set_up_default_resources.py b/set_up_default_resources.py +index f5cd0ac..5bbb5e9 100755 +--- a/set_up_default_resources.py ++++ b/set_up_default_resources.py +@@ -96,21 +96,24 @@ class UseCase: + MPS3_MAX_SRAM_SZ = 2 * 1024 * 1024 # 2 MiB (2 banks of 1 MiB each) + + +-def load_use_case_resources(current_file_dir: Path) -> typing.List[UseCase]: ++def load_use_case_resources( ++ current_file_dir: Path, use_case_resources_file: Path ++) -> typing.List[UseCase]: + """ + Load use case metadata resources + + Parameters + ---------- + current_file_dir: Directory of the current script ++ use_case_resources_file: Path to a JSON file containing the use case ++ metadata resources. + + Returns + ------- + The use cases resources object parsed to a dict + """ + +- resources_path = current_file_dir / "scripts" / "py" / "use_case_resources.json" +- with open(resources_path, encoding="utf8") as f: ++ with open(use_case_resources_file, encoding="utf8") as f: + use_cases = json.load(f) + return [ + UseCase( +@@ -579,7 +582,8 @@ def set_up_resources( + additional_npu_config_names: tuple = (), + arena_cache_size: int = 0, + check_clean_folder: bool = False, +- additional_requirements_file: Path = "" ++ additional_requirements_file: Path = "", ++ use_case_resources_file: Path = "", + ) -> Path: + """ + Helpers function that retrieve the output from a command. +@@ -597,6 +601,8 @@ def set_up_resources( + additional_requirements_file (str): Path to a requirements.txt file if + additional packages need to be + installed. ++ use_case_resources_file (str): Path to a JSON file containing the use case ++ metadata resources. + + Returns + ------- +@@ -619,7 +625,7 @@ def set_up_resources( + ) + logging.info("Using Python version: %s", sys.version_info) + +- json_uc_res = load_use_case_resources(current_file_dir) ++ json_uc_res = load_use_case_resources(current_file_dir, use_case_resources_file) + setup_script_hash = get_md5sum_for_file(Path(__file__).resolve()) + + metadata_dict, setup_script_hash_verified = initialize_resources_directory( +@@ -706,6 +712,12 @@ if __name__ == "__main__": + type=str, + default=Path(__file__).parent.resolve() / 'scripts' / 'py' / 'requirements.txt' + ) ++ parser.add_argument( ++ "--use-case-resources-file", ++ help="Path to the use case resources file", ++ type=str, ++ default=Path(__file__).parent.resolve() / 'scripts' / 'py' / 'use_case_resources.json' ++ ) + + args = parser.parse_args() + +@@ -724,4 +736,5 @@ if __name__ == "__main__": + args.arena_cache_size, + args.clean, + args.requirements_file, ++ args.use_case_resources_file, + ) +-- +2.34.1 + diff --git a/release_changes/202403041609.change b/release_changes/202403041609.change new file mode 100644 index 00000000..e0710f5a --- /dev/null +++ b/release_changes/202403041609.change @@ -0,0 +1 @@ +mlek: Reduce build time with reduced use case resources metadata