From f076f40e60e301a017016ce33e8939da3d61a68d Mon Sep 17 00:00:00 2001 From: Stijn Tintel Date: Fri, 18 Aug 2023 15:33:38 +0300 Subject: [PATCH 1/3] CI: rework into multiple workflows Github Actions does not support adding dependencies between multiple workflows. For this reason, we previously defined all CI actions in a single workflow file. As we really don't like this approach, here is another attempt to improve the situation. We now have a meta workflow, that will call other workflows based on the changed files. If any file is being changed that would result in a changed build container, we call the build_container workflow. The build_container workflow calls the build_willow workflow once it's done. If the changed files would not result in a changed build container, we directly call the build_willow workflow. This will make the CI more efficient, and faster in case we don't change any files that would result in a changed build container. It will also work around the limitation of Github Actions that prevent us from publishing container images to GHCR for PRs openend from forks. This will still be a problem when people want to submit changes to the build container, but we're still investigating options how to fix this properly. --- .github/workflows/build-container.yml | 53 ++++++++++++++++++ ...build-and-publish.yml => build-willow.yml} | 54 +++---------------- .github/workflows/meta.yml | 43 +++++++++++++++ 3 files changed, 104 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/build-container.yml rename .github/workflows/{build-and-publish.yml => build-willow.yml} (72%) create mode 100644 .github/workflows/meta.yml diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml new file mode 100644 index 00000000..ba0b0024 --- /dev/null +++ b/.github/workflows/build-container.yml @@ -0,0 +1,53 @@ +--- +name: build-container + +env: + GH_TOKEN: ${{ github.token }} + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +on: workflow_call + +jobs: + build-container: + runs-on: ubuntu-22.04 + outputs: + json: ${{ steps.metadata.outputs.json }} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: extract metadata + id: metadata + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=tag + type=ref,event=pr + type=ref,event=branch + + - name: build container + uses: docker/build-push-action@v4 + with: + context: . + file: ./Dockerfile + push: true + labels: ${{ steps.metadata.outputs.labels }} + tags: ${{ steps.metadata.outputs.tags }} + + - name: debug + run: echo "${{ fromJSON(steps.metadata.outputs.json).tags[0] }}" + + build-willow: + uses: ./.github/workflows/build-willow.yml + needs: build-container + with: + container-image: ${{ fromJSON(needs.build-container.outputs.json).tags[0] }} diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-willow.yml similarity index 72% rename from .github/workflows/build-and-publish.yml rename to .github/workflows/build-willow.yml index 2a4ace65..ca5041ce 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-willow.yml @@ -1,55 +1,18 @@ --- -name: build_container - +name: build-willow env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} on: - pull_request: - push: - tag: - - '*' + workflow_call: + inputs: + container-image: + required: true + type: string jobs: - build_container: - runs-on: ubuntu-22.04 - outputs: - json: ${{ steps.metadata.outputs.json }} - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: extract metadata - id: metadata - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=tag - type=ref,event=pr - type=ref,event=branch - - - name: build container - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: true - labels: ${{ steps.metadata.outputs.labels }} - tags: ${{ steps.metadata.outputs.tags }} - - - name: debug - run: echo "${{ fromJSON(steps.metadata.outputs.json).tags[0] }}" - build-willow-dist: strategy: matrix: @@ -57,9 +20,8 @@ jobs: - ESP32_S3_BOX - ESP32_S3_BOX_LITE runs-on: ubuntu-22.04 - needs: build_container container: - image: ${{ fromJSON(needs.build_container.outputs.json).tags[0] }} + image: ${{ inputs.container-image }} credentials: username: ${{ github.actor }} password: ${{ secrets.github_token }} @@ -77,7 +39,7 @@ jobs: - name: checkout uses: actions/checkout@v3 - name: debug - run: echo ${{ needs.build_container.outputs.tags }} + run: echo ${{ inputs.container-tag }} - name: ls -al run: ls -al - name: ls -al / diff --git a/.github/workflows/meta.yml b/.github/workflows/meta.yml new file mode 100644 index 00000000..f4be54f9 --- /dev/null +++ b/.github/workflows/meta.yml @@ -0,0 +1,43 @@ +--- +name: meta + +env: + GH_TOKEN: ${{ github.token }} + +on: + pull_request: + push: + +jobs: + trigger_workflow: + name: trigger workflow based on changed files + runs-on: ubuntu-22.04 + outputs: + container_any_changed: ${{ steps.changed_files_yaml.outputs.container_any_changed }} + steps: + - uses: actions/checkout@v3 + with: + # fetch full history to be able to gather changed files in all new commits + fetch-depth: 0 + + - uses: tj-actions/changed-files@v37 + id: changed_files_yaml + with: + files_yaml: | + container: + - '.github/workflows/build-container.yml' + - 'Dockerfile' + - 'container.gitconfig' + - 'utils.sh' + + build_container: + if: ${{ needs.trigger_workflow.outputs.container_any_changed == 'true' }} + uses: ./.github/workflows/build-container.yml + needs: trigger_workflow + + build_willow: + if: ${{ needs.trigger_workflow.outputs.container_any_changed == 'false' }} + uses: ./.github/workflows/build-willow.yml + needs: trigger_workflow + with: + container-image: ghcr.io/toverainc/willow:main From 03ac28e9019493b23ba7b2f0c2be9ed972e6aab8 Mon Sep 17 00:00:00 2001 From: Stijn Tintel Date: Fri, 18 Aug 2023 19:23:08 +0300 Subject: [PATCH 2/3] patches: drop directory Since cbd03e37ad1c ("utils.sh: use willow branch from our ESP-ADF fork") we no longer need to apply patches when preparing the build container. Drop the patches directory as it is no longer used since that commit. --- patches/esp-adf-multinet6.patch | 58 --------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 patches/esp-adf-multinet6.patch diff --git a/patches/esp-adf-multinet6.patch b/patches/esp-adf-multinet6.patch deleted file mode 100644 index 0516b661..00000000 --- a/patches/esp-adf-multinet6.patch +++ /dev/null @@ -1,58 +0,0 @@ -diff --git a/components/audio_recorder/recorder_sr.c b/components/audio_recorder/recorder_sr.c -index f8a449b..f0272f3 100644 ---- deps/esp-adf/components/audio_recorder/recorder_sr.c -+++ deps/esp-adf/components/audio_recorder/recorder_sr.c -@@ -185,8 +185,15 @@ static esp_err_t recorder_mn_detect(recorder_sr_t *recorder_sr, int16_t *buffer, - } - if (mn_state == ESP_MN_STATE_DETECTED) { - esp_mn_results_t *mn_result = multinet->get_results(recorder_sr->mn_handle); -+ printf("ESP MN detected command text:%s\n", mn_result->string); -+ -+ for (int i = 0; i < mn_result->num; i++) { -+ printf("ESP MN TOP %d, command_id: %d, phrase_id: %d, prob: %f\n", -+ i+1, mn_result->command_id[i], mn_result->phrase_id[i], mn_result->prob[i]); -+ } -+ - if (recorder_sr->mn_monitor) { -- recorder_sr->mn_monitor(mn_result->command_id[0], recorder_sr->mn_monitor_ctx); -+ recorder_sr->mn_monitor(mn_result->phrase_id[0], recorder_sr->mn_monitor_ctx); - } - #if CONFIG_IDF_TARGET_ESP32 - recorder_sr_enable_wakenet_aec(recorder_sr); -@@ -199,7 +206,12 @@ static esp_err_t recorder_mn_detect(recorder_sr_t *recorder_sr, int16_t *buffer, - recorder_sr_enable_wakenet_aec(recorder_sr); - #endif - detect_flag = 0; -- ESP_LOGI(TAG, "MN dect quit"); -+ esp_mn_results_t *mn_result = multinet->get_results(recorder_sr->mn_handle); -+ printf("ESP MN detected other text:%s\n", mn_result->string); -+ ESP_LOGI(TAG, "MN TIMEOUT - No valid command detected"); -+ recorder_sr->mn_monitor(0, recorder_sr->mn_monitor_ctx); -+ -+ return ESP_OK; - } - } - return ESP_OK; -@@ -453,9 +465,8 @@ static esp_err_t recorder_sr_mn_enable(void *handle, bool enable) - if (recorder_sr->mn_enable && !recorder_sr->mn_handle) { - char *mn_name = esp_srmodel_filter(recorder_sr->models, ESP_MN_PREFIX, recorder_sr->mn_language); - multinet = esp_mn_handle_from_name(mn_name); -- recorder_sr->mn_handle = multinet->create(mn_name, 5760); -+ recorder_sr->mn_handle = multinet->create(mn_name, 2000); - AUDIO_NULL_CHECK(TAG, recorder_sr->mn_handle, return ESP_FAIL); -- esp_mn_commands_update_from_sdkconfig((esp_mn_iface_t *)multinet, recorder_sr->mn_handle); - } - return ESP_OK; - #else -@@ -547,10 +571,9 @@ recorder_sr_handle_t recorder_sr_create(recorder_sr_cfg_t *cfg, recorder_sr_ifac - char *mn_name = esp_srmodel_filter(recorder_sr->models, ESP_MN_PREFIX, recorder_sr->mn_language); - AUDIO_NULL_CHECK(TAG, mn_name, goto _failed); - multinet = esp_mn_handle_from_name(mn_name); -- recorder_sr->mn_handle = multinet->create(mn_name, 5760); -+ recorder_sr->mn_handle = multinet->create(mn_name, 2000); - AUDIO_NULL_CHECK(TAG, recorder_sr->mn_handle, goto _failed); - recorder_sr->mn_enable = true; -- esp_mn_commands_update_from_sdkconfig((esp_mn_iface_t *)multinet, recorder_sr->mn_handle); - } - #endif - recorder_sr->events = xEventGroupCreate(); From 73656011911fd1ff55ca9c46ca465ea487b73ead Mon Sep 17 00:00:00 2001 From: Stijn Tintel Date: Fri, 18 Aug 2023 19:28:06 +0300 Subject: [PATCH 3/3] CI: only build Willow when needed There is no point in running a Willow build when there aren't any changes to source files, or other files included in the Willow images. --- .github/workflows/meta.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/meta.yml b/.github/workflows/meta.yml index f4be54f9..22ed6444 100644 --- a/.github/workflows/meta.yml +++ b/.github/workflows/meta.yml @@ -14,6 +14,7 @@ jobs: runs-on: ubuntu-22.04 outputs: container_any_changed: ${{ steps.changed_files_yaml.outputs.container_any_changed }} + image_any_changed: ${{ steps.changed_files_yaml.outputs.image_any_changed }} steps: - uses: actions/checkout@v3 with: @@ -29,6 +30,13 @@ jobs: - 'Dockerfile' - 'container.gitconfig' - 'utils.sh' + image: + - 'CMakeLists.txt' + - 'main/**' + - 'partitions_willow.csv' + - 'sdkconfig.willow' + - 'spiffs/**' + build_container: if: ${{ needs.trigger_workflow.outputs.container_any_changed == 'true' }} @@ -36,7 +44,7 @@ jobs: needs: trigger_workflow build_willow: - if: ${{ needs.trigger_workflow.outputs.container_any_changed == 'false' }} + if: ${{ needs.trigger_workflow.outputs.container_any_changed == 'false' && needs.trigger_workflow.outputs.image_any_changed == 'true' }} uses: ./.github/workflows/build-willow.yml needs: trigger_workflow with: