shellcheckLint parallel processing

## shellcheckLint enhancements Added ability to shellcheck all git files in parallel using xargs with compatiility with json, json1, checkstyle formats ## Breaking changes - removed bin/test binary - removed bin/runBuildContainer - removed Docker::runBuildContainer - added GNU sed to alpine image - changed runUnitTests hook to rely on a test.sh file needed to be present in the target repository. ## Github workflow - refactored bin/doc - ensure installRequirements executed - removed uneeded steps for pre-commit - unit test generate checkstyle even if error - cspell - remove global dictionaries config key and adapt cspell rules accordingly - replaced technote-space/workflow-conclusion-action@v3 by AbsoLouie/[email protected]
fchastanet · May 8, 2024 · 4a0641e · 4a0641e
1 parent 37ae985
commit 4a0641e
Show file tree

Hide file tree

Showing 34 changed files with 532 additions and 5,357 deletions.
diff --git a/.cspell/bash.txt b/.cspell/bash.txt
@@ -52,6 +52,7 @@ friday
 frégate
 getopt
 getsys
+GKHF
 gpgsign
 gsub
 HISTFILE

diff --git a/.cspell/codespellrc-dic.txt b/.cspell/codespellrc-dic.txt
@@ -1,2 +1,3 @@
+adpat -> adapt
 fileTest -> fileTest
 tst -> test
diff --git a/.cspell/myAwk.txt b/.cspell/myAwk.txt
@@ -0,0 +1,8 @@
+apos
+asort
+gensub
+getline
+gsub
+substr
+tolower
+varchar
diff --git a/.cspell/readme.txt b/.cspell/readme.txt
@@ -11,23 +11,38 @@ bingitisancestorof
 bingitrenamebranch
 binmysql
 Brzóska
+Chastanet
 cklmurl
+elif
+esac
+exitcode
 fchastanet
 fplain
+Inlines
+inlines
 installfile
 Jetbrains
+lastpipe
 libassertsh
 libutilssh
 loadprofile
 mandatorysoftware
 manpages
 markdownlint
 Michał
+noargs
 openapi
+posix
 Powerlevel
 powerlevel
+readlink
+realpath
 Scrasnups
 setupsh
+shopt
+Slatis
+templating
+TMPDIR
 Tratif
 unserialize
 untar

diff --git a/.cspell/softwares.txt b/.cspell/softwares.txt
@@ -96,10 +96,13 @@ MAILHOG
 mailserver
 mattrose
 mbstring
+megalinter
 minikube
 Minikube
 mlocate
 Mlocate
+Msys
+MSYS
 MULTIPARTS
 mypy
 mysql
@@ -129,6 +132,8 @@ resolvconf
 runc
 scrasnups
 screenlock
+shellcheck
+shellcheckrc
 SpacesAftertabs
 stylelint
 Sume

diff --git a/.docker/Dockerfile.alpine b/.docker/Dockerfile.alpine
@@ -17,7 +17,9 @@ RUN \
     "gettext" \
     "git" \
     "jq" \
-    "parallel"
+    "parallel" \
+    # GNU sed
+    "sed"
 
 # Create a default www-data user
 COPY createUser.alpine.sh /usr/local/bin/createUser.sh

diff --git a/.framework-config b/.framework-config
@@ -25,7 +25,7 @@ fi
 # describe the functions that will be skipped from being imported
 FRAMEWORK_FUNCTIONS_IGNORE_REGEXP="${FRAMEWORK_FUNCTIONS_IGNORE_REGEXP:-^(Namespace::functions|Functions::myFunction|Namespace::requireSomething|IMPORT::dir::file|Acquire::ForceIPv4)$}"
 # describe the files that do not contain function to be imported
-NON_FRAMEWORK_FILES_REGEXP="${NON_FRAMEWORK_FILES_REGEXP:-(^bin/|^hooks/|^test.sh$|^.github/|^.docker/createUser.|.framework-config|.bats$|/testsData/|^manualTests/|/_.sh$|/ZZZ.sh$|/__all.sh$|^src/_binaries|^src/_includes|^src/batsHeaders.sh$|^src/_standalone)}"
+NON_FRAMEWORK_FILES_REGEXP="${NON_FRAMEWORK_FILES_REGEXP:-(^bin/|^hooks/|^test.sh$|^preCommitTest.sh$|^.github/|^.docker/createUser.|.framework-config|.bats$|/testsData/|^manualTests/|/_.sh$|/ZZZ.sh$|/__all.sh$|^src/_binaries|^src/_includes|^src/batsHeaders.sh$|^src/_standalone)}"
 # describe the files that are allowed to not have an associated bats file
 BATS_FILE_NOT_NEEDED_REGEXP="${BATS_FILE_NOT_NEEDED_REGEXP:-(^bin/|.framework-config|^.docker/|^.github/|.bats$|/testsData/|^manualTests/|/_.sh$|/ZZZ.sh$|/__all.sh$|^src/batsHeaders.sh$|^src/_includes)}"
 # describe the files that are allowed to not have a function matching the filename

diff --git a/.github/preCommitGeneration.sh b/.github/preCommitGeneration.sh
@@ -10,4 +10,7 @@ awk \
       print "###############################################################################"
     }
   ''' .pre-commit-config.yaml >.pre-commit-config-github.yaml
-sed -i -E '0,/fail_fast: true/s//fail_fast: false/' .pre-commit-config-github.yaml
+sed -i -E \
+  -e '0,/fail_fast: true/s//fail_fast: false/' \
+  -e 's/stages: \[\] # GITHUB/stages: \[manual\] # GITHUB/' \
+  .pre-commit-config-github.yaml
diff --git a/.github/workflows/docsify-gh-pages.yml b/.github/workflows/docsify-gh-pages.yml
@@ -34,7 +34,7 @@ jobs:
 
       - name: Check if doc up to date
         run: |
-          ./bin/doc
+          ./bin/doc --ci
 
       - name: Setup Pages
         uses: actions/configure-pages@v4

diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml
@@ -199,26 +199,19 @@ jobs:
             echo "branch_name=${GITHUB_REF##*/}"
           ) >> "${GITHUB_ENV}"
 
-      - name: Cache pre-commit
-        uses: actions/cache@v4
-        env:
-          cache_name: pre-commit
-          hash: ${{hashFiles('**/.pre-commit-config-github.yaml')}}
-        with:
-          path: ~/.cache/pre-commit
-          key: ${{ runner.os }}-${{ env.cache_name }}-${{ env.hash }}
-          restore-keys: ${{ runner.os }}-${{ env.cache_name }}-
-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Install pre-commit
-        run: pip install pre-commit
-
       - uses: fchastanet/[email protected]
 
+      - name: Install requirements
+        run: |
+          set -exo pipefail
+
+          bin/installRequirements
+
       - name: Run pre-commit
         uses: pre-commit/[email protected]
         id: preCommit
@@ -382,13 +375,15 @@ jobs:
       - name: run unit tests
         id: unitTests
         run: |
-          set -x
-          set -o errexit
-          chmod -R 777 logs
+          set -exo pipefail
 
           bin/installRequirements
 
+          chmod 777 logs
+
           docker pull "scrasnups/build:${{env.image_tag}}"
+
+          status=0
           docker run --rm \
             -e KEEP_TEMP_FILES=0 \
             -e BATS_FIX_TEST=0 \
@@ -401,15 +396,15 @@ jobs:
               /bash/vendor/bats/bin/bats \
               ${{env.batsOptions}} \
               --formatter junit -o logs -r src 2>&1 |
-            tee "logs/bats-${{ env.job_tag }}.log" || exit 1
+            tee "logs/bats-${{ env.job_tag }}.log" || status=$?
 
           awk '/xml version="1.0"/{flag=1} flag; /<\/testsuites>/{flag=0}' \
             "logs/bats-${{ env.job_tag }}.log" >"logs/junit-${{ env.job_tag }}.xml"
+          exit "${status}"
 
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v4
-        # https://stackoverflow.com/a/70549615/3045926
-        if: ${{ always() && steps.unitTests.conclusion == 'success' }}
+        if: ${{ always() }}
         with:
           token: ${{ github.token }}
           check_name: JUnit ${{ env.image_tag }}
@@ -419,7 +414,7 @@ jobs:
           report_paths: "logs/**.xml"
 
       - name: Upload Test Results
-        if: ${{ always() && steps.unitTests.conclusion == 'success' }}
+        if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
           name: Test Results ${{ env.image_tag }}
@@ -443,7 +438,7 @@ jobs:
     steps:
       # run this action to get the workflow conclusion
       # You can get the conclusion via env (env.WORKFLOW_CONCLUSION)
-      - uses: technote-space/workflow-conclusion-action@v3
+      - uses: AbsoLouie/workflow-conclusion-[email protected]
 
       - uses: akatov/commit-status-updater@a9e988ec5454692ff7745a509452422a35172ad6
         with:

diff --git a/.pre-commit-config-github.yaml b/.pre-commit-config-github.yaml
@@ -155,7 +155,7 @@ repos:
           )
 
   - repo: https://github.com/fchastanet/bash-tools-framework
-    rev: 3.2.0
+    rev: 3.2.1
     hooks:
       - id: fixShebangExecutionBit
       - id: awkLint
@@ -214,8 +214,7 @@ repos:
         name: build doc files for github actions
         language: script
         entry: bin/doc
-        args: [--verbose]
+        args: [--verbose, --ci]
         pass_filenames: false
         require_serial: true
-        always_run: true
-        stages: [manual]
+        stages: [manual] # GITHUB
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -150,7 +150,7 @@ repos:
           )
 
   - repo: https://github.com/fchastanet/bash-tools-framework
-    rev: 3.2.0
+    rev: 3.2.1
     hooks:
       - id: fixShebangExecutionBit
       - id: awkLint
@@ -209,8 +209,7 @@ repos:
         name: build doc files for github actions
         language: script
         entry: bin/doc
-        args: [--verbose]
+        args: [--verbose, --ci]
         pass_filenames: false
         require_serial: true
-        always_run: true
-        stages: [manual]
+        stages: [] # GITHUB
diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml
@@ -72,8 +72,8 @@
   name: Run unit tests
   description: run unit test using bats in docker isolated environment
   language: script
-  entry: bin/test
-  args: [-r, src, -j, "30"]
+  entry: preCommitTest.sh
+  args: ["scrasnups/build:bash-tools-ubuntu-5.3", -r, src, -j, "30"]
   pass_filenames: false
   always_run: true
   require_serial: true

diff --git a/.shellcheckrc b/.shellcheckrc
@@ -6,4 +6,4 @@ enable=quote-safe-variables
 enable=require-double-brackets
 source-path=SCRIPTDIR
 # special property used by shellcheckLint
-exclude=(^bin/bash-tpl$|\.bats$|\.tpl$|/testsData/)
+exclude=(^bin/|\.bats$|\.tpl$|src/batsHeaders.sh$|/testsData/)
diff --git a/README.md b/README.md
@@ -1,24 +1,5 @@
 # bash-tools-framework
 
-- [1. Excerpt](#1-excerpt)
-  - [1.1. Compile command](#11-compile-command)
-  - [1.2. Build tools](#12-build-tools)
-  - [1.3. Internal tools](#13-internal-tools)
-- [2. Framework](#2-framework)
-  - [2.1. Excerpt](#21-excerpt)
-  - [2.2. Usage](#22-usage)
-- [3. Development Environment](#3-development-environment)
-  - [3.1. Install dev dependencies](#31-install-dev-dependencies)
-  - [3.2. Precommit hook](#32-precommit-hook)
-  - [3.3. UT](#33-ut)
-    - [3.3.1. Debug bats](#331-debug-bats)
-  - [3.4. connect to container manually](#34-connect-to-container-manually)
-  - [3.5. auto generated bash doc](#35-auto-generated-bash-doc)
-  - [3.6. github page](#36-github-page)
-- [4. Troubleshooting](#4-troubleshooting)
-  - [4.1. compile.bats embed not working on alpine investigation](#41-compilebats-embed-not-working-on-alpine-investigation)
-- [5. Acknowledgements](#5-acknowledgements)
-
 <!-- remove -->
 
 > **_NOTE:_** Documentation is best viewed on
@@ -36,7 +17,7 @@
 
 <!-- markdownlint-disable MD013 -->
 
-[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/dbelyaev/action-checkstyle?logo=github&sort=semver)](https://github.com/dbelyaev/action-checkstyle/releases)
+[![GitHub release (latest SemVer)](https://img.shields.io/github/release/fchastanet/bash-tools-framework?logo=github&sort=semver)](https://github.com/fchastanet/bash-tools-framework/releases)
 [![GitHubLicense](https://img.shields.io/github/license/Naereen/StrapDown.js.svg)](https://github.com/fchastanet/bash-tools-framework/blob/master/LICENSE)
 [![CI/CD](https://github.com/fchastanet/bash-tools-framework/actions/workflows/lint-test.yml/badge.svg)](https://github.com/fchastanet/bash-tools-framework/actions?query=workflow%3A%22Lint+and+test%22+branch%3Amaster)
 [![ProjectStatus](http://opensource.box.com/badges/active.svg)](http://opensource.box.com/badges "Project Status")
@@ -47,6 +28,25 @@
 
 <!-- markdownlint-restore -->
 
+- [1. Excerpt](#1-excerpt)
+  - [1.1. Compile command](#11-compile-command)
+  - [1.2. Build tools](#12-build-tools)
+  - [1.3. Internal tools](#13-internal-tools)
+- [2. Framework](#2-framework)
+  - [2.1. Excerpt](#21-excerpt)
+  - [2.2. Usage](#22-usage)
+- [3. Development Environment](#3-development-environment)
+  - [3.1. Install dev dependencies](#31-install-dev-dependencies)
+  - [3.2. Precommit hook](#32-precommit-hook)
+  - [3.3. UT](#33-ut)
+    - [3.3.1. Debug bats](#331-debug-bats)
+  - [3.4. connect to container manually](#34-connect-to-container-manually)
+  - [3.5. auto generated bash doc](#35-auto-generated-bash-doc)
+  - [3.6. github page](#36-github-page)
+- [4. Troubleshooting](#4-troubleshooting)
+  - [4.1. compile.bats embed not working on alpine investigation](#41-compilebats-embed-not-working-on-alpine-investigation)
+- [5. Acknowledgements](#5-acknowledgements)
+
 ## 1. Excerpt
 
 [Full documentation can be found here](/#/FrameworkIndex) but here an excerpt of
@@ -68,11 +68,9 @@ see related documentation [Compile command](doc/CompileCommand.md).
 
 ### 1.3. Internal tools
 
-- **test** : test this framework by launching bats inside docker container with
-  the needed dependencies
+- **test.sh** : test this framework by launching bats inside docker container
+  with the needed dependencies
 - **doc** : generate markdown documentation for this framework
-- **runBuildContainer** : run docker container with the Dockerfile of this
-  project, allowing to build doc and run tests
 
 ## 2. Framework
 
@@ -241,16 +239,21 @@ see related documentation [Compile command](doc/CompileCommand.md).
 
 Dependencies are automatically installed when used.
 
-`bin/test` script will install the following libraries inside `vendor` folder:
+`bin/installRequirements` script will install the following libraries inside
+`vendor` folder:
 
 - [bats-core/bats-core](https://github.com/bats-core/bats-core.git)
 - [bats-core/bats-support](https://github.com/bats-core/bats-support.git)
 - [bats-core/bats-assert](https://github.com/bats-core/bats-assert.git)
 - [Flamefire/bats-mock](https://github.com/Flamefire/bats-mock.git)
+- hadolint
+- shellcheck
 
 `bin/doc` script will install:
 
 - [reconquest/shdoc](https://github.com/reconquest/shdoc)
+- hadolint
+- shellcheck
 
 To avoid checking for libraries update and have an impact on performance, a file
 is created in vendor dir.
@@ -277,19 +280,18 @@ All the methods of this framework are unit tested, you can run the unit tests
 using the following command
 
 ```bash
-./bin/test -r src
+./test.sh scrasnups/build:bash-tools-ubuntu-5.3 -r src -j 30
 ```
 
 Launch UT on different environments:
 
 ```bash
-VENDOR="alpine" BASH_TAR_VERSION=4.4 BASH_IMAGE=bash SKIP_BUILD=0 SKIP_USER=1 ./bin/test -r src
-VENDOR="alpine" BASH_TAR_VERSION=5.0 BASH_IMAGE=bash SKIP_BUILD=0 SKIP_USER=1 ./bin/test -r src
-VENDOR="alpine" BASH_TAR_VERSION=5.1 BASH_IMAGE=bash SKIP_BUILD=0 SKIP_USER=1 ./bin/test -r src
-
-VENDOR="ubuntu" BASH_TAR_VERSION=4.4 BASH_IMAGE=ubuntu:20.04 SKIP_BUILD=0 SKIP_USER=1 ./bin/test -r src
-VENDOR="ubuntu" BASH_TAR_VERSION=5.0 BASH_IMAGE=ubuntu:20.04 SKIP_BUILD=0 SKIP_USER=1 ./bin/test -r src
-VENDOR="ubuntu" BASH_TAR_VERSION=5.1 BASH_IMAGE=ubuntu:20.04 SKIP_BUILD=0 SKIP_USER=1 ./bin/test -r src
+./test.sh scrasnups/build:bash-tools-ubuntu-4.4 -r src -j 30
+./test.sh scrasnups/build:bash-tools-ubuntu-5.0 -r src -j 30
+./test.sh scrasnups/build:bash-tools-ubuntu-5.3 -r src -j 30
+./test.sh scrasnups/build:bash-tools-alpine-4.4 -r src -j 30
+./test.sh scrasnups/build:bash-tools-alpine-5.0 -r src -j 30
+./test.sh scrasnups/build:bash-tools-alpine-5.3 -r src -j 30
 ```
 
 #### 3.3.1. Debug bats
-Original file line number
+Diff line change
@@ Expand Up / @@ -52,6 +52,7 @@ friday @@
     frégate
     getopt
     getsys
+    GKHF
     gpgsign
     gsub
     HISTFILE
@@ Expand Down @@