From 28143ee0beb1799225dbc3dd18f0ce4217b231ce Mon Sep 17 00:00:00 2001 From: Adrian Stanea Date: Thu, 7 Nov 2024 14:44:57 +0200 Subject: [PATCH] tools/license: add exclude list for license check - Removed verbose output from license.sh. Now script will output only paths to files that are missing license headers to stdout. Logs are directed to stderr. - Improved error message for unhandled extensions. Signed-off-by: Adrian Stanea --- tools/exclude_list.sh | 166 ++++++++++++++++++ tools/license-header/add_license_header.sh | 5 +- .../batch_add_license_headers.sh | 4 + tools/license-header/scan_missing_headers.sh | 91 ++++------ tools/license.sh | 102 +++++++---- 5 files changed, 281 insertions(+), 87 deletions(-) create mode 100644 tools/exclude_list.sh diff --git a/tools/exclude_list.sh b/tools/exclude_list.sh new file mode 100644 index 000000000..b14a6db87 --- /dev/null +++ b/tools/exclude_list.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +# NOTE: paths containing * are treated as regex => escape them with \ (ex: \*) + +OMIT_DIRS_LIST=( + .\* + .git\* + .vscode + android + apple + build + ci/\* + docs + tmp + tools + windows + # Directories specific to your local environment that are not part of the repository + venv +) +OMIT_DIRS_LIST=("${OMIT_DIRS_LIST[@]/#/*\/}") # */ prefix to each element +GITIGNORE_DIRS_LIST=( + # .gitinore + CMakeFiles + build\* + deps + iio-emu + html/\* + build_arm64-v8a/\* + .cache/\* + build/\* + .vscode/\* + windows/\* + bin/\* + lib/\* + docs/_build/\* + share/\* + # ci/flatpak/.gitinore + ci/flatpak/build + ci/flatpak/repo +) +GITIGNORE_DIRS_LIST=("${GITIGNORE_DIRS_LIST[@]/#/*\/}") # */ prefix to each element + +OMIT_FILES_LIST=( + .clang-format + .clangformatignore + .cmake-format + .gitignore + .gitmodules + LICENSE + \*.md + \*.png + \*.rst + azure-pipelines.yml + requirements.txt + \*.html + \*.svg + \*.icns + \*.ico + \*.qmodel + \*.ui + \*.json + \*.qrc + \*.ts + \*.gif + \*.theme + \*.ttf + \*.zip + \*.csv + \*.bin + \*.xml + \*.cmakein + # Files specific to your local environment that are not part of the repository + \*.build + \*.git + \*.gitrepo + \*.mat + \*.user +) +OMIT_FILES_LIST=("${OMIT_FILES_LIST[@]/#/*\/}") +GITINORE_FILES_LIST=( + # .gitinore + CMakeLists.txt.user + CMakeCache.txt + cmake_install.cmake + Makefile + moc_\*.cpp + \*_automoc.cpp + ui_\*.h + resources/scopy_osp.html + resources/stylesheets/default.qss + resources/stylesheets/light.qss + resources/credits.html + resources/scopy_home.html + resources/about.html + \*qt.conf + \*.swp + \*.DS_Store + android-build + android\*.sh + \*.apk + \*.aab + ci/general/gh-actions.envs + core/include/scopy-core_config.h + core/include/scopy-core_export.h + core/include/core/scopy-core_config.h + core/include/core/scopy-core_export.h + common/include/common/scopy-common_config.h + common/include/common/scopy-common_export.h + gr-util/include/gr-util/scopy-gr-util_export.h + gui/include/gui/scopy-gui_export.h + gui/include/gui/scopy-gui_config.h + iio-widgets/include/iio-widgets/scopy-iio-widgets_export.h + iioutil/include/iioutil/scopy-iioutil_export.h + pluginbase/include/pluginbase/scopy-pluginbase_config.h + pluginbase/include/pluginbase/scopy-pluginbase_export.h + # gui/res/.gitinore + gui/res/about.html + gui/res/scopy_osp.html + gui/res/buildinfo.html + gui/res/credits.html + # ci/flatpak/.gitinore + ci/flatpak/Scopy.flatpak + ci/flatpak/.flatpak-builder + ci/flatpak/org.adi.Scopy.json + ci/flatpak/tmp.json + # plugins/adc/.gitinore + plugins/adc/include/adc/scopy-adc_export.h + plugins/adc/include/adc/scopy-adc_config.h + # plugins/dac/.gitinore + plugins/dac/include/dac/scopy-dac_export.h + plugins/dac/include/dac/scopy-dac_config.h + # plugins/m2k/.gitinore + plugins/m2k/include/m2k/scopy-m2k_export.h + plugins/m2k/include/m2k/scopy-m2k_config.h + # plugins/pqm/.gitinore + plugins/pqm/include/pqm/scopy-pqm_export.h + plugins/pqm/include/pqm/scopy-pqm_config.h + # plugins/test/.gitinore + plugins/test/include/test/scopy-test_export.h + # plugins/swiot/.gitinore + plugins/swiot/include/swiot/scopy-swiot_export.h + plugins/swiot/include/swiot/scopy-swiot_config.h + # plugins/test2/.gitinore + plugins/test2/include/test2/scopy-test2_export.h + # plugins/guitest/.gitinore + plugins/guitest/include/guitest/scopy-guitest_export.h + # plugins/debugger/.gitinore + plugins/debugger/include/debugger/scopy-debugger_export.h + plugins/debugger/include/debugger/scopy-debugger_config.h + # plugins/datalogger/.gitinore + plugins/datalogger/include/datalogger/scopy-datalogger_export.h + plugins/datalogger/include/datalogger/scopy-datalogger_config.h + # plugins/bareminimum/.gitinore + plugins/bareminimum/include/bareminimum/scopy-bareminimum_export.h + # plugins/m2k/m2k-gui/.gitinore + plugins/m2k/m2k-gui/include/m2k-gui/scopy-m2k-gui_export.h + plugins/m2k/m2k-gui/include/m2k-gui/scopy-m2k-gui_config.h + plugins/m2k/m2k-gui/gr-gui/include/gr-gui/scopy-gr-gui_export.h + plugins/m2k/m2k-gui/gr-gui/include/gr-gui/scopy-gr-gui_config.h + plugins/m2k/m2k-gui/sigrok-gui/include/sigrok-gui/scopy-sigrok-gui_export.h + plugins/m2k/m2k-gui/sigrok-gui/include/sigrok-gui/scopy-sigrok-gui_config.h + # plugins/regmap/.gitinore + plugins/regmap/include/regmap/scopy-regmap_export.h + plugins/regmap/include/regmap/scopy-regmap_config.h +) +GITINORE_FILES_LIST=("${GITINORE_FILES_LIST[@]/#/*\/}") diff --git a/tools/license-header/add_license_header.sh b/tools/license-header/add_license_header.sh index fdea63e23..20252b921 100755 --- a/tools/license-header/add_license_header.sh +++ b/tools/license-header/add_license_header.sh @@ -240,9 +240,10 @@ function main() { FILE_EXTENSION="${FILE##*.}" - COMMENT_STYLE=${COMMENT_STYLES[$FILE_EXTENSION]} + COMMENT_STYLE=${COMMENT_STYLES[$FILE_EXTENSION]:-} if [[ -z "$COMMENT_STYLE" ]]; then - echo "Unsupported file extension: $FILE_EXTENSION" >&2 + echo "Error: Unsupported file extension '${FILE_EXTENSION}' found in file '${FILE}'" >&2 + echo "Define 'COMMENT_STYLE' for '$FILE_EXTENSION' in .comment_styles.conf" >&2 exit 1 fi diff --git a/tools/license-header/batch_add_license_headers.sh b/tools/license-header/batch_add_license_headers.sh index 317757189..99eb59f04 100755 --- a/tools/license-header/batch_add_license_headers.sh +++ b/tools/license-header/batch_add_license_headers.sh @@ -81,6 +81,10 @@ function process_files() { "$add_license_header" "$file" \ --template "$TEMPLATE_FILE" \ --params "$PARAMS_FILE" + if [[ $? -ne 0 ]]; then + echo "Error: Failed to add license header to $file" + exit 1 + fi done <"$file_list" } diff --git a/tools/license-header/scan_missing_headers.sh b/tools/license-header/scan_missing_headers.sh index 145322ece..179831665 100755 --- a/tools/license-header/scan_missing_headers.sh +++ b/tools/license-header/scan_missing_headers.sh @@ -9,7 +9,6 @@ BASE_DIR_PATH="" FILES_WITHOUT_LICENSE=() FILE_EXTENSIONS=() N_LINES=80 # When searching for license headers, use the first N lines of the file -VERBOSE=false # Define colors used for logging NC='\033[0m' # No Color @@ -38,14 +37,12 @@ Options: -h, --help Display this help message. - -v, --verbose Display verbose output. - EOF } function parse_arguments() { - LONG_OPTS=dirs:,files:,path:,help,verbose - OPTIONS=d:f:p:h:v + LONG_OPTS=dirs:,files:,path:,help + OPTIONS=d:f:p:h VALID_ARGS=$(getopt --options=$OPTIONS --longoptions=$LONG_OPTS --name "$0" -- "$@") if [[ $? -ne 0 ]]; then @@ -56,7 +53,7 @@ function parse_arguments() { getopt --test >/dev/null && true if [[ $? -ne 4 ]]; then - echo "$(getopt --test) failed in this environment." + echo "$(getopt --test) failed in this environment." >&2 exit 1 fi @@ -84,10 +81,6 @@ function parse_arguments() { BASE_DIR_PATH="$2" shift 2 ;; - -v | --verbose) - VERBOSE=true - shift - ;; --) shift break @@ -248,7 +241,8 @@ function identify_license() { depth=$(echo "$relative_path" | awk -F'/' '{print NF-1}') indent=$(printf "%*s" $((depth * 4)) "") - local header=$(read_header "$file" "$n_lines" | normalize_text) + # Sanitize the header text -> remove null characters and normalize the text + local header=$(read_header "$file" "$n_lines" | tr -d '\0' | normalize_text) ((total_checks++)) @@ -256,9 +250,7 @@ function identify_license() { ((no_license_count++)) FILES_WITHOUT_LICENSE+=("$relative_path") - if $VERBOSE; then - echo -e "${indent}${RED}├── $relative_path (No License)${NC}" - fi + echo -e "${indent}${RED}├── $relative_path (No License)${NC}" >&2 # Stop execution if no license is found return fi @@ -307,22 +299,20 @@ function identify_license() { ((unknown_license_count++)) fi - if $VERBOSE; then - # Join the elements of found_licenses_list with a | separator - local joined_licenses=$( - IFS='|' - echo "${found_licenses_list[*]}" - ) - - # Check if the joined string is empty - if [ -z "$joined_licenses" ]; then - joined_licenses="Unknown" - fi + # Join the elements of found_licenses_list with a | separator + local joined_licenses=$( + IFS='|' + echo "${found_licenses_list[*]}" + ) - # Update the echo command to use the joined string - echo -e "${indent}${color}├── $relative_path (${joined_licenses})${NC}" + # Check if the joined string is empty + if [ -z "$joined_licenses" ]; then + joined_licenses="Unknown" fi + # Update the echo command to use the joined string + echo -e "${indent}${color}├── $relative_path (${joined_licenses})${NC}" >&2 + local found=false for license in "${found_licenses_list[@]}"; do if [[ "$license" == "Scopy_GPL" ]]; then @@ -346,16 +336,16 @@ function scan_directory() { # Use the --build option to exclude directories if [ ${#OMITTED_DIRS[@]} -gt 0 ]; then for dir in "${OMITTED_DIRS[@]}"; do - dir_excludes+=(-o -iname "$dir") + dir_excludes+=(-o -iwholename "$dir") done - dir_excludes=(-type d \( -iname "${dir_excludes[@]:2}" \) -prune) + dir_excludes=(-type d \( -iwholename "${dir_excludes[@]:2}" \) -prune) fi # Use the --files option to exclude figles if [ ${#OMITTED_FILES[@]} -gt 0 ]; then for file in "${OMITTED_FILES[@]}"; do - file_excludes+=(-o -iname "$file") + file_excludes+=(-o -iwholename "$file") done - file_excludes=(-type f ! \( -iname "${file_excludes[@]:2}" \)) + file_excludes=(-type f ! \( -iwholename "${file_excludes[@]:2}" \)) fi # Build the find command @@ -393,41 +383,38 @@ function main() { unknown_license_count=0 # Store the number of unique file extensions - if $VERBOSE; then - # Find all files recursively in the base directory and extract unique file extensions - mapfile -t FILE_EXTENSIONS < <(scan_directory "$BASE_DIR_PATH" -type f | awk -F. '{if (NF>1) print $NF}' | sort -u) - echo "Unique file extensions: ${FILE_EXTENSIONS[*]}" - fi + # Find all files recursively in the base directory and extract unique file extensions + mapfile -t FILE_EXTENSIONS < <(scan_directory "$BASE_DIR_PATH" -type f | awk -F. '{if (NF>1) print $NF}' | sort -u) + echo "Unique file extensions: ${FILE_EXTENSIONS[*]}" >&2 + echo "Scanning project for license headers from path: $BASE_DIR_PATH ..." >&2 while read -r file; do if [[ -f "$file" ]]; then identify_license "$file" "$N_LINES" fi done < <(scan_directory "$BASE_DIR_PATH" | sort) - if $VERBOSE; then - echo -e "##################################################" - echo -e "License count summary:" - echo -e "LGPL : [$lgpl_count/$total_checks]" - echo -e "Scopy_GPL: [$scopy_gpl_count/$total_checks]" - echo -e "GPL: [$gpl_count/$total_checks]" - echo -e "ADI-BSD: [$adi_bsd_count/$total_checks]" - echo -e "GNU Radio: [$gr_count/$total_checks]" - echo -e "No-license: [$no_license_count/$total_checks]" - echo -e "Unknown-license: [$unknown_license_count/$total_checks]\n" - echo "Scanning project for license headers from path: $BASE_DIR_PATH ..." >&2 - fi - + echo -e "##################################################" >&2 + echo -e "License count summary:" >&2 + echo -e "LGPL : [$lgpl_count/$total_checks]" >&2 + echo -e "Scopy_GPL: [$scopy_gpl_count/$total_checks]" >&2 + echo -e "GPL: [$gpl_count/$total_checks]" >&2 + echo -e "ADI-BSD: [$adi_bsd_count/$total_checks]" >&2 + echo -e "GNU Radio: [$gr_count/$total_checks]" >&2 + echo -e "No-license: [$no_license_count/$total_checks]" >&2 + echo -e "Unknown-license: [$unknown_license_count/$total_checks]\n" >&2 + echo -e "##################################################" >&2 if [ ${#FILES_WITHOUT_LICENSE[@]} -gt 0 ]; then for file in "${FILES_WITHOUT_LICENSE[@]}"; do echo -e "$BASE_DIR_PATH/$file" done exit 1 + else + echo "All files have a form of license." >&2 fi - - echo "All files have a license." + echo -e "##################################################" >&2 if [ "$total_checks" -ne "$scopy_gpl_count" ]; then - echo "Error: Not all files have the Scopy GPL license." + echo "Error: Not all files have the Scopy GPL license." >&2 exit 1 fi exit 0 diff --git a/tools/license.sh b/tools/license.sh index ef92b4709..65400ce28 100755 --- a/tools/license.sh +++ b/tools/license.sh @@ -6,57 +6,92 @@ CI_SCRIPT=${CI_SCRIPT:-"OFF"} REPOSTORY=$(git rev-parse --show-toplevel) PATH_OPTION=" --path $REPOSTORY" -OMIT_DIRS_OPTION=" --dirs .git*,build,.vscode,android,apple,ci,docs,tmp,tools,windows" -OMIT_FILES_OPTION=" --files .clang-format,.clangformatignore,.cmake-format,.gitignore,.gitmodules,*.md,LICENSE,*.png,*.rst,azure-pipelines.yml,requirements.txt,*.html,*.svg,*.icns,*.ico,*.qmodel,*.ui,*.json,*.qrc,*.ts,*.gif,*.theme,*.ttf,*.zip,*.csv,*.bin,*.xml,*.cmakein" - -VERBOSE="" -CLEANUP="true" -# VERBOSE: Increasese processing time but provides detailed output. Use for manual inspection -# VERBOSE="--verbose" -# CLEANUP="" # true - LICENSE_UTILS_PATH=$REPOSTORY/tools/license-header RESULTS_PATH=$LICENSE_UTILS_PATH/results -cleanup() { - if [ "$CLEANUP" == "true" ] && [ -d "$RESULTS_PATH" ]; then - rm -rf "$RESULTS_PATH" - fi -} +source $REPOSTORY/tools/exclude_list.sh + +# Build options +OMIT_DIRS_OPTION="" +if [ ${#OMIT_DIRS_LIST[@]} -gt 0 ] && [ ${#GITIGNORE_DIRS_LIST[@]} -gt 0 ]; then + OMIT_DIRS_OPTION=" --dirs $( + IFS=, + echo "${OMIT_DIRS_LIST[*]},${GITIGNORE_DIRS_LIST[*]}" + )" +elif [ ${#OMIT_DIRS_LIST[@]} -gt 0 ]; then + OMIT_DIRS_OPTION=" --dirs $( + IFS=, + echo "${OMIT_DIRS_LIST[*]}" + )" +elif [ ${#GITIGNORE_DIRS_LIST[@]} -gt 0 ]; then + OMIT_DIRS_OPTION=" --dirs $( + IFS=, + echo "${GITIGNORE_DIRS_LIST[*]}" + )" +fi + +OMIT_FILES_OPTION="" +if [ ${#OMIT_FILES_LIST[@]} -gt 0 ] && [ ${#GITINORE_FILES_LIST[@]} -gt 0 ]; then + OMIT_FILES_OPTION=" --files $( + IFS=, + echo "${OMIT_FILES_LIST[*]},${GITINORE_FILES_LIST[*]}" + )" +elif [ ${#OMIT_FILES_LIST[@]} -gt 0 ]; then + OMIT_FILES_OPTION=" --files $( + IFS=, + echo "${OMIT_FILES_LIST[*]}" + )" +elif [ ${#GITINORE_FILES_LIST[@]} -gt 0 ]; then + OMIT_FILES_OPTION=" --files $( + IFS=, + echo "${GITINORE_FILES_LIST[*]}" + )" +fi main() { mkdir -p "$RESULTS_PATH" - if [ -f "$RESULTS_PATH/scan.txt" ]; then - rm "$RESULTS_PATH/scan.txt" - fi - touch "$RESULTS_PATH"/scan.txt + [ -f "$RESULTS_PATH/scan_output.txt" ] && rm "$RESULTS_PATH/scan_output.txt" + touch "$RESULTS_PATH"/scan_output.txt + + [ -f "$RESULTS_PATH/log.txt" ] && rm "$RESULTS_PATH/log.txt" + touch "$RESULTS_PATH"/log.txt echo "# Running license scanner utility ..." $LICENSE_UTILS_PATH/scan_missing_headers.sh \ $PATH_OPTION \ $OMIT_DIRS_OPTION \ $OMIT_FILES_OPTION \ - $VERBOSE \ - >$RESULTS_PATH/scan.txt + >$RESULTS_PATH/scan_output.txt 2>$RESULTS_PATH/log.txt EXIT_CODE=$? - echo "## Stored results under: $RESULTS_PATH/scan.txt" + + echo "## Missing header scan paths: $RESULTS_PATH/scan_output.txt" + echo "## Missing headers (detailed logs): $RESULTS_PATH/log.txt" # CI should fail if missing headers are detected if [ "$CI_SCRIPT" == "ON" ]; then + echo "## CI mode: Exiting with scan results" + echo "## Log results:" + cat "$RESULTS_PATH"/log.txt echo "## Report results:" - cat "$RESULTS_PATH"/scan.txt + cat "$RESULTS_PATH"/scan_output.txt + + if [ $EXIT_CODE -ne 0 ]; then + echo "### Error: Missing headers found" + else + echo "### Success: All scanned files contain license headers" + fi exit $EXIT_CODE fi - if [ $EXIT_CODE -ne 0 ] && [ "$VERBOSE" != "--verbose" ]; then - pushd "$LICENSE_UTILS_PATH" || exit 1 + if [ $EXIT_CODE -ne 0 ]; then + pushd "$LICENSE_UTILS_PATH" || exit 1 >/dev/null echo "### Warning: Missing headers found" - + echo "# Adding missing headers ..." TEMPLATE_OPTION=" --template $LICENSE_UTILS_PATH/templates/Scopy/LICENSE" PARAMS_OPTION=" --params $LICENSE_UTILS_PATH/templates/Scopy/params.conf" - $LICENSE_UTILS_PATH/batch_add_license_headers.sh $RESULTS_PATH/scan.txt $TEMPLATE_OPTION $PARAMS_OPTION + $LICENSE_UTILS_PATH/batch_add_license_headers.sh $RESULTS_PATH/scan_output.txt $TEMPLATE_OPTION $PARAMS_OPTION ERROR_CODE=$? - popd || exit 1 + popd || exit 1 >/dev/null if [ $ERROR_CODE -ne 0 ]; then echo "### Error: Failed to add missing headers" @@ -64,14 +99,15 @@ main() { else echo "### Success: Missing headers added" fi - - echo "# Formatting files ..." - "$REPOSTORY"/tools/format.sh >/dev/null 2>&1 - exit 0 + elif [ $EXIT_CODE -eq 0 ]; then + echo "### All scanned files contain license headers ... no action required" else - echo "### Success: All scanned files contain license headers" + echo "### Erorr: Unknown error" + exit 1 fi + echo "# Formatting files ..." + "$REPOSTORY"/tools/format.sh >/dev/null 2>&1 + exit 0 } -trap cleanup EXIT main "$@"