Skip to content

Commit

Permalink
tools/license: add exclude list for license check
Browse files Browse the repository at this point in the history 
- Removed verbose output from license.sh. Now script
will output only paths to files that are missing license headers to stdout.
Logs are directed to stderr.
- Improved error message for unhandled extensions.

Signed-off-by: Adrian Stanea <[email protected]>
  • Loading branch information
@Adrian-Stanea @AlexandraTrifan
Adrian-Stanea authored and AlexandraTrifan committed Nov 8, 2024
1 parent 6e2de66 commit 28143ee
Show file tree
Hide file tree
Showing 5 changed files with 281 additions and 87 deletions.
166 changes: 166 additions & 0 deletions tools/exclude_list.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
#!/bin/bash

# NOTE: paths containing * are treated as regex => escape them with \ (ex: \*)

OMIT_DIRS_LIST=(
.\*
.git\*
.vscode
android
apple
build
ci/\*
docs
tmp
tools
windows
# Directories specific to your local environment that are not part of the repository
venv
)
OMIT_DIRS_LIST=("${OMIT_DIRS_LIST[@]/#/*\/}") # */ prefix to each element
GITIGNORE_DIRS_LIST=(
# .gitinore
CMakeFiles
build\*
deps
iio-emu
html/\*
build_arm64-v8a/\*
.cache/\*
build/\*
.vscode/\*
windows/\*
bin/\*
lib/\*
docs/_build/\*
share/\*
# ci/flatpak/.gitinore
ci/flatpak/build
ci/flatpak/repo
)
GITIGNORE_DIRS_LIST=("${GITIGNORE_DIRS_LIST[@]/#/*\/}") # */ prefix to each element

OMIT_FILES_LIST=(
.clang-format
.clangformatignore
.cmake-format
.gitignore
.gitmodules
LICENSE
\*.md
\*.png
\*.rst
azure-pipelines.yml
requirements.txt
\*.html
\*.svg
\*.icns
\*.ico
\*.qmodel
\*.ui
\*.json
\*.qrc
\*.ts
\*.gif
\*.theme
\*.ttf
\*.zip
\*.csv
\*.bin
\*.xml
\*.cmakein
# Files specific to your local environment that are not part of the repository
\*.build
\*.git
\*.gitrepo
\*.mat
\*.user
)
OMIT_FILES_LIST=("${OMIT_FILES_LIST[@]/#/*\/}")
GITINORE_FILES_LIST=(
# .gitinore
CMakeLists.txt.user
CMakeCache.txt
cmake_install.cmake
Makefile
moc_\*.cpp
\*_automoc.cpp
ui_\*.h
resources/scopy_osp.html
resources/stylesheets/default.qss
resources/stylesheets/light.qss
resources/credits.html
resources/scopy_home.html
resources/about.html
\*qt.conf
\*.swp
\*.DS_Store
android-build
android\*.sh
\*.apk
\*.aab
ci/general/gh-actions.envs
core/include/scopy-core_config.h
core/include/scopy-core_export.h
core/include/core/scopy-core_config.h
core/include/core/scopy-core_export.h
common/include/common/scopy-common_config.h
common/include/common/scopy-common_export.h
gr-util/include/gr-util/scopy-gr-util_export.h
gui/include/gui/scopy-gui_export.h
gui/include/gui/scopy-gui_config.h
iio-widgets/include/iio-widgets/scopy-iio-widgets_export.h
iioutil/include/iioutil/scopy-iioutil_export.h
pluginbase/include/pluginbase/scopy-pluginbase_config.h
pluginbase/include/pluginbase/scopy-pluginbase_export.h
# gui/res/.gitinore
gui/res/about.html
gui/res/scopy_osp.html
gui/res/buildinfo.html
gui/res/credits.html
# ci/flatpak/.gitinore
ci/flatpak/Scopy.flatpak
ci/flatpak/.flatpak-builder
ci/flatpak/org.adi.Scopy.json
ci/flatpak/tmp.json
# plugins/adc/.gitinore
plugins/adc/include/adc/scopy-adc_export.h
plugins/adc/include/adc/scopy-adc_config.h
# plugins/dac/.gitinore
plugins/dac/include/dac/scopy-dac_export.h
plugins/dac/include/dac/scopy-dac_config.h
# plugins/m2k/.gitinore
plugins/m2k/include/m2k/scopy-m2k_export.h
plugins/m2k/include/m2k/scopy-m2k_config.h
# plugins/pqm/.gitinore
plugins/pqm/include/pqm/scopy-pqm_export.h
plugins/pqm/include/pqm/scopy-pqm_config.h
# plugins/test/.gitinore
plugins/test/include/test/scopy-test_export.h
# plugins/swiot/.gitinore
plugins/swiot/include/swiot/scopy-swiot_export.h
plugins/swiot/include/swiot/scopy-swiot_config.h
# plugins/test2/.gitinore
plugins/test2/include/test2/scopy-test2_export.h
# plugins/guitest/.gitinore
plugins/guitest/include/guitest/scopy-guitest_export.h
# plugins/debugger/.gitinore
plugins/debugger/include/debugger/scopy-debugger_export.h
plugins/debugger/include/debugger/scopy-debugger_config.h
# plugins/datalogger/.gitinore
plugins/datalogger/include/datalogger/scopy-datalogger_export.h
plugins/datalogger/include/datalogger/scopy-datalogger_config.h
# plugins/bareminimum/.gitinore
plugins/bareminimum/include/bareminimum/scopy-bareminimum_export.h
# plugins/m2k/m2k-gui/.gitinore
plugins/m2k/m2k-gui/include/m2k-gui/scopy-m2k-gui_export.h
plugins/m2k/m2k-gui/include/m2k-gui/scopy-m2k-gui_config.h
plugins/m2k/m2k-gui/gr-gui/include/gr-gui/scopy-gr-gui_export.h
plugins/m2k/m2k-gui/gr-gui/include/gr-gui/scopy-gr-gui_config.h
plugins/m2k/m2k-gui/sigrok-gui/include/sigrok-gui/scopy-sigrok-gui_export.h
plugins/m2k/m2k-gui/sigrok-gui/include/sigrok-gui/scopy-sigrok-gui_config.h
# plugins/regmap/.gitinore
plugins/regmap/include/regmap/scopy-regmap_export.h
plugins/regmap/include/regmap/scopy-regmap_config.h
)
GITINORE_FILES_LIST=("${GITINORE_FILES_LIST[@]/#/*\/}")
5 changes: 3 additions & 2 deletions tools/license-header/add_license_header.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,9 +240,10 @@ function main() {

FILE_EXTENSION="${FILE##*.}"

COMMENT_STYLE=${COMMENT_STYLES[$FILE_EXTENSION]}
COMMENT_STYLE=${COMMENT_STYLES[$FILE_EXTENSION]:-}
if [[ -z "$COMMENT_STYLE" ]]; then
echo "Unsupported file extension: $FILE_EXTENSION" >&2
echo "Error: Unsupported file extension '${FILE_EXTENSION}' found in file '${FILE}'" >&2
echo "Define 'COMMENT_STYLE' for '$FILE_EXTENSION' in .comment_styles.conf" >&2
exit 1
fi

Expand Down
4 changes: 4 additions & 0 deletions tools/license-header/batch_add_license_headers.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,10 @@ function process_files() {
"$add_license_header" "$file" \
--template "$TEMPLATE_FILE" \
--params "$PARAMS_FILE"
if [[ $? -ne 0 ]]; then
echo "Error: Failed to add license header to $file"
exit 1
fi
done <"$file_list"
}

Expand Down
91 changes: 39 additions & 52 deletions tools/license-header/scan_missing_headers.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ BASE_DIR_PATH=""
FILES_WITHOUT_LICENSE=()
FILE_EXTENSIONS=()
N_LINES=80 # When searching for license headers, use the first N lines of the file
VERBOSE=false

# Define colors used for logging
NC='\033[0m' # No Color
Expand Down Expand Up @@ -38,14 +37,12 @@ Options:
-h, --help Display this help message.
-v, --verbose Display verbose output.
EOF
}

function parse_arguments() {
LONG_OPTS=dirs:,files:,path:,help,verbose
OPTIONS=d:f:p:h:v
LONG_OPTS=dirs:,files:,path:,help
OPTIONS=d:f:p:h
VALID_ARGS=$(getopt --options=$OPTIONS --longoptions=$LONG_OPTS --name "$0" -- "$@")

if [[ $? -ne 0 ]]; then
Expand All @@ -56,7 +53,7 @@ function parse_arguments() {

getopt --test >/dev/null && true
if [[ $? -ne 4 ]]; then
echo "$(getopt --test) failed in this environment."
echo "$(getopt --test) failed in this environment." >&2
exit 1
fi

Expand Down Expand Up @@ -84,10 +81,6 @@ function parse_arguments() {
BASE_DIR_PATH="$2"
shift 2
;;
-v | --verbose)
VERBOSE=true
shift
;;
--)
shift
break
Expand Down Expand Up @@ -248,17 +241,16 @@ function identify_license() {
depth=$(echo "$relative_path" | awk -F'/' '{print NF-1}')
indent=$(printf "%*s" $((depth * 4)) "")

local header=$(read_header "$file" "$n_lines" | normalize_text)
# Sanitize the header text -> remove null characters and normalize the text
local header=$(read_header "$file" "$n_lines" | tr -d '\0' | normalize_text)

((total_checks++))

if ! has_license_disclaimer "$header"; then
((no_license_count++))
FILES_WITHOUT_LICENSE+=("$relative_path")

if $VERBOSE; then
echo -e "${indent}${RED}├── $relative_path (No License)${NC}"
fi
echo -e "${indent}${RED}├── $relative_path (No License)${NC}" >&2
# Stop execution if no license is found
return
fi
Expand Down Expand Up @@ -307,22 +299,20 @@ function identify_license() {
((unknown_license_count++))
fi

if $VERBOSE; then
# Join the elements of found_licenses_list with a | separator
local joined_licenses=$(
IFS='|'
echo "${found_licenses_list[*]}"
)

# Check if the joined string is empty
if [ -z "$joined_licenses" ]; then
joined_licenses="Unknown"
fi
# Join the elements of found_licenses_list with a | separator
local joined_licenses=$(
IFS='|'
echo "${found_licenses_list[*]}"
)

# Update the echo command to use the joined string
echo -e "${indent}${color}├── $relative_path (${joined_licenses})${NC}"
# Check if the joined string is empty
if [ -z "$joined_licenses" ]; then
joined_licenses="Unknown"
fi

# Update the echo command to use the joined string
echo -e "${indent}${color}├── $relative_path (${joined_licenses})${NC}" >&2

local found=false
for license in "${found_licenses_list[@]}"; do
if [[ "$license" == "Scopy_GPL" ]]; then
Expand All @@ -346,16 +336,16 @@ function scan_directory() {
# Use the --build option to exclude directories
if [ ${#OMITTED_DIRS[@]} -gt 0 ]; then
for dir in "${OMITTED_DIRS[@]}"; do
dir_excludes+=(-o -iname "$dir")
dir_excludes+=(-o -iwholename "$dir")
done
dir_excludes=(-type d \( -iname "${dir_excludes[@]:2}" \) -prune)
dir_excludes=(-type d \( -iwholename "${dir_excludes[@]:2}" \) -prune)
fi
# Use the --files option to exclude figles
if [ ${#OMITTED_FILES[@]} -gt 0 ]; then
for file in "${OMITTED_FILES[@]}"; do
file_excludes+=(-o -iname "$file")
file_excludes+=(-o -iwholename "$file")
done
file_excludes=(-type f ! \( -iname "${file_excludes[@]:2}" \))
file_excludes=(-type f ! \( -iwholename "${file_excludes[@]:2}" \))
fi

# Build the find command
Expand Down Expand Up @@ -393,41 +383,38 @@ function main() {
unknown_license_count=0

# Store the number of unique file extensions
if $VERBOSE; then
# Find all files recursively in the base directory and extract unique file extensions
mapfile -t FILE_EXTENSIONS < <(scan_directory "$BASE_DIR_PATH" -type f | awk -F. '{if (NF>1) print $NF}' | sort -u)
echo "Unique file extensions: ${FILE_EXTENSIONS[*]}"
fi
# Find all files recursively in the base directory and extract unique file extensions
mapfile -t FILE_EXTENSIONS < <(scan_directory "$BASE_DIR_PATH" -type f | awk -F. '{if (NF>1) print $NF}' | sort -u)
echo "Unique file extensions: ${FILE_EXTENSIONS[*]}" >&2

echo "Scanning project for license headers from path: $BASE_DIR_PATH ..." >&2
while read -r file; do
if [[ -f "$file" ]]; then
identify_license "$file" "$N_LINES"
fi
done < <(scan_directory "$BASE_DIR_PATH" | sort)

if $VERBOSE; then
echo -e "##################################################"
echo -e "License count summary:"
echo -e "LGPL : [$lgpl_count/$total_checks]"
echo -e "Scopy_GPL: [$scopy_gpl_count/$total_checks]"
echo -e "GPL: [$gpl_count/$total_checks]"
echo -e "ADI-BSD: [$adi_bsd_count/$total_checks]"
echo -e "GNU Radio: [$gr_count/$total_checks]"
echo -e "No-license: [$no_license_count/$total_checks]"
echo -e "Unknown-license: [$unknown_license_count/$total_checks]\n"
echo "Scanning project for license headers from path: $BASE_DIR_PATH ..." >&2
fi

echo -e "##################################################" >&2
echo -e "License count summary:" >&2
echo -e "LGPL : [$lgpl_count/$total_checks]" >&2
echo -e "Scopy_GPL: [$scopy_gpl_count/$total_checks]" >&2
echo -e "GPL: [$gpl_count/$total_checks]" >&2
echo -e "ADI-BSD: [$adi_bsd_count/$total_checks]" >&2
echo -e "GNU Radio: [$gr_count/$total_checks]" >&2
echo -e "No-license: [$no_license_count/$total_checks]" >&2
echo -e "Unknown-license: [$unknown_license_count/$total_checks]\n" >&2
echo -e "##################################################" >&2
if [ ${#FILES_WITHOUT_LICENSE[@]} -gt 0 ]; then
for file in "${FILES_WITHOUT_LICENSE[@]}"; do
echo -e "$BASE_DIR_PATH/$file"
done
exit 1
else
echo "All files have a form of license." >&2
fi

echo "All files have a license."
echo -e "##################################################" >&2
if [ "$total_checks" -ne "$scopy_gpl_count" ]; then
echo "Error: Not all files have the Scopy GPL license."
echo "Error: Not all files have the Scopy GPL license." >&2
exit 1
fi
exit 0
Expand Down
Loading

0 comments on commit 28143ee

Please sign in to comment.