From d1e8aad8990fc50991c8b56576a08db5be50ea5c Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Wed, 23 Oct 2024 09:36:49 +0200 Subject: [PATCH] Add pre-commit config, incl. fixes --- .github/workflows/main.yml | 10 +- .markdownlint.yaml | 268 +++++++++++++++++++ .pre-commit-config.yaml | 90 +++++++ .yamllint | 54 ++++ README.md | 43 ++- ci/azure-pipelines.example.yml | 169 ++++++------ ci/azure-pipelines.multiple_pkgs.example.yml | 32 ++- ci/azure_commit_range.py | 10 +- ci/commit-range/action.yml | 5 +- ci/main/action.yml | 6 + ci/modified-packages/action.yml | 19 +- ci/organization-packages.sh | 0 ci/organization-packages/action.yaml | 5 +- ci/travis.example.yml | 4 +- ci/travis.multiple_pkgs.example.yml | 4 + docs/CI_Travis_Setup.md | 2 + installer/lsb_release.py | 0 installer/tue-get-dep.bash | 4 +- pyproject.toml | 3 + setup/tue-env-config.bash | 6 +- 20 files changed, 626 insertions(+), 108 deletions(-) create mode 100644 .markdownlint.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint mode change 100644 => 100755 ci/azure_commit_range.py mode change 100644 => 100755 ci/organization-packages.sh mode change 100644 => 100755 installer/lsb_release.py create mode 100644 pyproject.toml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22ea2a8f7..df2ac8f23 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,6 +92,8 @@ jobs: run: docker info - name: Script id: script + # yamllint disable rule:indentation + # yamllint disable rule:line-length run: | #! /usr/bin/env bash GITHUB_REF=${GITHUB_REF#refs/heads/} @@ -174,6 +176,8 @@ jobs: echo "image=${TARGET_IMAGE}" >> "${GITHUB_OUTPUT}" echo -e "\e[35;1mtag=${IMAGE_TAG} >> \${GITHUB_OUTPUT}\e[0m" echo "tag=${IMAGE_TAG}" >> "${GITHUB_OUTPUT}" + # yamllint enable rule:line-length + # yamllint enable rule:indentation - name: Generate artifacts id: artifacts if: github.event_name != 'pull_request' @@ -207,12 +211,16 @@ jobs: - uses: actions/download-artifact@v4 - name: Read matrix output id: read + # yamllint disable rule:line-length run: | result="$(find . -name ${{ needs.docker_generation_tue_env.outputs.key }} -exec cat {} \; | jq --slurp --raw-input 'split("\n")[:-1] | .[] | split(":") | {(.[0]) : [(.[1])]}' | jq -cs '[.[] | to_entries] | flatten | reduce .[] as $dot ({}; .[$dot.key] += $dot.value) | { ${{ needs.docker_generation_tue_env.outputs.key }} : . }' )" echo -e "\e[35;1mresult=${result} >> \${GITHUB_OUTPUT}\e[0m" - echo "result=${result}" >> $GITHUB_OUTPUT + echo "result=${result}" >> ${GITHUB_OUTPUT} + # yamllint enable rule:line-length - name: Generate new manifest files + # yamllint disable rule:indentation run: | ci/docker-manifest-generator.sh \ --images-json='${{ toJSON(steps.read.outputs.result) }}' \ --tag="${{ needs.docker_generation_tue_env.outputs.tag }}" + # yamllint enable rule:indentation diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 000000000..99f0e7b65 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,268 @@ +# yamllint disable rule:line-length +# Example markdownlint configuration with all properties set to their default value + +# Default state for all rules +default: true + +# Path to configuration file to extend +extends: null + +# MD001/heading-increment: Heading levels should only increment by one level at a time: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md001.md +MD001: true + +# MD003/heading-style: Heading style: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md003.md +MD003: + # Heading style + style: "consistent" + +# MD004/ul-style: Unordered list style: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md004.md +MD004: + # List style + style: "consistent" + +# MD005/list-indent: Inconsistent indentation for list items at the same level: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md005.md +MD005: true + +# MD007/ul-indent: Unordered list indentation: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md007.md +MD007: + # Spaces for indent + indent: 2 + # Whether to indent the first level of the list + start_indented: false + # Spaces for first level indent (when start_indented is set) + start_indent: 2 + +# MD009/no-trailing-spaces: Trailing spaces: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md009.md +MD009: + # Spaces for line break + br_spaces: 2 + # Allow spaces for empty lines in list items + list_item_empty_lines: false + # Include unnecessary breaks + strict: false + +# MD010/no-hard-tabs: Hard tabs: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md010.md +MD010: + # Include code blocks + code_blocks: true + # Fenced code languages to ignore + ignore_code_languages: [] + # Number of spaces for each hard tab + spaces_per_tab: 1 + +# MD011/no-reversed-links: Reversed link syntax: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md011.md +MD011: true + +# MD012/no-multiple-blanks: Multiple consecutive blank lines: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md012.md +MD012: + # Consecutive blank lines + maximum: 1 + +# MD013/line-length: Line length: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md +MD013: + # Number of characters + line_length: 120 + # Number of characters for headings + heading_line_length: 120 + # Number of characters for code blocks + code_block_line_length: 120 + # Include code blocks + code_blocks: true + # Include tables + tables: true + # Include headings + headings: true + # Strict length checking + strict: false + # Stern length checking + stern: false + +# MD014/commands-show-output: Dollar signs used before commands without showing output: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md014.md +MD014: true + +# MD018/no-missing-space-atx: No space after hash on atx style heading: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md018.md +MD018: true + +# MD019/no-multiple-space-atx: Multiple spaces after hash on atx style heading: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md019.md +MD019: true + +# MD020/no-missing-space-closed-atx: No space inside hashes on closed atx style heading: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md020.md +MD020: true + +# MD021/no-multiple-space-closed-atx: Multiple spaces inside hashes on closed atx style heading: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md021.md +MD021: true + +# MD022/blanks-around-headings: Headings should be surrounded by blank lines: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md022.md +MD022: + # Blank lines above heading + lines_above: 1 + # Blank lines below heading + lines_below: 1 + +# MD023/heading-start-left: Headings must start at the beginning of the line: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md023.md +MD023: true + +# MD024/no-duplicate-heading: Multiple headings with the same content: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md024.md +MD024: + # Only check sibling headings + allow_different_nesting: false + # Only check sibling headings + siblings_only: false + +# MD025/single-title/single-h1: Multiple top-level headings in the same document: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md025.md +MD025: + # Heading level + level: 1 + # RegExp for matching title in front matter + front_matter_title: "^\\s*title\\s*[:=]" + +# MD026/no-trailing-punctuation: Trailing punctuation in heading: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md026.md +MD026: + # Punctuation characters + punctuation: ".,;:!。，；：！" + +# MD027/no-multiple-space-blockquote: Multiple spaces after blockquote symbol: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md027.md +MD027: true + +# MD028/no-blanks-blockquote: Blank line inside blockquote: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md028.md +MD028: true + +# MD029/ol-prefix: Ordered list item prefix: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md029.md +MD029: + # List style + style: "one_or_ordered" + +# MD030/list-marker-space: Spaces after list markers: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md030.md +MD030: + # Spaces for single-line unordered list items + ul_single: 1 + # Spaces for single-line ordered list items + ol_single: 1 + # Spaces for multi-line unordered list items + ul_multi: 1 + # Spaces for multi-line ordered list items + ol_multi: 1 + +# MD031/blanks-around-fences: Fenced code blocks should be surrounded by blank lines: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md031.md +MD031: + # Include list items + list_items: true + +# MD032/blanks-around-lists: Lists should be surrounded by blank lines: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md032.md +MD032: true + +# MD033/no-inline-html: Inline HTML: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md033.md +MD033: + # Allowed elements + allowed_elements: [] + +# MD034/no-bare-urls: Bare URL used: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md034.md +MD034: true + +# MD035/hr-style: Horizontal rule style: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md035.md +MD035: + # Horizontal rule style + style: "consistent" + +# MD036/no-emphasis-as-heading: Emphasis used instead of a heading: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md036.md +MD036: + # Punctuation characters + punctuation: ".,;:!?。，；：！？" + +# MD037/no-space-in-emphasis: Spaces inside emphasis markers: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md037.md +MD037: true + +# MD038/no-space-in-code: Spaces inside code span elements: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md038.md +MD038: true + +# MD039/no-space-in-links: Spaces inside link text: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md039.md +MD039: true + +# MD040/fenced-code-language: Fenced code blocks should have a language specified: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md040.md +MD040: + # List of languages + allowed_languages: [] + # Require language only + language_only: false + +# MD041/first-line-heading/first-line-h1: First line in a file should be a top-level heading: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md041.md +MD041: + # Heading level + level: 1 + # RegExp for matching title in front matter + front_matter_title: "^\\s*title\\s*[:=]" + +# MD042/no-empty-links: No empty links: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md042.md +MD042: true + +# MD043/required-headings: Required heading structure: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md043.md +MD043: false +#MD043: +# # List of headings +# headings: [] +# # Match case of headings +# match_case: false + +# MD044/proper-names: Proper names should have the correct capitalization: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md044.md +MD044: + # List of proper names + names: [] + # Include code blocks + code_blocks: true + # Include HTML elements + html_elements: true + +# MD045/no-alt-text: Images should have alternate text (alt text): https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md045.md +MD045: true + +# MD046/code-block-style: Code block style: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md046.md +MD046: + # Block style + style: "consistent" + +# MD047/single-trailing-newline: Files should end with a single newline character: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md047.md +MD047: true + +# MD048/code-fence-style: Code fence style: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md048.md +MD048: + # Code fence style + style: "consistent" + +# MD049/emphasis-style: Emphasis style: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md049.md +MD049: + # Emphasis style + style: "consistent" + +# MD050/strong-style: Strong style: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md050.md +MD050: + # Strong style + style: "consistent" + +# MD051/link-fragments: Link fragments should be valid: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md051.md +MD051: true + +# MD052/reference-links-images: Reference links and images should use a label that is defined: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md052.md +MD052: + # Include shortcut syntax + shortcut_syntax: false + +# MD053/link-image-reference-definitions: Link and image reference definitions should be needed: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md053.md +MD053: + # Ignored definitions + ignored_definitions: + - "//" + +# MD054/link-image-style: Link and image style: https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md054.md +MD054: + # Allow auto-links + autolink: true + # Allow inline links and images + inline: true + # Allow full reference links and images + full: true + # Allow collapsed reference links and images + collapsed: true + # Allow shortcut reference links and images + shortcut: true + # Allow URLs as inline links + url_inline: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..df08fa0f1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,90 @@ +ci: + # autofix_commit_msg: | + # [pre-commit.ci] auto fixes from pre-commit.com hooks + # + # for more information, see https://pre-commit.ci + autofix_prs: false + autoupdate_branch: "" + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-builtin-literals + # - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + # - id: check-illegal-windows-names + - id: check-json + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + exclude: '(^setup\.bash$|^setup/tue-data.bash$|^setup/tue-env.bash$|^setup/tue-functions.bash$|^setup/tue-misc.bash$)' + - id: check-symlinks + - id: check-toml + - id: check-vcs-permalinks + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: destroyed-symlinks + # - id: detect-aws-credentials + - id: detect-private-key + # - id: double-quote-string-fixer + - id: end-of-file-fixer + # - id: file-contents-sorter + # files: [] + - id: fix-byte-order-marker + - id: fix-encoding-pragma + args: [--remove] + # - id: forbid-new-submodules + # - id: forbid-submodules + - id: mixed-line-ending + args: ["--fix", "no"] + # - id: name-tests-test + # args: [ + # "--pytest-test-first" # test_.*\.py + # # "--pytest" # .*_test\.py + # # "--unittest" # test.*\.py + # ] + - id: no-commit-to-branch + - id: pretty-format-json + # - id: requirements-txt-fixer + # - id: sort-simple-yaml + # files: [] + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + args: [--check, --diff, --color] + + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.29.0 + hooks: + - id: yamllint + args: [--strict] + + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.13.0 + hooks: + - id: markdownlint-cli2 + args: ["--fix", "--config", ".markdownlint.yaml"] + exclude: '^LICENSE.md$' + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + files: '\S\.(bash|sh)' + + - repo: https://github.com/pre-commit-ci/pre-commit-ci-config + rev: v1.6.1 + hooks: + - id: check-pre-commit-ci-config diff --git a/.yamllint b/.yamllint new file mode 100644 index 000000000..7cecaee05 --- /dev/null +++ b/.yamllint @@ -0,0 +1,54 @@ +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + colons: + max-spaces-before: 0 + max-spaces-after: 1 + commas: + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: + require-starting-space: false + min-spaces-from-content: 1 + level: warning + comments-indentation: + level: warning + document-end: disable + document-start: + present: false + empty-lines: + max: 2 + max-start: 0 + max-end: 1 + # empty-values: + # forbid-in-block-mappings: true + # forbid-in-flow-mappings: true + hyphens: + max-spaces-after: 1 + indentation: + check-multi-line-strings: true + indent-sequences: consistent + spaces: 2 + key-duplicates: enable + # key-ordering: disable + line-length: + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: true + max: 120 + new-line-at-end-of-file: enable + new-lines: + type: unix + # octal-values: disable + # quoted-strings: disable + trailing-spaces: enable + # truthy: + # level: warning diff --git a/README.md b/README.md index dc394184f..91387e50d 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,20 @@ Standard `tue-env` installation with targets from [tue-env-targets](https://gith 1. Bootstrap the package manager For default ROS1 + + ```bash source <(wget -O - https://raw.githubusercontent.com/tue-robotics/tue-env/master/installer/bootstrap.bash) # for default ROS1 ``` + + Or for ROS2 + + ```bash source <(wget -O - https://raw.githubusercontent.com/tue-robotics/tue-env/master/installer/bootstrap.bash) --ros-version=2 # for ROS2 ``` + 2. Install target(s) @@ -40,7 +47,8 @@ Standard `tue-env` installation with targets from [tue-env-targets](https://gith #### Virtual environment -When a new environment is created, a python virtual environment is created by default. This can be disabled by adding `--create-virtualenv=false` to the bootstrap script. +When a new environment is created, a python virtual environment is created by default. +This can be disabled by adding `--create-virtualenv=false` to the bootstrap script. When you want to remove the virtual environment from an existing environment, you can run: @@ -76,7 +84,10 @@ This will rename the old targets folder with a timestamp and clone the new targe #### Custom repository list -Various utility commands use `_tue-repos-do` to execute a command in all repositories. By default, this command will execute the command in all repositories in the `tue-env` workspace. This can be changed by setting the `TUE_ENV_REPOS_DO_DIRS` environment variable to a list of repositories folders. This can be done by adding the setting it with: +Various utility commands use `_tue-repos-do` to execute a command in all repositories. +By default, this command will execute the command in all repositories in the `tue-env` workspace. +This can be changed by setting the `TUE_ENV_REPOS_DO_DIRS` environment variable to a list of repositories folders. +This can be done by adding the setting it with: ```bash tue-env config [environment] set TUE_ENV_REPOS_DO_DIRS PATH1:PATH2:PATH3 @@ -86,7 +97,8 @@ tue-env config [environment] set TUE_ENV_REPOS_DO_DIRS '${TUE_ENV_REPOS_DIR}'/so #### Add an SSH key to GitHub to gain access to this repository -Add the public part of your ssh-key (`cat ~/.ssh/.pub`, where `` is the name of your ssh-key) to GitHub > Settings > SSH and GPG keys and `New SSH key` +Add the public part of your ssh-key (`cat ~/.ssh/.pub`, +where `` is the name of your ssh-key) to GitHub > Settings > SSH and GPG keys and `New SSH key` To generate a new ssh keypair: @@ -118,13 +130,15 @@ tue-get install hero-dev # For Hero In the current implementation of `tue-get` this doesn't happen automatically. However, we provide an alias to `catkin build`/`colcon build` as `tue-make` which would build the `tue-env` workspace. -Upon executing the installation instructions mentioned in the previous section, `~/.tue/setup.bash` is automatically added in `.bashrc`. Sourcing `.bashrc` would make `tue-env` available to the bash session. +Upon executing the installation instructions mentioned in the previous section, `~/.tue/setup.bash` is automatically +added in `.bashrc`. Sourcing `.bashrc` would make `tue-env` available to the bash session. ### Pre-built docker images This repository provides various pre-built docker images. To use them, first install the docker target (`tue-get install docker`). -A list of ready to use docker images generated by this repository can be found [here](https://github.com/orgs/tue-robotics/packages) +A list of ready to use docker images generated by this repository can be found +[here](https://github.com/orgs/tue-robotics/packages) ## Different environments @@ -149,7 +163,8 @@ Or use `tue-env set-default` if you want this to be the default. ## Guidelines on creating a new target -The targets directory is located at `.env/targets` relative to the root of the environment directory. The target directory of each environment can be accessed by: +The targets directory is located at `.env/targets` relative to the root of the environment directory. +The target directory of each environment can be accessed by: ```bash tue-env targets # Will change the directory to the targets directory of the current environment @@ -182,7 +197,8 @@ ignore the bash script or the YAML file add respectively `.ci_ignore_bash` or `. ### Naming conventions -Name of the target must start with `ros-` only if it is a `catkin`/`colcon` (ROS) package. It's `install.yaml` file must be in the format of [ROS target](#ros-package-install). +Name of the target must start with `ros-` only if it is a `catkin`/`colcon` (ROS) package. +It's `install.yaml` file must be in the format of [ROS target](#ros-package-install). ### Writing `install.yaml` @@ -233,7 +249,8 @@ Taking the above into account, the following combinations for `install.yaml` are source: null ``` -Both ROS distro specific as default can be 'null'. Preferred usage is default for current and feature distributions and exceptions for old distributions. +Both ROS distro specific as default can be 'null'. +Preferred usage is default for current and feature distributions and exceptions for old distributions. ##### ROS package source removal @@ -302,7 +319,9 @@ Preferred usage is default for current and feature distributions and exceptions #### (Target / System / PIP / PIP3 / PPA / Snap / Gem)-now -The default installation method for targets of type `system`, `pip(3)`, `ppa`, `snap` and `gem` is to collect all such targets in a list and install them simultaneously at the end of the `tue-get install` procedure. To install such a dependency immediately for a specific target, use the target type as `X-now`: +The default installation method for targets of type `system`, `pip(3)`, `ppa`, `snap` and `gem` is to collect all such +targets in a list and install them simultaneously at the end of the `tue-get install` procedure. +To install such a dependency immediately for a specific target, use the target type as `X-now`: ```yaml - type: [target/system/pip/pip3/ppa/snap/gem]-now @@ -333,6 +352,7 @@ Only use the corresponding bash function in `install.bash` if no other solution Add an APT source including the key which it is signed with. + ```yaml - type: apt-key-source distribution-extensions: [, [, ..., ]] (Minimal one extension required, use "" for no extension, just distribution name) @@ -343,6 +363,7 @@ Add an APT source including the key which it is signed with. repo-url: source-file: or ``` + ### Writing `install.bash` @@ -365,6 +386,7 @@ The use of the following variables is prohibited in `install.bash`: The following functions provided with `tue-env` must be preferred over any generally used methods of installing packages: + | Function Name | Description | |----------------------------------|------------------------------------------------------------------------------------------------------------| | `tue-install-add-text` | To add/replace text in a file with `sudo` taken into account | @@ -388,6 +410,7 @@ generally used methods of installing packages: | `tue-install-system` | To add `deb` package to a list of packages to be installed at the end with `apt-get` | | `tue-install-system-now` | To install `deb` packages with `apt-get` right away, but ignores it if already installed | | `tue-install-get-releases` | To get a released asset from a github repository and place it in the requested directory | + The input arguments for each of the above-mentioned commands can be found by simply executing the command in a bash session (provided `tue-env` is correctly @@ -401,6 +424,7 @@ system > ppa > apt-key-source > pip = pip3 > snap > gem > git > dpkg The following logging functions can be used: + | Function Name | Description | |-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `tue-install-debug` | Labeled print to the log file; Also prints to stdout when running `tue-get` with `--debug` | @@ -410,6 +434,7 @@ The following logging functions can be used: | `tue-install-echo` | Labeled echo to log file and stdout | | `tue-install-tee` | Plain print to log file and stdout | | `tue-install-pipe` | Executes the command with its arguments. Both stdout and stderr are captured and printed to the log file and stdout. The stderr is converted to red. Return code of the command is preserved | + ## CI diff --git a/ci/azure-pipelines.example.yml b/ci/azure-pipelines.example.yml index 9d806d510..1866ca688 100644 --- a/ci/azure-pipelines.example.yml +++ b/ci/azure-pipelines.example.yml @@ -15,83 +15,92 @@ pool: vmImage: ubuntu-latest steps: -- checkout: none -- task: Cache@2 - displayName: Cache ccache - inputs: - key: '$(Agent.OS) | ccache | $(System.TeamProject) | $(PACKAGE) | $(Build.SourceVersion)' - restoreKeys: | - $(Agent.OS) | ccache | $(System.TeamProject) | $(PACKAGE) - $(Agent.OS) | ccache | $(System.TeamProject) - $(Agent.OS) | ccache - path: $(HOME)/.ccache -- task: Cache@2 - displayName: Cache pip - inputs: - key: '$(Agent.OS) | pip | $(System.TeamProject) | $(PACKAGE) | $(Build.SourceVersion)' - restoreKeys: | - $(Agent.OS) | pip | $(System.TeamProject) | $(PACKAGE) - $(Agent.OS) | pip | $(System.TeamProject) - $(Agent.OS) | pip - path: $(HOME)/.cache/pip -- task: Bash@3 - displayName: Get CI scripts - inputs: - targetType: inline - script: | - wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/install-package.sh - wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/build-package.sh - wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/test-package.sh - wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/set-branch.sh - echo "##vso[task.setvariable variable=package]${BUILD_REPOSITORY_NAME#*/}" - condition: and(succeeded(), not(canceled())) -- task: Bash@3 - displayName: Set Branch - inputs: - targetType: inline - script: | - source ./set-branch.sh --branch=${SYSTEM_PULLREQUEST_TARGETBRANCH:-${BUILD_SOURCEBRANCH#refs/heads/}} - echo "##vso[task.setvariable variable=BRANCH]$BRANCH" - condition: and(succeeded(), not(canceled())) -- task: DownloadSecureFile@1 # Only when using SSH - name: deployKey - displayName: 'Download Deploy Key' - inputs: - secureFile: deploy_key - condition: and(succeeded(), not(canceled())) -- task: Bash@3 - displayName: Install - env: - SSH_KNOWN_HOSTS: "github.com" # Only when using SSH - inputs: - targetType: inline - script: | - # START SSH ONLY - echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m" - mkdir -p "${HOME}"/.ssh - cp $(deployKey.secureFilePath) "${HOME}"/.ssh/ci_ssh_key - if [[ -n "${SSH_KNOWN_HOSTS}" ]] - then - for host in ${SSH_KNOWN_HOSTS} - do - echo -e "\e[35;1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m" - ssh-keyscan -t rsa -H "${host}" 2>&1 | tee -a "${HOME}"/.ssh/known_hosts - done - fi - # END SSH ONLY - bash install-package.sh --package=$PACKAGE --branch=$BRANCH --commit=$BUILD_SOURCEVERSION --pullrequest=${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-false} --image="ghcr.io/tue-robotics/tue-env-ros-noetic" --ssh --ssh-key="${HOME}"/.ssh/ci_ssh_key # Last two arguments only when using SSH - condition: and(succeeded(), not(canceled())) -- task: Bash@3 - displayName: Build - inputs: - targetType: inline - script: | - bash build-package.sh --package=$PACKAGE - condition: and(succeeded(), not(canceled())) -- task: Bash@3 - displayName: Test - inputs: - targetType: inline - script: | - bash test-package.sh --package=$PACKAGE - condition: and(succeeded(), not(canceled())) + - checkout: none + - task: Cache@2 + displayName: Cache ccache + inputs: + key: '$(Agent.OS) | ccache | $(System.TeamProject) | $(PACKAGE) | $(Build.SourceVersion)' + restoreKeys: | + $(Agent.OS) | ccache | $(System.TeamProject) | $(PACKAGE) + $(Agent.OS) | ccache | $(System.TeamProject) + $(Agent.OS) | ccache + path: $(HOME)/.ccache + - task: Cache@2 + displayName: Cache pip + inputs: + key: '$(Agent.OS) | pip | $(System.TeamProject) | $(PACKAGE) | $(Build.SourceVersion)' + restoreKeys: | + $(Agent.OS) | pip | $(System.TeamProject) | $(PACKAGE) + $(Agent.OS) | pip | $(System.TeamProject) + $(Agent.OS) | pip + path: $(HOME)/.cache/pip + - task: Bash@3 + displayName: Get CI scripts + inputs: + targetType: inline + script: | + wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/install-package.sh + wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/build-package.sh + wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/test-package.sh + wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/set-branch.sh + echo "##vso[task.setvariable variable=package]${BUILD_REPOSITORY_NAME#*/}" + condition: and(succeeded(), not(canceled())) + - task: Bash@3 + displayName: Set Branch + inputs: + targetType: inline + script: | + source ./set-branch.sh --branch=${SYSTEM_PULLREQUEST_TARGETBRANCH:-${BUILD_SOURCEBRANCH#refs/heads/}} + echo "##vso[task.setvariable variable=BRANCH]$BRANCH" + condition: and(succeeded(), not(canceled())) + - task: DownloadSecureFile@1 # Only when using SSH + name: deployKey + displayName: 'Download Deploy Key' + inputs: + secureFile: deploy_key + condition: and(succeeded(), not(canceled())) + - task: Bash@3 + displayName: Install + env: + SSH_KNOWN_HOSTS: "github.com" # Only when using SSH + inputs: + targetType: inline + # yamllint disable rule:indentation + script: | + # START SSH ONLY + echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m" + mkdir -p "${HOME}"/.ssh + cp $(deployKey.secureFilePath) "${HOME}"/.ssh/ci_ssh_key + if [[ -n "${SSH_KNOWN_HOSTS}" ]] + then + for host in ${SSH_KNOWN_HOSTS} + do + echo -e "\e[35;1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m" + ssh-keyscan -t rsa -H "${host}" 2>&1 | tee -a "${HOME}"/.ssh/known_hosts + done + fi + # END SSH ONLY + bash install-package.sh \ + --package=${PACKAGE} \ + --branch=${BRANCH} \ + --commit=${BUILD_SOURCEVERSION} \ + --pullrequest=${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-false} \ + --image="ghcr.io/tue-robotics/tue-env-ros-noetic" \ + --ssh \ + --ssh-key="${HOME}"/.ssh/ci_ssh_key # Last two arguments only when using SSH + # yamllint enable rule:indentation + condition: and(succeeded(), not(canceled())) + - task: Bash@3 + displayName: Build + inputs: + targetType: inline + script: | + bash build-package.sh --package=$PACKAGE + condition: and(succeeded(), not(canceled())) + - task: Bash@3 + displayName: Test + inputs: + targetType: inline + script: | + bash test-package.sh --package=$PACKAGE + condition: and(succeeded(), not(canceled())) diff --git a/ci/azure-pipelines.multiple_pkgs.example.yml b/ci/azure-pipelines.multiple_pkgs.example.yml index c9cbaff70..d001c3482 100644 --- a/ci/azure-pipelines.multiple_pkgs.example.yml +++ b/ci/azure-pipelines.multiple_pkgs.example.yml @@ -42,8 +42,23 @@ stages: COMMIT_RANGE=$(python3 ./azure_commit_range.py) BRANCH=${SYSTEM_PULLREQUEST_TARGETBRANCH:-${BUILD_SOURCEBRANCH#refs/heads/}} DEFAULT_BRANCH=$(git remote show origin 2>/dev/null | grep HEAD | awk '{print $3}') - ALL=$(([ "$BUILD_REASON" == "Schedule" ] || (([ "$BUILD_REASON" == "BatchedCI" ] || [ "$BUILD_REASON" == "IndividualCI" ] || [ "$BUILD_REASON" == "Manual" ]) && [ "$BRANCH" == "$DEFAULT_BRANCH" ])) && echo "true") - source ./packages.sh --pullrequest=${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-false} --branch=$SYSTEM_PULLREQUEST_TARGETBRANCH --commit-range=$COMMIT_RANGE --all=$ALL # add all folders which need to be excluded as an argument + ALL=$( \ + ( \ + [ "$BUILD_REASON" == "Schedule" ] || \ + ( \ + ( \ + [ "$BUILD_REASON" == "BatchedCI" ] || \ + [ "$BUILD_REASON" == "IndividualCI" ] || \ + [ "$BUILD_REASON" == "Manual" ] \ + ) && \ + [ "$BRANCH" == "${DEFAULT_BRANCH}" ] \ + ) \ + ) && echo "true") + source ./packages.sh \ + --pullrequest=${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-false} \ + --branch=${SYSTEM_PULLREQUEST_TARGETBRANCH} \ + --commit-range=${COMMIT_RANGE} \ + --all=${ALL} # add all folders which need to be excluded as an argument echo "##vso[task.setVariable variable=json_string;isOutput=true]$PACKAGES_DICT" - stage: install_build_test @@ -52,7 +67,9 @@ stages: ${{ else }}: displayName: 'Install, build & test:' dependsOn: package_selection + # yamllint disable rule:line-length condition: and(succeeded(), not(canceled()), ne(stageDependencies.package_selection.outputs['package_selection.matrix.json_string'], '{}')) + # yamllint enable rule:line-length jobs: - job: "" # This lets the job name be just the name of the matrix key condition: and(succeeded(), not(canceled())) @@ -108,6 +125,7 @@ stages: SSH_KNOWN_HOSTS: github.com # Only when using SSH inputs: targetType: inline + # yamllint disable rule:indentation script: | # START SSH ONLY echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m" @@ -122,7 +140,15 @@ stages: done fi # END SSH ONLY - bash install-package.sh --package=${PACKAGE} --branch=${BRANCH} --commit=${BUILD_SOURCEVERSION} --pullrequest=${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-false} --image="ghcr.io/tue-robotics/tue-env-ros-noetic" --ssh --ssh-key="${HOME}"/.ssh/ci_ssh_key # Last two arguments only when using SSH + bash install-package.sh \ + --package=${PACKAGE} \ + --branch=${BRANCH} \ + --commit=${BUILD_SOURCEVERSION} \ + --pullrequest=${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-false} \ + --image="ghcr.io/tue-robotics/tue-env-ros-noetic" \ + --ssh \ + --ssh-key="${HOME}"/.ssh/ci_ssh_key # Last two arguments only when using SSH + # yamllint enable rule:indentation condition: and(succeeded(), not(canceled())) - task: Bash@3 displayName: Build diff --git a/ci/azure_commit_range.py b/ci/azure_commit_range.py old mode 100644 new mode 100755 index ddf16f61b..936be493c --- a/ci/azure_commit_range.py +++ b/ci/azure_commit_range.py @@ -1,10 +1,5 @@ #! /usr/bin/env python3 -import json -import os -import sys -from urllib.request import urlopen - """ Script to determine the commit range of a build in azure pipelines. @@ -15,6 +10,11 @@ If the range is just 1 commit, an empty string is printed. """ +import json +import os +import sys +from urllib.request import urlopen + TEAM_FOUNDATION_URI = os.getenv("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI") TEAM_PROJECT = os.getenv("SYSTEM_TEAMPROJECT") BUILD_ID = os.getenv("BUILD_BUILDID") diff --git a/ci/commit-range/action.yml b/ci/commit-range/action.yml index 6a1cb3cb2..9b81a86fb 100644 --- a/ci/commit-range/action.yml +++ b/ci/commit-range/action.yml @@ -11,6 +11,7 @@ runs: id: commit-range env: commit_list: ${{ toJson(github.event.commits) }} # Doing this inside run doesn't work because of piping + # yamllint disable rule:indentation run: | if [ "${GITHUB_EVENT_NAME}" == "push" ] then @@ -23,7 +24,8 @@ runs: NEWEST=$(echo "${commit_list}" | jq 'last.id' | tr -d '"') OLDEST_PARENTS=($(git show --no-patch --format="%P" ${OLDEST})) NUMBER_OLDEST_PARENTS=${#OLDEST_PARENTS[@]} - COMMIT_RANGE="${OLDEST}^${NUMBER_OLDEST_PARENTS}...${NEWEST}" # Take second parent if possible, see https://stackoverflow.com/a/2222920 + # Take second parent if possible, see https://stackoverflow.com/a/2222920 + COMMIT_RANGE="${OLDEST}^${NUMBER_OLDEST_PARENTS}...${NEWEST}" fi elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ] then @@ -33,4 +35,5 @@ runs: fi echo -e "\e[35;1m COMMIT_RANGE = ${COMMIT_RANGE} \e[0m" echo "commit-range=${COMMIT_RANGE}" >> $GITHUB_OUTPUT + # yamllint enable rule:indentation shell: bash diff --git a/ci/main/action.yml b/ci/main/action.yml index 2041f9207..52d0f7085 100644 --- a/ci/main/action.yml +++ b/ci/main/action.yml @@ -45,6 +45,7 @@ runs: - name: Install id: install + # yamllint disable rule:indentation run: | echo "::group::install-package.sh" GITHUB_REF=${GITHUB_REF#refs/heads/} @@ -94,10 +95,12 @@ runs: "${SSH_ARGS[@]}" \ "${DEBUG_ARGS[@]}" echo "::endgroup::" + # yamllint enable rule:indentation shell: bash - name: Build id: build + # yamllint disable rule:indentation run: | echo "::group::build-package.sh" DEBUG_ARGS=() @@ -111,10 +114,12 @@ runs: --package=${{ inputs.package }} \ "${DEBUG_ARGS[@]}" echo "::endgroup::" + # yamllint enable rule:indentation shell: bash - name: Test id: test + # yamllint disable rule:indentation run: | echo "::group::test-package.sh" DEBUG_ARGS=() @@ -128,4 +133,5 @@ runs: --package=${{ inputs.package }} \ "${DEBUG_ARGS[@]}" echo "::endgroup::" + # yamllint enable rule:indentation shell: bash diff --git a/ci/modified-packages/action.yml b/ci/modified-packages/action.yml index 31e6f09ae..73ef846d9 100644 --- a/ci/modified-packages/action.yml +++ b/ci/modified-packages/action.yml @@ -18,6 +18,7 @@ runs: steps: - name: Modified packages id: packages + # yamllint disable rule:indentation run: | PULLREQUEST=${{ github.event.number }} PULLREQUEST=${PULLREQUEST:-false} @@ -25,10 +26,22 @@ runs: GITHUB_REF=${GITHUB_REF#refs/tags/} BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF}} DEFAULT_BRANCH=${{ github.event.repository.default_branch }} - if [[ "${GITHUB_EVENT_NAME}" == "schedule" ]] || ([[ "${GITHUB_EVENT_NAME}" == "push" ]] || [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]] && [[ "${BRANCH}" == "${DEFAULT_BRANCH}" ]]) + if \ + [[ "${GITHUB_EVENT_NAME}" == "schedule" ]] || \ + ( \ + [[ "${GITHUB_EVENT_NAME}" == "push" ]] || \ + [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]] && \ + [[ "${BRANCH}" == "${DEFAULT_BRANCH}" ]] \ + ) then ALL=true fi - source ${{ github.action_path }}/../packages.sh --pullrequest=${PULLREQUEST} --branch=${GITHUB_BASE_REF} --commit-range=${{ inputs.commit-range }} --all=${ALL} ${{ inputs.ignored-packages }} - echo "packages=${PACKAGES_LIST}" >> $GITHUB_OUTPUT + source ${{ github.action_path }}/../packages.sh \ + --pullrequest=${PULLREQUEST} \ + --branch=${GITHUB_BASE_REF} \ + --commit-range=${{ inputs.commit-range }} \ + --all=${ALL} \ + ${{ inputs.ignored-packages }} + echo "packages=${PACKAGES_LIST}" >> ${GITHUB_OUTPUT} + # yamllint enable rule:indentation shell: bash --noprofile --norc -o pipefail {0} diff --git a/ci/organization-packages.sh b/ci/organization-packages.sh old mode 100644 new mode 100755 diff --git a/ci/organization-packages/action.yaml b/ci/organization-packages/action.yaml index cf0d1a353..445b1ef99 100644 --- a/ci/organization-packages/action.yaml +++ b/ci/organization-packages/action.yaml @@ -26,7 +26,10 @@ runs: - name: Organization packages id: packages run: | - source ${{ github.action_path }}/../organization-packages.sh --organization=${{ inputs.organization }} --type=${{ inputs.package-type }} --token=${{ inputs.token }} + source ${{ github.action_path }}/../organization-packages.sh \ + --organization=${{ inputs.organization }} \ + --type=${{ inputs.package-type }} \ + --token=${{ inputs.token }} echo -e "\e[35;1mpackages=${packages_list} >> \${GITHUB_OUTPUT}\e[0m" echo "packages=${packages_list}" >> ${GITHUB_OUTPUT} shell: bash --noprofile --norc -o pipefail {0} diff --git a/ci/travis.example.yml b/ci/travis.example.yml index 26be5b306..bdf10fa84 100644 --- a/ci/travis.example.yml +++ b/ci/travis.example.yml @@ -15,7 +15,9 @@ before_install: - source set-branch.sh --branch=$TRAVIS_BRANCH install: - - bash install-package.sh --package=$PACKAGE --branch=$BRANCH --commit=$TRAVIS_COMMIT --pullrequest=$TRAVIS_PULL_REQUEST + # yamllint disable rule:line-length + - bash install-package.sh --package=${PACKAGE} --branch=$BRANCH --commit=$TRAVIS_COMMIT --pullrequest=$TRAVIS_PULL_REQUEST + # yamllint enable rule:line-length before_script: - bash build-package.sh --package=$PACKAGE diff --git a/ci/travis.multiple_pkgs.example.yml b/ci/travis.multiple_pkgs.example.yml index 98663226c..ca8604769 100644 --- a/ci/travis.multiple_pkgs.example.yml +++ b/ci/travis.multiple_pkgs.example.yml @@ -13,14 +13,18 @@ before_install: - wget https://raw.githubusercontent.com/tue-robotics/tue-env/master/ci/set-branch.sh # Determine packages modified in this commit - ALL=$([ "$TRAVIS_EVENT_TYPE" == "cron" ] && echo "true") || true + # yamllint disable rule:line-length - source packages.sh --pullrequest=$TRAVIS_PULL_REQUEST --branch=$TRAVIS_PULL_REQUEST_BRANCH --commit-range=$TRAVIS_COMMIT_RANGE --all=$ALL # add all folders which need to be excluded as an argument + # yamllint enable rule:line-length # Set BRANCH based on default branch - source set-branch.sh --branch=$TRAVIS_BRANCH env: + # yamllint disable rule:line-length # Run following in root of repo: # for item in $(ls | xargs ls -dl 2>/dev/null | grep "^d" | grep -v "\." | awk '{print $NF}'); do echo " - PACKAGE=$item"; done # Copy it below and remove unnecessary items + # yamllint disable rule:line-length - PACKAGE=package1 - PACKAGE=package2 - PACKAGE=package3 diff --git a/docs/CI_Travis_Setup.md b/docs/CI_Travis_Setup.md index b80fb1628..e2590e9cc 100644 --- a/docs/CI_Travis_Setup.md +++ b/docs/CI_Travis_Setup.md @@ -42,10 +42,12 @@ Steps to enable ssh support with travis and gitlab are as follows: 1. Add the following lines to .travis.yml under `install` + ```bash - SSH_KEY_PRIVATE=$(cat deploy_key) - bash install-package.sh --package=$PACKAGE --branch=$TRAVIS_BRANCH --commit=$TRAVIS_COMMIT --pullrequest=$TRAVIS_PULL_REQUEST --ssh --ssh-key="$SSH_KEY_PRIVATE" ``` + ## On GITLAB diff --git a/installer/lsb_release.py b/installer/lsb_release.py old mode 100644 new mode 100755 diff --git a/installer/tue-get-dep.bash b/installer/tue-get-dep.bash index aa00f521a..fecfb3ed4 100755 --- a/installer/tue-get-dep.bash +++ b/installer/tue-get-dep.bash @@ -21,7 +21,7 @@ function _show_dep fi local indent_str - indent_str=$(perl -E 'say "--" x '$indent) + indent_str=$(perl -E 'say "--" x '"${indent}") if [ -n "$2" ] then @@ -34,7 +34,7 @@ function _show_dep else if [ -n "$LEVEL" ] then - if [[ $indent > $LEVEL ]] + if [[ ${indent} -gt ${LEVEL} ]] then return fi diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..c18fdf4dd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 120 +target-version = ["py38"] diff --git a/setup/tue-env-config.bash b/setup/tue-env-config.bash index 1eb7e66f4..68bcefb7a 100755 --- a/setup/tue-env-config.bash +++ b/setup/tue-env-config.bash @@ -150,7 +150,7 @@ function tue-env-set echo -e "[tue-env](config) Environment '${tue_env}' has '${option}' set to '${value}'" } -function _main +function _tue-env-config-main { if [ -z "$1" ] then @@ -190,4 +190,6 @@ function _main fi } -_main "$@" +_tue-env-config-main "$@" + +unset -f _tue-env-config-main