From 8fd7272ec6d4f2e3fac8f15f11b2d4f4c8a3b663 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 16 Aug 2024 15:40:00 +0200 Subject: [PATCH 1/2] Add tue-install-apt-key-source --- README.md | 61 +++++++---- installer/parse_install_yaml.py | 39 +++++++ installer/tue-install-impl.bash | 183 ++++++++++++++++++++++++++++++++ 3 files changed, 261 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index cf15c8b20..0fc7cf547 100644 --- a/README.md +++ b/README.md @@ -329,6 +329,21 @@ Only use the corresponding bash function in `install.bash` if no other solution version: [branch/commit/tag] (Optional field) ``` +#### APT-KEY-SOURCE(-now) + +Add an APT source including the key which it is signed with. + +```yaml +- type: apt-key-source + distribution-extensions: [, [, ..., ]] (Minimal one extension required, use "" for no extension, just distribution name) + key-file: + key-fingerprint: + key-url: + repo-components: [[, ..., ]] (Optional field) + repo-url: + source-file: or +``` + ### Writing `install.bash` The use of the following variables is prohibited in `install.bash`: @@ -349,27 +364,29 @@ The use of the following variables is prohibited in `install.bash`: The following functions provided with `tue-env` must be preferred over any generally used methods of installing packages: -| Function Name | Description | -|---------------------------------|----------------------------------------------------------------------------------------------------------------| -| `tue-install-add-text` | To add/replace text in a file with `sudo` taken into account | -| `tue-install-apt-get-update` | Make sure that during next `tue-install-system-now` call `apt-get` is updated | -| `tue-install-cp` | Analogous to `cp` but takes `sudo` into account and the source should be relative to target | -| `tue-install-ln` | Analogous to `ln -s` but takes `sudo` into account and the source should be relative to target or absolute | -| `tue-install-dpkg` | To install a debian dpkg file | -| `tue-install-git` | To install a git repository | -| `tue-install-pip` | To add a python pip3 package to a list to be installed at the end | -| `tue-install-pip3` | To add a python pip3 package to a list to be installed at the end | -| `tue-install-pip-now` | To install python pip3 package, but ignores it if already installed | -| `tue-install-pip3-now` | To install python pip3 package, but ignores it if already installed | -| `tue-install-ppa` | To add one PPA/DEB to a list to be added with `apt-add-repository` at the end, before apt-get | -| `tue-install-ppa-now` | To add a PPA/DEB with `apt-add-repository`, use ^ inside of a DEB and spaces between items | -| `tue-install-snap` | To add a snap package to a list to be installed at the end | -| `tue-install-snap-now` | To install a snap | -| `tue-install-gem` | To add a gem package to a list to be installed at the end | -| `tue-install-gem-now` | To install a gem | -| `tue-install-system` | To add `deb` package to a list of packages to be installed at the end with `apt-get` | -| `tue-install-system-now` | To install `deb` packages with `apt-get` right away, but ignores it if already installed | -| `tue-install-get-releases` | To get a released asset from a github repository and place it in the requested directory | +| Function Name | Description | +|----------------------------------|------------------------------------------------------------------------------------------------------------| +| `tue-install-add-text` | To add/replace text in a file with `sudo` taken into account | +| `tue-install-apt-get-update` | Make sure that during next `tue-install-system-now` call `apt-get` is updated | +| `tue-install-cp` | Analogous to `cp` but takes `sudo` into account and the source should be relative to target | +| `tue-install-ln` | Analogous to `ln -s` but takes `sudo` into account and the source should be relative to target or absolute | +| `tue-install-dpkg` | To install a debian dpkg file | +| `tue-install-git` | To install a git repository | +| `tue-install-pip` | To add a python pip3 package to a list to be installed at the end | +| `tue-install-pip3` | To add a python pip3 package to a list to be installed at the end | +| `tue-install-pip-now` | To install python pip3 package, but ignores it if already installed | +| `tue-install-pip3-now` | To install python pip3 package, but ignores it if already installed | +| `tue-install-ppa` | To add one PPA/DEB to a list to be added with `apt-add-repository` at the end, before apt-get | +| `tue-install-ppa-now` | To add a PPA/DEB with `apt-add-repository`, use ^ inside of a DEB and spaces between items | +| `tue-install-apt-key-source` | To add one key- and source file to a list to be added with at the end to the apt sources, before apt-get | +| `tue-install-apt-key-source-now` | To add one key- and source file to the apt sources | +| `tue-install-snap` | To add a snap package to a list to be installed at the end | +| `tue-install-snap-now` | To install a snap | +| `tue-install-gem` | To add a gem package to a list to be installed at the end | +| `tue-install-gem-now` | To install a gem | +| `tue-install-system` | To add `deb` package to a list of packages to be installed at the end with `apt-get` | +| `tue-install-system-now` | To install `deb` packages with `apt-get` right away, but ignores it if already installed | +| `tue-install-get-releases` | To get a released asset from a github repository and place it in the requested directory | The input arguments for each of the above-mentioned commands can be found by simply executing the command in a bash session (provided `tue-env` is correctly @@ -377,7 +394,7 @@ installed). A general remark about the preferred order of the package repositories: -system > ppa > pip = pip3 > snap > gem > git > dpkg +system > ppa > apt-key-source > pip = pip3 > snap > gem > git > dpkg #### Logging diff --git a/installer/parse_install_yaml.py b/installer/parse_install_yaml.py index 5ea1f8b5e..bf274ac90 100755 --- a/installer/parse_install_yaml.py +++ b/installer/parse_install_yaml.py @@ -59,6 +59,39 @@ def type_git(install_item: Mapping, allowed_keys: Optional[List[str]] = None) -> return " ".join(command) +def type_apt_key_source(install_item: Mapping) -> str: + dist_exts = install_item.get("distribution-extensions") + key_file = install_item.get("key-file") + key_fingerprint = install_item.get("key-fingerprint") + key_url = install_item.get("key-url") + repo_components = install_item.get("repo-components", []) # Optional field + repo_url = install_item.get("repo-url") + source_file = install_item.get("source-file") + + args: List[str] = [] + + if not isinstance(dist_exts, list): + raise ValueError("distribution-extensions should be a list") + if len(dist_exts) < 1: + raise ValueError("At least one distribution extension should be specified") + for dist_ext in dist_exts: + args.append(f"--distribution-extension={dist_ext}") + + args.append(f"--key-file={key_file}") + args.append(f"--key-fingerprint={key_fingerprint}") + args.append(f"--key-url={key_url}") + + if not isinstance(repo_components, list): + raise ValueError("repo-components should be a list") + for repo_component in repo_components: + args.append(f"--repo-component={repo_component}") + + args.append(f"--repo-url={repo_url}") + args.append(f"--source-file={source_file}") + + return " ".join(args) + + def catkin_git(source: Mapping) -> str: """ Function to generate installation command for catkin git targets from the extracted yaml @@ -244,6 +277,12 @@ def get_distro_item( command = f"tue-install-{install_type} {pkg_name}" + elif install_type in ["apt-key-source", "apt-key-source-now"]: + if now and "now" not in install_type: + install_type += "-now" + + command = f"tue-install-{install_type} {type_apt_key_source(install_item)}" + else: raise ValueError(f"Unknown install type: '{install_type}'") diff --git a/installer/tue-install-impl.bash b/installer/tue-install-impl.bash index e906e4705..6031bd0ee 100755 --- a/installer/tue-install-impl.bash +++ b/installer/tue-install-impl.bash @@ -1083,6 +1083,179 @@ function tue-install-ppa-now # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +function tue-install-apt-key-source +{ + tue-install-debug "tue-install-apt-key-source $*" + + if [ -z "$6" ] + then + tue-install-error "Invalid tue-install-apt-key-source call: needs minimal 6 arguments" + fi + local args + args="$*" + + tue-install-debug "Adding '${args}' to apt-key-source list" + TUE_INSTALL_APT_KEY_SOURCES="${TUE_INSTALL_APT_KEY_SOURCES} ${args// /^}" # Replace space by ^ to support for-loops later +} + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + +function tue-install-apt-key-source-now +{ + tue-install-debug "tue-install-apt-key-source-now $*" + + local distribution_extensions key_file key_fingerprint key_url repo_url source_file + distribution_extensions=() + repo_components=() + if [[ -n $1 ]] + then + for i in "$@" + do + case $i in + --distribution-extension=* ) + distribution_extensions+=("${i#*=}") ;; + --key-file=* ) + key_file="${i#*=}" ;; + --key-fingerprint=* ) + key_fingerprint="${i#*=}" ;; + --key-url=* ) + key_url="${i#*=}" ;; + --repo-component=* ) + repo_components+=("${i#*=}") ;; + --repo-url=* ) + repo_url="${i#*=}" ;; + --source-file=* ) + source_file="${i#*=}" ;; + * ) + tue-install-error "Unknown input variable ${i}" ;; + esac + done + fi + + if [[ ! ${#distribution_extensions[@]} ]] + then + tue-install-error "Invalid tue-install-apt-key-source-now call: needs at least 1 distribution-extension" + fi + + if [[ -z "${key_file}" ]] + then + tue-install-error "Invalid tue-install-apt-key-source-now call: needs key-file" + fi + + if [[ -z "${key_fingerprint}" ]] + then + tue-install-error "Invalid tue-install-apt-key-source-now call: needs key-fingerprint" + fi + + if [[ -z "${key_url}" ]] + then + tue-install-error "Invalid tue-install-apt-key-source-now call: needs key-url" + fi + + if [[ -z "${repo_url}" ]] + then + tue-install-error "Invalid tue-install-apt-key-source-now call: needs repo-url" + fi + + if [[ -z "${source_file}" ]] + then + tue-install-error "Invalid tue-install-apt-key-source-now call: needs source-file" + fi + + local key_folder name + name=${TUE_INSTALL_CURRENT_TARGET%-setup} + key_folder=$(dirname "${key_file}") + + if [[ $(dirname "${source_file}") == "." ]] + then + tue-install-debug "Making source_file '${source_file}' relative to '/etc/apt/sources.list.d/'" + source_file="/etc/apt/sources.list.d/${source_file}" + fi + + local arch distribution + arch=$(dpkg --print-architecture) + distribution=$(lsb_release -cs) + + local distribution_components + distribution_components=() + for dist_ext in "${distribution_extensions[@]}" + do + distribution_components+=("${distribution}${dist_ext:+-${dist_ext}}") + done + + local source_url + source_url="deb [arch=${arch} signed-by=${key_file}] ${repo_url} ${distribution_components[*]} ${repo_components[*]}" + + local apt_needs_to_be_updated key_needs_to_be_added source_needs_to_be_added + key_needs_to_be_added=false + source_needs_to_be_added=false + apt_needs_to_be_updated=false + + if [[ ! -f "${key_file}" ]] + then + tue-install-debug "Keyring '${key_file}' doesn't exist yet." + key_needs_to_be_added=true + elif ! gpg --import-options show-only --import < "${key_file}" | grep -q "${key_fingerprint}" &> /dev/null + then + tue-install-debug "Keyring '${key_file}' doesn't match the fingerprint '${key_fingerprint}'." + key_needs_to_be_added=true + else + tue-install-debug "Not updating the existing GPG of the ${name} repository with fingerprint '${key_fingerprint}'" + fi + + if [[ "${key_needs_to_be_added}" == "true" ]] + then + tue-install-debug "Making sure '${key_folder}' folder exists with the correct permissions." + tue-install-pipe sudo install -m 0755 -d "${key_folder}" || tue-install-error "Could not create '${key_folder}' folder" + tue-install-debug "Downloading gpg key of ${name} repo with fingerprint '${key_fingerprint}'." + curl -fsSL "${key_url}" | sudo gpg --dearmor --yes -o "${key_file}" + tue-install-debug "Successfully added/updated ${name} repository GPG key" + + apt_needs_to_be_updated=true + fi + + if [[ ! -f "${source_file}" ]] + then + tue-install-debug "Adding ${name} sources to apt-get" + source_needs_to_be_added=true + elif [[ $(cat "${source_file}") != "${source_url}" ]] + then + tue-install-debug "Updating ${name} sources to apt-get" + source_needs_to_be_added=true + else + tue-install-debug "${name} sources already added to apt-get" + fi + + if [[ "${source_needs_to_be_added}" == "true" ]] + then + echo "${source_url}" | sudo tee "${source_file}" > /dev/null + tue-install-debug "Successfully added/updated ${name} source file in apt" + + apt_needs_to_be_updated=true + fi + + if [[ "${apt_needs_to_be_updated}" == "true" ]] + then + tue-install-apt-get-update + fi +} + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + +function tue-install-apt-key-sources-now +{ + tue-install-debug "install-apt-key-sources-now $*" + # shellcheck disable=SC2048 + for args in $* + do + args="${args//^/ }" + # shellcheck disable=SC2086 + tue-install-apt-key-source-now ${args} + done +} + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + function _tue-install-pip { local pv name @@ -1752,6 +1925,7 @@ mkdir -p "${TUE_INSTALL_INSTALLED_DIR}" # Initialize the installer variables TUE_INSTALL_GIT_PULL_Q=() +TUE_INSTALL_APT_KEY_SOURCES= TUE_INSTALL_SYSTEMS= TUE_INSTALL_PPA= TUE_INSTALL_PIP3S= @@ -1833,6 +2007,15 @@ fi # Remove temp directories rm -rf "$TUE_INSTALL_STATE_DIR" +# Installing all the ppa repo's, which are collected during install +if [[ -n "${TUE_INSTALL_APT_KEY_SOURCES}" ]] +then + TUE_INSTALL_CURRENT_TARGET="APT-KEY-SOURCE" + + tue-install-debug "calling: tue-install-apt-key-sources-now${TUE_INSTALL_APT_KEY_SOURCES}" + tue-install-apt-key-sources-now "${TUE_INSTALL_APT_KEY_SOURCES}" +fi + # Installing all the ppa repo's, which are collected during install if [ -n "$TUE_INSTALL_PPA" ] From cb5da773d1b0f0a6c9c4cc6f126002cac210b7a9 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 20 Aug 2024 15:24:32 +0200 Subject: [PATCH 2/2] Bump VERSION to 1.26.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d7b1d35c1..5ff8c4f5d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.25.14 +1.26.0