From 988ffa30815be0ed476ae022b4f54544393e3f62 Mon Sep 17 00:00:00 2001 From: Rachel Menge Date: Fri, 10 May 2024 09:58:23 -0700 Subject: [PATCH 01/51] dracut: Patch microcode output check based on CONFIG_MICROCODE_AMD/INTEL (#9061) Dracut outputs this message while generating initramfs: "dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_(AMD|INTEL)!=y" However, these config options were removed in the kernel commit e6bcfdd "x86/microcode: Hide the config knob" (e6bcfdd). Additionally, the kernel enables "CONFIG_MICROCODE" Therefore, take upstream dracut patch (6c80408) which fixes microcode check. --- ...-based-on-CONFIG_MICROCODE_AMD-INTEL.patch | 45 +++++++++++++++++++ SPECS/dracut/dracut.spec | 6 ++- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 SPECS/dracut/0010-fix-remove-microcode-check-based-on-CONFIG_MICROCODE_AMD-INTEL.patch diff --git a/SPECS/dracut/0010-fix-remove-microcode-check-based-on-CONFIG_MICROCODE_AMD-INTEL.patch b/SPECS/dracut/0010-fix-remove-microcode-check-based-on-CONFIG_MICROCODE_AMD-INTEL.patch new file mode 100644 index 00000000000..6d3295c8e26 --- /dev/null +++ b/SPECS/dracut/0010-fix-remove-microcode-check-based-on-CONFIG_MICROCODE_AMD-INTEL.patch @@ -0,0 +1,45 @@ +From 6c80408c8644a0add1907b0593eb83f90d6247b1 Mon Sep 17 00:00:00 2001 +From: Antonio Alvarez Feijoo +Date: Mon, 14 Aug 2023 12:28:11 +0200 +Subject: [PATCH] fix(dracut.sh): remove microcode check based on + CONFIG_MICROCODE_[AMD|INTEL] + +`CONFIG_MICROCODE_AMD` and `CONFIG_MICROCODE_INTEL` are hidden since +https://lore.kernel.org/all/20230810160805.081212701@linutronix.de/, therefore +this check is wrong and early microcode is always disabled. +--- + dracut.sh | 11 ++++------- + 1 file changed, 4 insertions(+), 7 deletions(-) + +diff --git a/dracut.sh b/dracut.sh +index e0abdb3b0..3b292910f 100755 +--- a/dracut.sh ++++ b/dracut.sh +@@ -1561,23 +1561,20 @@ fi + + if [[ $early_microcode == yes ]]; then + if [[ $hostonly ]]; then +- if [[ $(get_cpu_vendor) == "AMD" ]]; then +- check_kernel_config CONFIG_MICROCODE_AMD || unset early_microcode +- elif [[ $(get_cpu_vendor) == "Intel" ]]; then +- check_kernel_config CONFIG_MICROCODE_INTEL || unset early_microcode ++ if [[ $(get_cpu_vendor) == "AMD" || $(get_cpu_vendor) == "Intel" ]]; then ++ check_kernel_config CONFIG_MICROCODE || unset early_microcode + else + unset early_microcode + fi + else +- ! check_kernel_config CONFIG_MICROCODE_AMD \ +- && ! check_kernel_config CONFIG_MICROCODE_INTEL \ ++ ! check_kernel_config CONFIG_MICROCODE \ + && unset early_microcode + fi + # Do not complain on non-x86 architectures as it makes no sense + case "${DRACUT_ARCH:-$(uname -m)}" in + x86_64 | i?86) + [[ $early_microcode != yes ]] \ +- && dwarn "Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y" ++ && dwarn "Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE!=y" + ;; + *) ;; + esac diff --git a/SPECS/dracut/dracut.spec b/SPECS/dracut/dracut.spec index abb4013b84d..a4dd1f02366 100644 --- a/SPECS/dracut/dracut.spec +++ b/SPECS/dracut/dracut.spec @@ -4,7 +4,7 @@ Summary: dracut to create initramfs Name: dracut Version: 059 -Release: 16%{?dist} +Release: 17%{?dist} # The entire source code is GPLv2+ # except install/* which is LGPLv2+ License: GPLv2+ AND LGPLv2+ @@ -32,6 +32,7 @@ Patch: 0006-dracut.sh-validate-instmods-calls.patch Patch: 0007-feat-dracut.sh-support-multiple-config-dirs.patch Patch: 0008-fix-dracut-systemd-rootfs-generator-cannot-write-out.patch Patch: 0009-install-systemd-executor.patch +Patch: 0010-fix-remove-microcode-check-based-on-CONFIG_MICROCODE_AMD-INTEL.patch BuildRequires: bash BuildRequires: kmod-devel @@ -216,6 +217,9 @@ ln -srv %{buildroot}%{_bindir}/%{name} %{buildroot}%{_sbindir}/%{name} %dir %{_sharedstatedir}/%{name}/overlay %changelog +* Thu May 03 2024 Rachel Menge - 059-17 +- Patch microcode output check based on CONFIG_MICROCODE_AMD/INTEL + * Wed Mar 27 2024 Cameron Baird - 059-16 - Remove x86-specific xen-acpi-processor driver from defaults From b12cf050575b65e97116ec12adc287d631682902 Mon Sep 17 00:00:00 2001 From: Betty <38226164+BettyRain@users.noreply.github.com> Date: Fri, 10 May 2024 11:07:32 -0700 Subject: [PATCH 02/51] python-cachetools: upgrade to 5.3.3 (#9075) Co-authored-by: Betty Lakes --- SPECS/python-cachetools/python-cachetools.signatures.json | 8 ++++---- SPECS/python-cachetools/python-cachetools.spec | 5 ++++- cgmanifest.json | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/SPECS/python-cachetools/python-cachetools.signatures.json b/SPECS/python-cachetools/python-cachetools.signatures.json index 7965883d822..2a9511efa7d 100644 --- a/SPECS/python-cachetools/python-cachetools.signatures.json +++ b/SPECS/python-cachetools/python-cachetools.signatures.json @@ -1,5 +1,5 @@ { - "Signatures": { - "cachetools-5.0.0.tar.gz": "486471dfa8799eb7ec503a8059e263db000cdda20075ce5e48903087f79d5fd6" - } - } + "Signatures": { + "cachetools-5.3.3.tar.gz": "ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105" + } +} \ No newline at end of file diff --git a/SPECS/python-cachetools/python-cachetools.spec b/SPECS/python-cachetools/python-cachetools.spec index be6c83ee778..5a21f421312 100644 --- a/SPECS/python-cachetools/python-cachetools.spec +++ b/SPECS/python-cachetools/python-cachetools.spec @@ -3,7 +3,7 @@ Summary: Various memoizing collections and decorators Name: python-%{pkgname} -Version: 5.0.0 +Version: 5.3.3 Release: 1%{?dist} License: MIT URL: https://github.com/tkem/cachetools @@ -43,6 +43,9 @@ python3 setup.py install --skip-build --root=%{buildroot} %{python3_sitelib}/*.egg-info %changelog +* Fri May 10 2024 Betty Lakes - 5.3.3-1 +- Updated to 5.3.3 + * Wed Feb 09 2022 Nick Samson - 5.0.0-1 - Updated to 5.0.0 diff --git a/cgmanifest.json b/cgmanifest.json index 68ae247929d..f8a69d9a3ae 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -21803,8 +21803,8 @@ "type": "other", "other": { "name": "python-cachetools", - "version": "5.0.0", - "downloadUrl": "https://pypi.python.org/packages/source/c/cachetools/cachetools-5.0.0.tar.gz" + "version": "5.3.3", + "downloadUrl": "https://pypi.python.org/packages/source/c/cachetools/cachetools-5.3.3.tar.gz" } } }, From 4ae9f5b9580f10aa91ebc248dcadbf9d2448989c Mon Sep 17 00:00:00 2001 From: Rachel Menge Date: Fri, 10 May 2024 13:10:08 -0700 Subject: [PATCH 03/51] systemd: suppress debug messages to console by default (#9011) During normal use, many low level debug messages are printing to the console which is causing usability issues for the user. To deal with this, install a sysctl configuration to run on boot which will change the default kernel printk value to "4 4 1 7". This PR is based off of findings from: #8331 Signed-off-by: Rachel Menge --- .../systemd}/10-console-messages.conf | 2 +- SPECS/systemd/systemd.signatures.json | 1 + SPECS/systemd/systemd.spec | 10 +++++++++- toolkit/resources/imageconfigs/iso_initrd.json | 5 ++--- toolkit/resources/imageconfigs/iso_initrd_arm64.json | 5 ++--- 5 files changed, 15 insertions(+), 8 deletions(-) rename {toolkit/resources/imageconfigs/additionalfiles/iso_initrd => SPECS/systemd}/10-console-messages.conf (60%) diff --git a/toolkit/resources/imageconfigs/additionalfiles/iso_initrd/10-console-messages.conf b/SPECS/systemd/10-console-messages.conf similarity index 60% rename from toolkit/resources/imageconfigs/additionalfiles/iso_initrd/10-console-messages.conf rename to SPECS/systemd/10-console-messages.conf index e83d8c415c5..f8f1c0eda30 100644 --- a/toolkit/resources/imageconfigs/additionalfiles/iso_initrd/10-console-messages.conf +++ b/SPECS/systemd/10-console-messages.conf @@ -1,2 +1,2 @@ # Suppress debug messages on console -kernel.printk = 4 4 1 7 \ No newline at end of file +kernel.printk = 4 4 1 7 diff --git a/SPECS/systemd/systemd.signatures.json b/SPECS/systemd/systemd.signatures.json index bdd75b35fe1..27e5c35758d 100644 --- a/SPECS/systemd/systemd.signatures.json +++ b/SPECS/systemd/systemd.signatures.json @@ -1,5 +1,6 @@ { "Signatures": { + "10-console-messages.conf": "6cfc12931ac75df710aa1d323cdb98592935692cd9d32260d6118f9ad03a42e2", "10-map-count.conf": "bad4f0bbf268860582402bb4131b7b3586bf1f098a8bf89a6adf503191bc8913", "10-oomd-defaults.conf": "f7c8e9d3455bf54795063e3914956b4e15ba52f493b0f2304abef6326a80c6cb", "10-oomd-per-slice-defaults.conf": "3f0d6882312affecddfdc5a204a44c34b90c573839f2a4850a26b44f528520bd", diff --git a/SPECS/systemd/systemd.spec b/SPECS/systemd/systemd.spec index 24c4157f642..d0c204c376a 100644 --- a/SPECS/systemd/systemd.spec +++ b/SPECS/systemd/systemd.spec @@ -50,7 +50,7 @@ Version: 255 # determine the build information from local checkout Version: %(tools/meson-vcs-tag.sh . error | sed -r 's/-([0-9])/.^\1/; s/-g/_g/') %endif -Release: 12%{?dist} +Release: 13%{?dist} # FIXME - hardcode to 'stable' for now as that's what we have in our blobstore %global stable 1 @@ -96,6 +96,7 @@ Source14: 10-oomd-defaults.conf Source15: 10-oomd-per-slice-defaults.conf Source16: 10-timeout-abort.conf Source17: 10-map-count.conf +Source18: 10-console-messages.conf Source21: macros.sysusers Source22: sysusers.attr @@ -856,6 +857,10 @@ install -Dm0644 10-timeout-abort.conf.user %{buildroot}%{user_unit_dir}/service. # https://fedoraproject.org/wiki/Changes/IncreaseVmMaxMapCount install -Dm0644 -t %{buildroot}%{_prefix}/lib/sysctl.d/ %{SOURCE17} +%if 0%{?azl} +install -Dm0644 -t %{buildroot}%{_prefix}/lib/sysctl.d/ %{SOURCE18} +%endif + sed -i 's|#!/usr/bin/env python3|#!%{__python3}|' %{buildroot}/usr/lib/systemd/tests/run-unit-tests.py install -m 0644 -D -t %{buildroot}%{_rpmconfigdir}/macros.d/ %{SOURCE21} @@ -1192,6 +1197,9 @@ rm -f %{name}.lang # %autochangelog. So we need to continue manually maintaining the # changelog here. %changelog +* Thu May 02 2024 Rachel Menge - 255-13 +- Supply 10-console-messages.conf sysctl to lower the default kernel messages to the console + * Thu Apr 18 2024 Dan Streetman - 255-12 - move libidn2 recommends from core package to systemd-networkd diff --git a/toolkit/resources/imageconfigs/iso_initrd.json b/toolkit/resources/imageconfigs/iso_initrd.json index 567aaa02459..e6197504e40 100644 --- a/toolkit/resources/imageconfigs/iso_initrd.json +++ b/toolkit/resources/imageconfigs/iso_initrd.json @@ -37,8 +37,7 @@ "additionalfiles/iso_initrd/usr/lib/mariner/terminfo/mariner-installer": "/usr/lib/mariner/terminfo/m/mariner-installer", "additionalfiles/iso_initrd/usr/lib/systemd/system/getty@.service": "/usr/lib/systemd/system/getty@.service", "additionalfiles/iso_initrd/usr/lib/systemd/system/serial-getty@.service": "/usr/lib/systemd/system/serial-getty@.service", - "../manifests/image/local.repo": "/etc/yum.repos.d/mariner-iso.repo", - "additionalfiles/iso_initrd/10-console-messages.conf": "/etc/sysctl.d/10-console-messages.conf" + "../manifests/image/local.repo": "/etc/yum.repos.d/mariner-iso.repo" }, "Users": [ { @@ -49,4 +48,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/toolkit/resources/imageconfigs/iso_initrd_arm64.json b/toolkit/resources/imageconfigs/iso_initrd_arm64.json index 3274567033c..5e5b368f917 100644 --- a/toolkit/resources/imageconfigs/iso_initrd_arm64.json +++ b/toolkit/resources/imageconfigs/iso_initrd_arm64.json @@ -32,8 +32,7 @@ "additionalfiles/iso_initrd/usr/lib/mariner/terminfo/mariner-installer": "/usr/lib/mariner/terminfo/m/mariner-installer", "additionalfiles/iso_initrd/usr/lib/systemd/system/getty@.service": "/usr/lib/systemd/system/getty@.service", "additionalfiles/iso_initrd/usr/lib/systemd/system/serial-getty@.service": "/usr/lib/systemd/system/serial-getty@.service", - "../manifests/image/local.repo": "/etc/yum.repos.d/mariner-iso.repo", - "additionalfiles/iso_initrd/10-console-messages.conf": "/etc/sysctl.d/10-console-messages.conf" + "../manifests/image/local.repo": "/etc/yum.repos.d/mariner-iso.repo" }, "Users": [ { @@ -44,4 +43,4 @@ ] } ] -} \ No newline at end of file +} From e2faca99d8ebbce00625216ef133c5b323705436 Mon Sep 17 00:00:00 2001 From: Riken Maharjan <106988478+rikenm1@users.noreply.github.com> Date: Fri, 10 May 2024 15:55:35 -0700 Subject: [PATCH 04/51] Rename Pygments to pygments for python-rich (#9084) --- SPECS/python-rich/python-rich.spec | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/SPECS/python-rich/python-rich.spec b/SPECS/python-rich/python-rich.spec index 5f53aa43ea1..9e3e52d991e 100644 --- a/SPECS/python-rich/python-rich.spec +++ b/SPECS/python-rich/python-rich.spec @@ -1,6 +1,6 @@ Name: python-rich Version: 13.7.1 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Render rich text and beautiful formatting in the terminal Vendor: Microsoft Corporation Distribution: Azure Linux @@ -29,7 +29,7 @@ code, tracebacks, and more — out of the box. %package -n python3-rich Summary: %{summary} Requires: python3-markdown-it-py -Requires: python3-Pygments +Requires: python3-pygments Requires: python3-typing-extensions # This was previously misnamed, remove the obsolete in Fedora 38, EPEL 10 Obsoletes: python-rich < 10.16.1-2 @@ -59,6 +59,9 @@ code, tracebacks, and more — out of the box. %doc README.md %changelog +* Fri May 10 2024 Riken Maharjan - 13.7.1-3 +- Fix pygments name in Requires. + * Mon Mar 28 2024 Riken Maharjan - 13.7.1-2 - Initial Azure Linux import from Fedora 41 (license: MIT). - License verified. From 6eb795217647f5538b2af9255d6612072b8eb101 Mon Sep 17 00:00:00 2001 From: Lanze Liu <86434077+liulanze@users.noreply.github.com> Date: Sun, 12 May 2024 21:32:13 -0700 Subject: [PATCH 05/51] Add device waiting logic to ensure volumes are ready before proceeding with mounts (#8974) --- SPECS/dracut/20overlayfs/module-setup.sh | 3 +++ SPECS/dracut/20overlayfs/overlayfs-mount.sh | 9 ++++++--- SPECS/dracut/dracut.signatures.json | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/SPECS/dracut/20overlayfs/module-setup.sh b/SPECS/dracut/20overlayfs/module-setup.sh index 46ef040d113..8728c887b25 100755 --- a/SPECS/dracut/20overlayfs/module-setup.sh +++ b/SPECS/dracut/20overlayfs/module-setup.sh @@ -1,4 +1,6 @@ #!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. check() { return 0 @@ -14,5 +16,6 @@ installkernel() { } install() { + inst "grep" inst_hook pre-pivot 10 "$moddir/overlayfs-mount.sh" } diff --git a/SPECS/dracut/20overlayfs/overlayfs-mount.sh b/SPECS/dracut/20overlayfs/overlayfs-mount.sh index 7cec06e1f4d..83d677bbf4f 100755 --- a/SPECS/dracut/20overlayfs/overlayfs-mount.sh +++ b/SPECS/dracut/20overlayfs/overlayfs-mount.sh @@ -1,4 +1,6 @@ #!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. # Description: This script is designed to mount a DM-Verity root filesystem and # set up OverlayFS. It is driven by kernel parameters and is invoked during the @@ -64,9 +66,10 @@ mount_volatile_persistent_volume() { mount -t tmpfs tmpfs -o ${OVERLAY_MNT_OPTS} "${_overlay_mount}" || \ die "Failed to create overlay tmpfs at ${_overlay_mount}" else - # Check if /etc/mdadm.conf exists. - if [ -f "/etc/mdadm.conf" ]; then - mdadm --assemble ${_volume} || \ + # Check if the specified Overlay RAID volume is present in the system. + if mdadm --examine --scan | grep -qs "${_volume}"; then + # If the specified Overlay RAID volume is found, attempt to assemble it. + mdadm --assemble "${_volume}" || \ die "Failed to assemble RAID volume." fi diff --git a/SPECS/dracut/dracut.signatures.json b/SPECS/dracut/dracut.signatures.json index 93444c6c061..0e7651e1920 100644 --- a/SPECS/dracut/dracut.signatures.json +++ b/SPECS/dracut/dracut.signatures.json @@ -4,7 +4,7 @@ "dracut-059.tar.gz": "eabf0bb685420c1e1d5475b6855ef787104508f0135ff570312845256e0fcecf", "lgpl-2.1.txt": "dc626520dcd53a22f727af3ee42c770e56c97a64fe3adb063799d8ab032fe551", "megaraid.conf": "914824cdbe0c525b71efa05a75e453335b0068beb8bc28bef2a5866d74bf7dd4", - "module-setup.sh": "330af5c105793fb37434730ce0ff59467a9cc60a81a5e32193dc53235e9744c1", - "overlayfs-mount.sh": "e0d009b765fea58319f9b4b33f5399d6fea90f3ba98a4fdad43d80256e555027" + "module-setup.sh": "8f5a0d3cc393d78bcb523b0d53c578d2767d688f00e84b525355bbb31d753da4", + "overlayfs-mount.sh": "f049d0ad2e4a1d98acb329765900690883520fdc8fc9463f5081f6e7fb679695" } } From 2060ff308f81e43c86dadc987e15e30d8e8f97c4 Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Mon, 13 May 2024 10:14:54 -0700 Subject: [PATCH 06/51] [AUTOPATCHER-CORE] Upgrade python-execnet to 2.1.1 none (#8897) Co-authored-by: Osama Esmail Co-authored-by: osamaesmailmsft <110202916+osamaesmailmsft@users.noreply.github.com> --- .../python-execnet.signatures.json | 6 +- SPECS/python-execnet/python-execnet.spec | 60 +++++++++++++------ cgmanifest.json | 4 +- 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/SPECS/python-execnet/python-execnet.signatures.json b/SPECS/python-execnet/python-execnet.signatures.json index 496bd61b3b7..697afb10f0b 100644 --- a/SPECS/python-execnet/python-execnet.signatures.json +++ b/SPECS/python-execnet/python-execnet.signatures.json @@ -1,5 +1,5 @@ { - "Signatures": { - "execnet-1.9.0.tar.gz": "8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5" - } + "Signatures": { + "execnet-2.1.1.tar.gz": "5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3" + } } diff --git a/SPECS/python-execnet/python-execnet.spec b/SPECS/python-execnet/python-execnet.spec index 617c7fdbe8a..add0eba45df 100644 --- a/SPECS/python-execnet/python-execnet.spec +++ b/SPECS/python-execnet/python-execnet.spec @@ -1,30 +1,37 @@ %define pkgname execnet Summary: Python execution distributor Name: python-%{pkgname} -Version: 1.9.0 -Release: 2%{?dist} +Version: 2.1.1 +Release: 1%{?dist} License: MIT URL: https://codespeak.net/execnet/ Vendor: Microsoft Corporation Distribution: Azure Linux Source0: https://pypi.io/packages/source/e/%{pkgname}/%{pkgname}-%{version}.tar.gz -BuildRequires: python3-devel -BuildRequires: python3-setuptools -BuildRequires: python3-setuptools_scm -BuildRequires: python3-wheel +BuildRequires: python%{python3_pkgversion}-devel +BuildRequires: python%{python3_pkgversion}-hatchling +BuildRequires: python%{python3_pkgversion}-hatch-vcs +BuildRequires: python%{python3_pkgversion}-pathspec +BuildRequires: python%{python3_pkgversion}-pluggy +BuildRequires: python%{python3_pkgversion}-setuptools +BuildRequires: python%{python3_pkgversion}-setuptools_scm +BuildRequires: python%{python3_pkgversion}-sphinx +BuildRequires: python%{python3_pkgversion}-trove-classifiers +BuildRequires: python%{python3_pkgversion}-wheel %if %{with check} -BuildRequires: python3-pip +BuildRequires: python%{python3_pkgversion}-pip +BuildRequires: python%{python3_pkgversion}-pytest %endif BuildArch: noarch %description Python execution distributor -%package -n python3-%{pkgname} +%package -n python%{python3_pkgversion}-%{pkgname} Summary: Python execution distributor -Requires: python3 +Requires: python%{python3_pkgversion} -%description -n python3-%{pkgname} +%description -n python%{python3_pkgversion}-%{pkgname} execnet provides carefully tested means to ad-hoc interact with Python interpreters across version, platform and network barriers. It provides a minimal and fast API targetting the following uses: @@ -36,23 +43,38 @@ a minimal and fast API targetting the following uses: %prep %autosetup -n %{pkgname}-%{version} -%build -%py3_build +find . -type f -a \( -name '*.py' -o -name 'py.*' \) \ + -exec sed -i '1{/^#!/d}' {} \; \ + -exec chmod u=rw,go=r {} \; + +%pyproject_buildrequires -t +%build +%pyproject_wheel +make -C doc html PYTHONPATH=$(pwd)/src +# remove hidden file +rm doc/_build/html/.buildinfo + %install -%py3_install +%pyproject_install +%pyproject_save_files %{pkgname} %check -pip3 install tox -sed -i "s/pytest$/pytest==7.1.3/" tox.ini +pip3 install tox iniconfig +# sed -i "s/pytest$/pytest==7.1.3/" tox.ini LANG=en_US.UTF-8 tox -e py%{python3_version_nodots} -%files -n python3-%{pkgname} -%license LICENSE -%doc README.rst CHANGELOG.rst -%{python3_sitelib}/* +%files -n python%{python3_pkgversion}-%{pkgname} -f %{pyproject_files} +%doc README.rst +%doc doc/_build/html +%license %{python3_sitelib}/%{pkgname}-%{version}.dist-info/licenses/LICENSE %changelog +* Wed Apr 24 2024 Osama Esmail - 2.1.1-1 +- Auto-upgrade to 2.1.1 +- Replacing most of the %%py3... with %%pyproject... +- Redoing %%check section as well + * Wed Oct 26 2022 Pawel Winogrodzki - 1.9.0-2 - Freezing 'pytest' test dependency to version 7.1.3. diff --git a/cgmanifest.json b/cgmanifest.json index f8a69d9a3ae..cc66e7373a8 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -22283,8 +22283,8 @@ "type": "other", "other": { "name": "python-execnet", - "version": "1.9.0", - "downloadUrl": "https://pypi.io/packages/source/e/execnet/execnet-1.9.0.tar.gz" + "version": "2.1.1", + "downloadUrl": "https://pypi.io/packages/source/e/execnet/execnet-2.1.1.tar.gz" } } }, From 3c87ca06b603652c54bb61ba84409b0214112015 Mon Sep 17 00:00:00 2001 From: Rachel Menge Date: Mon, 13 May 2024 12:28:11 -0700 Subject: [PATCH 07/51] Remove 10-console-messages.conf from image configs (#9087) 10-console-messages.conf was removed from additional files. Remove lingering references which are causing build failures. --- toolkit/imageconfigs/full-aarch64.json | 8 +++----- toolkit/imageconfigs/full.json | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/toolkit/imageconfigs/full-aarch64.json b/toolkit/imageconfigs/full-aarch64.json index b33f9e3ef68..2bdb1541a64 100644 --- a/toolkit/imageconfigs/full-aarch64.json +++ b/toolkit/imageconfigs/full-aarch64.json @@ -17,8 +17,7 @@ "default": "kernel" }, "AdditionalFiles": { - "additionalconfigs/99-dhcp-en.network": "/etc/systemd/network/99-dhcp-en.network", - "../resources/imageconfigs/additionalfiles/iso_initrd/10-console-messages.conf": "/etc/sysctl.d/10-console-messages.conf" + "additionalconfigs/99-dhcp-en.network": "/etc/systemd/network/99-dhcp-en.network" } }, { @@ -31,9 +30,8 @@ "default": "kernel" }, "AdditionalFiles": { - "additionalconfigs/99-dhcp-en.network": "/etc/systemd/network/99-dhcp-en.network", - "../resources/imageconfigs/additionalfiles/iso_initrd/10-console-messages.conf": "/etc/sysctl.d/10-console-messages.conf" + "additionalconfigs/99-dhcp-en.network": "/etc/systemd/network/99-dhcp-en.network" } } ] -} \ No newline at end of file +} diff --git a/toolkit/imageconfigs/full.json b/toolkit/imageconfigs/full.json index 7b0d3c50a0d..152b78f5968 100644 --- a/toolkit/imageconfigs/full.json +++ b/toolkit/imageconfigs/full.json @@ -20,8 +20,7 @@ "default": "kernel" }, "AdditionalFiles": { - "additionalconfigs/99-dhcp-en.network": "/etc/systemd/network/99-dhcp-en.network", - "../resources/imageconfigs/additionalfiles/iso_initrd/10-console-messages.conf": "/etc/sysctl.d/10-console-messages.conf" + "additionalconfigs/99-dhcp-en.network": "/etc/systemd/network/99-dhcp-en.network" } }, { @@ -35,9 +34,8 @@ "default": "kernel" }, "AdditionalFiles": { - "additionalconfigs/99-dhcp-en.network": "/etc/systemd/network/99-dhcp-en.network", - "../resources/imageconfigs/additionalfiles/iso_initrd/10-console-messages.conf": "/etc/sysctl.d/10-console-messages.conf" + "additionalconfigs/99-dhcp-en.network": "/etc/systemd/network/99-dhcp-en.network" } } ] -} \ No newline at end of file +} From 1237a2c39846d8d32ee8d1764dec7de758846a23 Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Mon, 13 May 2024 13:29:17 -0700 Subject: [PATCH 08/51] Image Customizer: Fix passwords API. (#9070) --- toolkit/Makefile | 1 + toolkit/docs/building/building.md | 1 + toolkit/scripts/tools.mk | 1 + .../imagecustomizer/docs/configuration.md | 117 +++++++++--------- toolkit/tools/imagecustomizerapi/password.go | 36 ++++++ .../tools/imagecustomizerapi/password_test.go | 38 ++++++ .../tools/imagecustomizerapi/passwordtype.go | 14 +++ .../passwordtype_nonprod.go | 22 ++++ .../imagecustomizerapi/passwordtype_prod.go | 24 ++++ toolkit/tools/imagecustomizerapi/user.go | 27 ++-- .../pkg/imagecustomizerlib/customizeutils.go | 41 +++--- 11 files changed, 237 insertions(+), 85 deletions(-) create mode 100644 toolkit/tools/imagecustomizerapi/password.go create mode 100644 toolkit/tools/imagecustomizerapi/password_test.go create mode 100644 toolkit/tools/imagecustomizerapi/passwordtype.go create mode 100644 toolkit/tools/imagecustomizerapi/passwordtype_nonprod.go create mode 100644 toolkit/tools/imagecustomizerapi/passwordtype_prod.go diff --git a/toolkit/Makefile b/toolkit/Makefile index 88c8c94ec48..aa89eaef17f 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -71,6 +71,7 @@ RESOLVE_CYCLES_FROM_UPSTREAM ?= n IGNORE_VERSION_TO_RESOLVE_SELFDEP ?= n CACHED_PACKAGES_ARCHIVE ?= USE_CCACHE ?= n +BUILD_TOOLS_NONPROD ?= n # Tracing & Profiling support: https://go.dev/doc/diagnostics ENABLE_CPU_PROFILE ?= n diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index b5694433af0..1a40db1c8e4 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -828,6 +828,7 @@ To reproduce an ISO build, run the same make invocation as before, but set: | TARGET_ARCH | | The architecture of the machine that will run the package binaries. | USE_CCACHE | n | Use ccache automatically to speed up repeat package builds. | MAX_CPU | | Max number of CPUs used for package building. Use 0 for unlimited. Overrides `%_smp_ncpus_max` macro. +| BUILD_TOOLS_NONPROD | n | Enables non-production features in the go build tools. --- diff --git a/toolkit/scripts/tools.mk b/toolkit/scripts/tools.mk index 225a306c359..4c53155e82c 100644 --- a/toolkit/scripts/tools.mk +++ b/toolkit/scripts/tools.mk @@ -116,6 +116,7 @@ $(TOOL_BINS_DIR)/%: $(go_common_files) go test -ldflags="$(go_ldflags)" -test.short -covermode=atomic -coverprofile=$(BUILD_DIR)/tools/$*.test_coverage ./... && \ CGO_ENABLED=0 go build \ -ldflags="$(go_ldflags)" \ + $(if $(filter y,$(BUILD_TOOLS_NONPROD)),,-tags prod) \ -o $(TOOL_BINS_DIR) endif diff --git a/toolkit/tools/imagecustomizer/docs/configuration.md b/toolkit/tools/imagecustomizer/docs/configuration.md index 9254ddf920b..b67c380d6c2 100644 --- a/toolkit/tools/imagecustomizer/docs/configuration.md +++ b/toolkit/tools/imagecustomizer/docs/configuration.md @@ -163,9 +163,10 @@ os: - [user type](#user-type) - [name](#user-name) - [uid](#uid-int) - - [passwordHashed](#passwordhashed-bool) - - [password](#password-string) - - [passwordPath](#passwordpath-string) + - [password](#password-password) + - [password type](#password-type) + - [type](#password-type-type) + - [value](#password-type-value) - [passwordExpiresDays](#passwordexpiresdays-int) - [sshPublicKeyPaths](#sshpublickeypaths-string) - [primaryGroup](#primarygroup-string) @@ -789,6 +790,57 @@ Supported options: For further details, see: https://en.wikipedia.org/wiki/BIOS_boot_partition +## password type + +Specifies a password for a user. + +WARNING: Passwords should not be used in images used in production. + +This feature is intended for debugging purposes only. +As such, this feature has been disabled in official builds of the Image Customizer tool. + +Instead of using passwords, you should use an authentication system that relies on +cryptographic keys. +For example, SSH with Microsoft Entra ID authentication. + +Example: + +```yaml +os: + users: + - name: test + password: + type: locked +``` + +
+ +### type [string] + +The manner in which the password is provided. + +Supported options: + +- `locked`: Password login is disabled for the user. This is the default behavior. + +Options for debugging purposes only (disabled by default): + +- `plain-text`: The value is a plain-text password. + +- `hashed`: The value is a password that has been pre-hashed. + (For example, by using `openssl passwd`.) + +- `plain-text-file`: The value is a path to a file containing a plain-text password. + +- `hashed-file`: The value is a path to a file containing a pre-hashed password. + +
+ +### value [string] + +The password's value. +The meaning of this value depends on the type property. + ## mountPoint type ### idType [string] @@ -1113,61 +1165,11 @@ os: uid: 1000 ``` -### passwordHashed [bool] +### password [[password](#password-type)] -Default: `false`. +Specifies the user's password. -When set to true, specifies that the password provided by either `password` or -`passwordPath` has already been hashed and may be copied directly into the -`/etc/shadow` file. - -Example: - -```yaml -os: - users: - - name: test - # Generated by: - # PASSWORD="password" - # SALT=$(tr -dc "A-Za-z0-9" < /dev/urandom 2> /dev/null | head -c 12) - # openssl passwd -6 -salt "$SALT" "$PASSWORD" - password: "$6$XH9YwqAMPohT$YQ0fqon.KOXz9AfjP5LE6VHifnNcsIgxmeX/iM5VF1GpFJTOpnTY.UGVRA.Xb8gYdVFqkYnnpJwlaIU1LhNHB/" - passwordHashed: true -``` - -Note: Modern GPUs have gotten incredibly good at brute forcing hashed passwords. -While hashing passwords is still considered best practice, unless the password is -incredibly strong (32+ randomly generated characters), then it is recommended -that you treat a hashed password with the same care as a plain-text password. - -### password [string] - -Sets the user's password. - -Use of this property is strongly discouraged, except when debugging. - -Example: - -```yaml -os: - users: - - name: test - password: testpassword -``` - -### passwordPath [string] - -Sets the user's password. -The password is read from the file path specified. - -Example: - -```yaml -os: - users: - - name: test - passwordPath: test-password.txt -``` +WARNING: Passwords should not be used in images used in production. ### PasswordExpiresDays [int] @@ -1191,6 +1193,9 @@ os: File paths to SSH public key files. These public keys will be copied into the user's `~/.ssh/authorized_keys` file. +Note: It is preferable to use Microsoft Entra ID for SSH authentication, instead of +individual public keys. + Example: ```yaml diff --git a/toolkit/tools/imagecustomizerapi/password.go b/toolkit/tools/imagecustomizerapi/password.go new file mode 100644 index 00000000000..41636223416 --- /dev/null +++ b/toolkit/tools/imagecustomizerapi/password.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerapi + +import ( + "fmt" +) + +type Password struct { + // The way the password is provided. + Type PasswordType `yaml:"type"` + // The value of the password. + Value string `yaml:"value"` +} + +func (p *Password) IsValid() error { + err := p.Type.IsValid() + if err != nil { + return err + } + + switch p.Type { + case PasswordTypeLocked: + if p.Value != "" { + return fmt.Errorf("password value must be empty with type (%s)", p.Type) + } + + case PasswordTypePlainText, PasswordTypeHashed, PasswordTypePlainTextFile, PasswordTypeHashedFile: + if p.Value == "" { + return fmt.Errorf("password value can't be empty with type (%s)", p.Type) + } + } + + return nil +} diff --git a/toolkit/tools/imagecustomizerapi/password_test.go b/toolkit/tools/imagecustomizerapi/password_test.go new file mode 100644 index 00000000000..8a60074638c --- /dev/null +++ b/toolkit/tools/imagecustomizerapi/password_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPasswordIsValid(t *testing.T) { + password := Password{ + Type: PasswordTypeLocked, + Value: "", + } + err := password.IsValid() + assert.NoError(t, err) +} + +func TestPasswordLockedHasValue(t *testing.T) { + password := Password{ + Type: PasswordTypeLocked, + Value: "hello", + } + err := password.IsValid() + assert.Error(t, err) + assert.ErrorContains(t, err, "password value must be empty with type (locked)") +} + +func TestPasswordInvalidType(t *testing.T) { + password := Password{ + Type: "hello", + } + err := password.IsValid() + assert.Error(t, err) + assert.ErrorContains(t, err, "invalid password type (hello)") +} diff --git a/toolkit/tools/imagecustomizerapi/passwordtype.go b/toolkit/tools/imagecustomizerapi/passwordtype.go new file mode 100644 index 00000000000..a7939f471be --- /dev/null +++ b/toolkit/tools/imagecustomizerapi/passwordtype.go @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerapi + +type PasswordType string + +const ( + PasswordTypeLocked PasswordType = "locked" + PasswordTypePlainText PasswordType = "plain-text" + PasswordTypeHashed PasswordType = "hashed" + PasswordTypePlainTextFile PasswordType = "plain-text-file" + PasswordTypeHashedFile PasswordType = "hashed-file" +) diff --git a/toolkit/tools/imagecustomizerapi/passwordtype_nonprod.go b/toolkit/tools/imagecustomizerapi/passwordtype_nonprod.go new file mode 100644 index 00000000000..be65792b65b --- /dev/null +++ b/toolkit/tools/imagecustomizerapi/passwordtype_nonprod.go @@ -0,0 +1,22 @@ +//go:build !prod + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerapi + +import ( + "fmt" +) + +func (t PasswordType) IsValid() error { + switch t { + case PasswordTypeLocked, PasswordTypePlainText, PasswordTypeHashed, PasswordTypePlainTextFile, + PasswordTypeHashedFile: + // All good. + return nil + + default: + return fmt.Errorf("invalid password type (%s)", t) + } +} diff --git a/toolkit/tools/imagecustomizerapi/passwordtype_prod.go b/toolkit/tools/imagecustomizerapi/passwordtype_prod.go new file mode 100644 index 00000000000..436806ee517 --- /dev/null +++ b/toolkit/tools/imagecustomizerapi/passwordtype_prod.go @@ -0,0 +1,24 @@ +//go:build prod + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerapi + +import ( + "fmt" +) + +func (t PasswordType) IsValid() error { + switch t { + case PasswordTypeLocked: + // All good. + return nil + + case PasswordTypePlainText, PasswordTypeHashed, PasswordTypePlainTextFile, PasswordTypeHashedFile: + return fmt.Errorf("password type (%s) only supported in dev builds", t) + + default: + return fmt.Errorf("invalid password type (%s)", t) + } +} diff --git a/toolkit/tools/imagecustomizerapi/user.go b/toolkit/tools/imagecustomizerapi/user.go index bdc98284ba1..57ae4a11c65 100644 --- a/toolkit/tools/imagecustomizerapi/user.go +++ b/toolkit/tools/imagecustomizerapi/user.go @@ -10,17 +10,15 @@ import ( ) type User struct { - Name string `yaml:"name"` - UID *int `yaml:"uid"` - PasswordHashed bool `yaml:"passwordHashed"` - Password string `yaml:"password"` - PasswordPath string `yaml:"passwordPath"` - PasswordExpiresDays *int64 `yaml:"passwordExpiresDays"` - SSHPublicKeyPaths []string `yaml:"sshPublicKeyPaths"` - SSHPublicKeys []string `yaml:"sshPublicKeys"` - PrimaryGroup string `yaml:"primaryGroup"` - SecondaryGroups []string `yaml:"secondaryGroups"` - StartupCommand string `yaml:"startupCommand"` + Name string `yaml:"name"` + UID *int `yaml:"uid"` + Password *Password `yaml:"password"` + PasswordExpiresDays *int64 `yaml:"passwordExpiresDays"` + SSHPublicKeyPaths []string `yaml:"sshPublicKeyPaths"` + SSHPublicKeys []string `yaml:"sshPublicKeys"` + PrimaryGroup string `yaml:"primaryGroup"` + SecondaryGroups []string `yaml:"secondaryGroups"` + StartupCommand string `yaml:"startupCommand"` } func (u *User) IsValid() error { @@ -36,8 +34,11 @@ func (u *User) IsValid() error { } } - if u.Password != "" && u.PasswordPath != "" { - return fmt.Errorf("user (%s) is invalid:\nfields password and passwordPath must not both be specified", u.Name) + if u.Password != nil { + err := u.Password.IsValid() + if err != nil { + return fmt.Errorf("user (%s) is invalid:\n%w", u.Name, err) + } } if u.PasswordExpiresDays != nil { diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go b/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go index e086ac3d7f8..011b80fa03b 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go @@ -299,25 +299,34 @@ func addOrUpdateUser(user imagecustomizerapi.User, baseConfigPath string, imageC logger.Log.Infof("Adding/updating user (%s)", user.Name) - password := user.Password - if user.PasswordPath != "" { - // Read password from file. - passwordFullPath := file.GetAbsPathWithBase(baseConfigPath, user.PasswordPath) + hashedPassword := "" + if user.Password != nil { + passwordIsFile := user.Password.Type == imagecustomizerapi.PasswordTypePlainTextFile || + user.Password.Type == imagecustomizerapi.PasswordTypeHashedFile - passwordFileContents, err := os.ReadFile(passwordFullPath) - if err != nil { - return fmt.Errorf("failed to read password file (%s): %w", passwordFullPath, err) - } + passwordIsHashed := user.Password.Type == imagecustomizerapi.PasswordTypeHashed || + user.Password.Type == imagecustomizerapi.PasswordTypeHashedFile - password = string(passwordFileContents) - } + password := user.Password.Value + if passwordIsFile { + // Read password from file. + passwordFullPath := file.GetAbsPathWithBase(baseConfigPath, user.Password.Value) - // Hash the password. - hashedPassword := password - if !user.PasswordHashed { - hashedPassword, err = userutils.HashPassword(password) - if err != nil { - return err + passwordFileContents, err := os.ReadFile(passwordFullPath) + if err != nil { + return fmt.Errorf("failed to read password file (%s): %w", passwordFullPath, err) + } + + password = string(passwordFileContents) + } + + hashedPassword = password + if !passwordIsHashed { + // Hash the password. + hashedPassword, err = userutils.HashPassword(password) + if err != nil { + return err + } } } From 8693ee40fba6695f7df24196c6366c05111babf2 Mon Sep 17 00:00:00 2001 From: Riken Maharjan <106988478+rikenm1@users.noreply.github.com> Date: Tue, 14 May 2024 10:44:13 -0700 Subject: [PATCH 09/51] Fix tensorflow golden container (#9081) --- .pipelines/containerSourceData/tensorflow/tensorflow.pkg | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.pipelines/containerSourceData/tensorflow/tensorflow.pkg b/.pipelines/containerSourceData/tensorflow/tensorflow.pkg index b0d3dca2efd..ecda39a8d8c 100644 --- a/.pipelines/containerSourceData/tensorflow/tensorflow.pkg +++ b/.pipelines/containerSourceData/tensorflow/tensorflow.pkg @@ -1,13 +1,11 @@ ca-certificates curl python3 -python3-keras python3-numpy python3-pip python3-setuptools -python3-tensorboard python3-tensorboard-data-server python3-tensorboard-plugin-wit +python3-pygments python3-tensorflow -python3-tensorflow-estimator which From 48b1f7bfd27ccdc29f41ca0e293a42fdd28a136f Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Tue, 14 May 2024 14:42:07 -0700 Subject: [PATCH 10/51] Image Customizer: Improve docs for chroot. (#9101) --- toolkit/tools/imagecustomizer/README.md | 43 +++++++++++++++++++++---- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/toolkit/tools/imagecustomizer/README.md b/toolkit/tools/imagecustomizer/README.md index 84d25477d0c..a4bafca69c5 100644 --- a/toolkit/tools/imagecustomizer/README.md +++ b/toolkit/tools/imagecustomizer/README.md @@ -3,10 +3,10 @@ The Azure Linux Image Customizer is a tool that can take an existing generic Azure Linux image and modify it to be suited for particular scenario. -MIC uses [chroot](https://en.wikipedia.org/wiki/Chroot) (and loopback block devices) to -customize the image. -This is the same technology used to build the Azure Linux images (along with most other -Linux distros). +The Image Customizer uses [chroot](https://en.wikipedia.org/wiki/Chroot) (and loopback +block devices) to customize the image. +This is the same technology used to build the Azure Linux images, along with most other +Linux distros. This is in contrast to some other image customization tools, like Packer, which customize the image by booting it inside a VM. @@ -22,9 +22,8 @@ Advantages: Disadvantages: -- Not all Linux tools play nicely when run under chroot. (Though most of the most - common tools do play nicely since they are used to build Linux images under chroot.) - So, some customizations aren't possible to do using MIC. +- Not all Linux tools play nicely when run under chroot. + So, some customizations aren't possible using the Image Customizer. (For example, initializing a Kubernetes cluster node.) ## Getting started @@ -83,3 +82,33 @@ Disadvantages: The customized image is placed in the file that you specified with the `--output-image-file` parameter. You can now use this image as you see fit. (For example, boot it in a Hyper-V VM.) + +## Things to avoid + +The Image Customizer tool provides the option to run custom scripts as part of the +customization process. +These can be used to handle scenarios not covered by the Image Customizer tool. +However, these scripts are only run within a chroot environment, which while it is kind +of similar to containers, is very explicitly not a sandbox environment. +So, such scripts have the ability to modify the host build system. + +In particular, you should be very wary of commands that have the ability to change the +runtime kernel settings. +And even commands that only read runtime kernel settings are probably doing the wrong +thing, since the host build system's kernel is likely entirely unrelated to the +customized OS's kernel. + +Examples of commands to avoid: + +- `ip` +- `iptables` +- `iptables-save` +- `ip6tables-save` +- `sysctl` + +Instead, you should you make use of config files that set the runtime kernel settings +during OS boot. + +Example config directories to use instead: + +- `/etc/sysctl.d` (`systemd-sysctl.service`) From 917e08f1e6d46af7c9efdc4c4a2307c578d4c94a Mon Sep 17 00:00:00 2001 From: Sam Meluch <109628994+sameluch@users.noreply.github.com> Date: Tue, 14 May 2024 18:31:23 -0700 Subject: [PATCH 11/51] Fix PTests for Python Packages Missing Pytest (#9102) --- SPECS/python-jmespath/python-jmespath.spec | 5 +++-- SPECS/python-markdown-it-py/python-markdown-it-py.spec | 10 +++++++++- SPECS/python-mdurl/python-mdurl.spec | 9 ++++++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/SPECS/python-jmespath/python-jmespath.spec b/SPECS/python-jmespath/python-jmespath.spec index fba8199818b..44bdc86ee87 100644 --- a/SPECS/python-jmespath/python-jmespath.spec +++ b/SPECS/python-jmespath/python-jmespath.spec @@ -1,7 +1,7 @@ Summary: Query Language for JSON Name: python-jmespath Version: 1.0.1 -Release: 1%{?dist} +Release: 2%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -12,6 +12,7 @@ BuildRequires: python3-devel BuildRequires: python3-setuptools %if 0%{?with_check} BuildRequires: python3-pip +BuildRequires: python3-pytest %endif BuildArch: noarch @@ -36,7 +37,7 @@ JMESPath (pronounced “james path”) allows you to declaratively specify how t ln -sfv jp.py %{buildroot}%{_bindir}/jp.py-%{python3_version} %check -pip3 install nose mock +pip3 install nose mock iniconfig %python3 setup.py test %files -n python3-jmespath diff --git a/SPECS/python-markdown-it-py/python-markdown-it-py.spec b/SPECS/python-markdown-it-py/python-markdown-it-py.spec index 1e6bd8c7b7d..00ee1db30ef 100644 --- a/SPECS/python-markdown-it-py/python-markdown-it-py.spec +++ b/SPECS/python-markdown-it-py/python-markdown-it-py.spec @@ -9,7 +9,7 @@ High speed & safe by default Summary: Python port of markdown-it Name: python-%{pypi_name} Version: 3.0.0 -Release: 5%{?dist} +Release: 6%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -18,6 +18,10 @@ Source0: %{url}/archive/v%{version}/%{pypi_name}-%{version}.tar.gz BuildRequires: python3-devel BuildRequires: python3-pip BuildRequires: python3-flit-core +%if 0%{?with_check} +BuildRequires: python3-pytest +BuildRequires: python3-mdurl +%endif BuildArch: noarch # The plugins extras creates a bootstrap loop %bcond plugins 1 @@ -52,6 +56,7 @@ sed -i '/"pytest-cov",/d' pyproject.toml %pyproject_save_files markdown_it %check +pip3 install iniconfig linkify-it-py pytest-regressions %pytest tests/ %files -n python3-%{pypi_name} -f %{pyproject_files} @@ -60,6 +65,9 @@ sed -i '/"pytest-cov",/d' pyproject.toml %{_bindir}/markdown-it %changelog +* Mon May 13 2024 Sam Meluch - 3.0.0-6 +- Add missing dep on pytest for check section + * Thu Mar 28 2024 Riken Maharjan - 3.0.0-5 - Initial Azure Linux import from Fedora 40 (license: MIT) - License verified diff --git a/SPECS/python-mdurl/python-mdurl.spec b/SPECS/python-mdurl/python-mdurl.spec index 2a0035a3916..f61c051cf48 100644 --- a/SPECS/python-mdurl/python-mdurl.spec +++ b/SPECS/python-mdurl/python-mdurl.spec @@ -3,7 +3,7 @@ URL utilities for markdown-it parser.} Summary: Markdown URL utilities Name: python-mdurl Version: 0.1.2 -Release: 7%{?dist} +Release: 8%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -12,6 +12,9 @@ Source0: %{url}/archive/%{version}/mdurl-%{version}.tar.gz BuildRequires: python3-devel BuildRequires: python3-pip BuildRequires: python3-flit-core +%if 0%{?with_check} +BuildRequires: python3-pytest +%endif BuildArch: noarch %description %{_description} @@ -39,6 +42,7 @@ sed -i "s/pytest-cov//" tests/requirements.txt %check +pip3 install iniconfig %pytest @@ -47,6 +51,9 @@ sed -i "s/pytest-cov//" tests/requirements.txt %license LICENSE %changelog +* Mon May 13 2024 Sam Meluch - 0.1.2-8 +- Add missing dep on pytest for check section + * Thu Mar 28 2024 Riken Maharjan - 0.1.2-7 - Initial Azure Linux import from Fedora 40 (license: MIT) - License verified From 1e7af5848282df58ba1f85109b8898ace4fbeb32 Mon Sep 17 00:00:00 2001 From: elainezhao96 <102555676+elainezhao96@users.noreply.github.com> Date: Wed, 15 May 2024 11:52:03 -0700 Subject: [PATCH 12/51] python-virt-firmware: package upgrade to 24.4 (#9091) --- .../python-virt-firmware.signatures.json | 2 +- .../python-virt-firmware.spec | 38 ++++++++++--------- cgmanifest.json | 4 +- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/SPECS/python-virt-firmware/python-virt-firmware.signatures.json b/SPECS/python-virt-firmware/python-virt-firmware.signatures.json index 3f560dc6d3e..1c7e2ca7e03 100644 --- a/SPECS/python-virt-firmware/python-virt-firmware.signatures.json +++ b/SPECS/python-virt-firmware/python-virt-firmware.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "python-virt-firmware-23.5.tar.gz": "4939452892d9cfda40c0adccd3b065f7b3b6f7aedaf75d5f021cd9db2a14c5d9" + "virt-firmware-24.4.tar.gz": "aea85a28339010e8fa6dc473dea649f9ae321b5a930bd4948eec4c8599679b05" } } diff --git a/SPECS/python-virt-firmware/python-virt-firmware.spec b/SPECS/python-virt-firmware/python-virt-firmware.spec index 64dafb99ec1..4724ba9c915 100644 --- a/SPECS/python-virt-firmware/python-virt-firmware.spec +++ b/SPECS/python-virt-firmware/python-virt-firmware.spec @@ -11,22 +11,23 @@ Distribution: Azure Linux }%{?-e:.%{-e*}}%{?-s:.%{-s*}}%{!?-n:%{?dist}} ## END: Set by rpmautospec -%global pypi_version 23.5 +%global pypi_version 24.4 Name: python-virt-firmware Version: %{pypi_version} -Release: 2%{?dist} +Release: 1%{?dist} Summary: Tools for virtual machine firmware volumes -License: GPLv2 +License: GPL-2.0-only URL: https://pypi.org/project/virt-firmware/ -Source0: https://files.pythonhosted.org/packages/c2/f8/204dc513d2d3f0f3d3aead03600f7db1b763cf02998ad7b35e7ac5ef6849/virt-firmware-%{pypi_version}.tar.gz#/python-virt-firmware-%{pypi_version}.tar.gz +Source0: https://files.pythonhosted.org/packages/ea/8d/b3417567c9b532879357fb2b6b6fc50a6b0b311f95b16b4845054852e062/virt-firmware-%{pypi_version}.tar.gz BuildArch: noarch BuildRequires: python3-devel BuildRequires: python3dist(cryptography) BuildRequires: python3dist(setuptools) BuildRequires: make help2man +BuildRequires: systemd systemd-rpm-macros %description Tools for ovmf / armvirt firmware volumes This is a small collection of tools @@ -38,21 +39,17 @@ to enroll secure boot certificates. Summary: %{summary} %{?python_provide:%python_provide python3-virt-firmware} Provides: virt-firmware +Conflicts: python3-virt-firmware-peutils < 23.9 +Obsoletes: python3-virt-firmware-peutils < 23.9 Requires: python3dist(cryptography) Requires: python3dist(setuptools) +Requires: python3dist(pefile) %description -n python3-virt-firmware Tools for ovmf / armvirt firmware volumes This is a small collection of tools for edk2 firmware images. They support decoding and printing the content of firmware volumes. Variable stores (OVMF_VARS.fd) can be modified, for example to enroll secure boot certificates. -%package -n python3-virt-firmware-peutils -Summary: %{summary} - peutils -Requires: python3dist(pefile) -Conflicts: python3-virt-firmware < 1.6 -%description -n python3-virt-firmware-peutils -Some utilities to inspect efi (pe) binaries. - %if %{with tests} %package -n python3-virt-firmware-tests Summary: %{summary} - test cases @@ -89,15 +86,19 @@ cp -ar tests %{buildroot}%{_datadir}/%{name} %{_bindir}/virt-fw-vars %{_bindir}/virt-fw-sigdb %{_bindir}/migrate-vars -%{_mandir}/man1/virt-*.1* -%{python3_sitelib}/virt/firmware -%{python3_sitelib}/virt_firmware-%{pypi_version}-py%{python3_version}.egg-info - -%files -n python3-virt-firmware-peutils -%{python3_sitelib}/virt/peutils %{_bindir}/pe-dumpinfo %{_bindir}/pe-listsigs %{_bindir}/pe-addsigs +%{_bindir}/pe-inspect +%{_bindir}/kernel-bootcfg +%{_bindir}/uefi-boot-menu +%{_mandir}/man1/virt-*.1* +%{_mandir}/man1/kernel-bootcfg.1* +%{_mandir}/man1/uefi-boot-menu.1* +%{_mandir}/man1/pe-*.1* +%{python3_sitelib}/virt/firmware +%{python3_sitelib}/virt/peutils +%{python3_sitelib}/virt_firmware-%{pypi_version}-py%{python3_version}.egg-info %if %{with tests} %files -n python3-virt-firmware-tests @@ -105,6 +106,9 @@ cp -ar tests %{buildroot}%{_datadir}/%{name} %endif %changelog +* Mon May 13 2024 Elaine Zhao - 24.4-1 +- update to version 24.4 + * Fri Jun 02 2023 Vince Perri - 23.5-2 - License verified. - Initial CBL-Mariner import from Fedora 39 (license: MIT). diff --git a/cgmanifest.json b/cgmanifest.json index cc66e7373a8..d5d135e3fa5 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -24773,8 +24773,8 @@ "type": "other", "other": { "name": "python-virt-firmware", - "version": "23.5", - "downloadUrl": "https://files.pythonhosted.org/packages/c2/f8/204dc513d2d3f0f3d3aead03600f7db1b763cf02998ad7b35e7ac5ef6849/virt-firmware-23.5.tar.gz" + "version": "24.4", + "downloadUrl": "https://files.pythonhosted.org/packages/ea/8d/b3417567c9b532879357fb2b6b6fc50a6b0b311f95b16b4845054852e062/virt-firmware-24.4.tar.gz" } } }, From a43631cfac1cf672b6160945654bb3761645cc60 Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Wed, 15 May 2024 11:59:34 -0700 Subject: [PATCH 13/51] Image Customizer: Handle grub-mkconfig images. (#9068) --- .../pkg/imagecustomizerlib/bootcustomizer.go | 169 ++++++++++ .../pkg/imagecustomizerlib/customizeboot.go | 231 +++++++------- .../imagecustomizerlib/customizeoverlay.go | 19 +- .../pkg/imagecustomizerlib/customizeutils.go | 72 ++++- .../pkg/imagecustomizerlib/customizeverity.go | 5 + .../imagecustomizerlib/defaultgrubutils.go | 300 ++++++++++++++++++ .../pkg/imagecustomizerlib/imageutils.go | 8 +- .../imagecustomizerlib/liveosisobuilder.go | 5 + 8 files changed, 673 insertions(+), 136 deletions(-) create mode 100644 toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go create mode 100644 toolkit/tools/pkg/imagecustomizerlib/defaultgrubutils.go diff --git a/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go new file mode 100644 index 00000000000..ef5d3e4617d --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerlib + +import ( + "fmt" + "strings" + + "github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi" + "github.com/microsoft/azurelinux/toolkit/tools/imagegen/installutils" + "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" +) + +type BootCustomizer struct { + // The contents of the /boot/grub2/grub.cfg file. + grubCfgContent string + + // The contents of the /etc/default/grub file. + defaultGrubFileContent string + + // Whether or not the image is using grub-mkconfig. + isGrubMkconfig bool +} + +func NewBootCustomizer(imageChroot *safechroot.Chroot) (*BootCustomizer, error) { + grubCfgContent, err := readGrub2ConfigFile(imageChroot) + if err != nil { + return nil, err + } + + defaultGrubFileContent, err := readDefaultGrubFile(imageChroot) + if err != nil { + return nil, err + } + + isGrubMkconfig := isGrubMkconfigConfig(grubCfgContent) + + b := &BootCustomizer{ + grubCfgContent: grubCfgContent, + defaultGrubFileContent: defaultGrubFileContent, + isGrubMkconfig: isGrubMkconfig, + } + return b, nil +} + +// Inserts new kernel command-line args into the grub config file. +func (b *BootCustomizer) AddKernelCommandLine(extraCommandLine string) error { + extraCommandLine = strings.TrimSpace(extraCommandLine) + if extraCommandLine == "" { + return nil + } + + if b.isGrubMkconfig { + defaultGrubFileContent, err := addExtraCommandLineToDefaultGrubFile(b.defaultGrubFileContent, extraCommandLine) + if err != nil { + return err + } + + b.defaultGrubFileContent = defaultGrubFileContent + } else { + // Add the args directly to the /boot/grub2/grub.cfg file. + grubCfgContent, err := appendKernelCommandLineArgs(b.grubCfgContent, extraCommandLine) + if err != nil { + return err + } + + b.grubCfgContent = grubCfgContent + } + + return nil +} + +// Gets the image's configured SELinux mode. +func (b *BootCustomizer) GetSELinuxMode(imageChroot *safechroot.Chroot) (imagecustomizerapi.SELinuxMode, error) { + var err error + var args []grubConfigLinuxArg + + // Get the SELinux kernel command-line args. + if b.isGrubMkconfig { + _, args, _, err = getDefaultGrubFileLinuxArgs(b.defaultGrubFileContent, defaultGrubFileVarNameCmdlineForSELinux) + if err != nil { + return "", err + } + } else { + args, _, err = getLinuxCommandLineArgs(b.grubCfgContent) + if err != nil { + return imagecustomizerapi.SELinuxModeDefault, err + } + } + + // Get the SELinux mode from the kernel command-line args. + selinuxMode, err := getSELinuxModeFromLinuxArgs(args) + if err != nil { + return imagecustomizerapi.SELinuxModeDefault, err + } + + if selinuxMode == imagecustomizerapi.SELinuxModeDefault { + // Get the SELinux mode from the /etc/selinux/config file. + selinuxMode, err = getSELinuxModeFromConfigFile(imageChroot) + if err != nil { + return imagecustomizerapi.SELinuxModeDefault, err + } + } + + return selinuxMode, nil +} + +// Update the image's SELinux kernel command-line args. +func (b *BootCustomizer) UpdateSELinuxCommandLine(selinuxMode imagecustomizerapi.SELinuxMode) error { + newSELinuxArgs, err := selinuxModeToArgs(selinuxMode) + if err != nil { + return err + } + + err = b.UpdateKernelCommandLineArgs(defaultGrubFileVarNameCmdlineForSELinux, selinuxArgNames, newSELinuxArgs) + if err != nil { + return err + } + + return nil +} + +func (b *BootCustomizer) UpdateKernelCommandLineArgs(defaultGrubFileVarName defaultGrubFileVarName, + argsToRemove []string, newArgs []string, +) error { + if b.isGrubMkconfig { + defaultGrubFileContent, err := updateDefaultGrubFileKernelCommandLineArgs(b.defaultGrubFileContent, + defaultGrubFileVarName, argsToRemove, newArgs) + if err != nil { + return err + } + + b.defaultGrubFileContent = defaultGrubFileContent + } else { + grubCfgContent, err := updateKernelCommandLineArgs(b.grubCfgContent, argsToRemove, newArgs) + if err != nil { + return err + } + + b.grubCfgContent = grubCfgContent + } + + return nil +} + +func (b *BootCustomizer) WriteToFile(imageChroot *safechroot.Chroot) error { + if b.isGrubMkconfig { + // Update /etc/defaukt/grub file. + err := writeDefaultGrubFile(b.defaultGrubFileContent, imageChroot) + if err != nil { + return err + } + + // Update /boot/grub2/grub.cfg file. + err = installutils.CallGrubMkconfig(imageChroot) + if err != nil { + return fmt.Errorf("failed to generate grub.cfg via grub2-mkconfig:\n%w", err) + } + } else { + // Update grub.cfg file. + err := writeGrub2ConfigFile(b.grubCfgContent, imageChroot) + if err != nil { + return err + } + } + + return nil +} diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go b/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go index dc6ad7b19a3..d4741e7737c 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go @@ -5,7 +5,6 @@ package imagecustomizerlib import ( "fmt" - "os" "path/filepath" "regexp" "strings" @@ -21,6 +20,8 @@ import ( ) var ( + selinuxArgNames = []string{"security", "selinux", "enforcing"} + // Finds the SELinux mode line in the /etc/selinux/config file. selinuxConfigModeRegex = regexp.MustCompile(`(?m)^SELINUX=(\w+)$`) ) @@ -152,14 +153,13 @@ func setInitrdPath(inputGrubCfgContent string, initrdPath string) (outputGrubCfg // Appends kernel command-line args to the linux command within a grub config file. func appendKernelCommandLineArgs(inputGrubCfgContent string, extraCommandLine string) (outputGrubCfgContent string, err error) { - _, insertAtToken, err := getLinuxCommandLineArgs(inputGrubCfgContent) + _, insertAt, err := getLinuxCommandLineArgs(inputGrubCfgContent) if err != nil { return "", err } // Insert args at the end of the line. - insertPoint := insertAtToken.Loc.Start.Index - outputGrubCfgContent = inputGrubCfgContent[:insertPoint] + extraCommandLine + " " + inputGrubCfgContent[insertPoint:] + outputGrubCfgContent = inputGrubCfgContent[:insertAt] + extraCommandLine + " " + inputGrubCfgContent[insertAt:] return outputGrubCfgContent, nil } @@ -178,33 +178,71 @@ type grubConfigLinuxArg struct { // // Returns: // - args: A list of kernel command-line arguments. -// - insertToken: A token that represents an appropriate insert point for any new args. -// In Azure Linux 2.0, this is the $kernelopts token. -func getLinuxCommandLineArgs(grub2Config string) ([]grubConfigLinuxArg, grub.Token, error) { +// - insertAt: An index that represents an appropriate insert point for any new args. +// For Azure Linux 2.0 images, this points to the index of the $kernelopts token. +func getLinuxCommandLineArgs(grub2Config string) ([]grubConfigLinuxArg, int, error) { linuxLine, err := findLinuxLine(grub2Config) if err != nil { - return nil, grub.Token{}, err + return nil, 0, err } // Skip the "linux" command and the kernel binary path arg. argTokens := linuxLine[2:] - args := []grubConfigLinuxArg(nil) - insertAtToken := (*grub.Token)(nil) + insertAt, err := findCommandLineInsertAt(argTokens) + if err != nil { + return nil, 0, err + } + + args, err := parseCommandLineArgs(argTokens) + if err != nil { + return nil, 0, err + } + + return args, insertAt, nil +} +// Takes a tokenized grub.cfg file and looks for an appropriate place to insert new args. +// Specifically, it looks for the index of the $kernelopts args. +func findCommandLineInsertAt(argTokens []grub.Token) (int, error) { + insertAtTokens := []grub.Token(nil) for i := range argTokens { argToken := argTokens[i] if argToken.Type != grub.WORD { - return nil, grub.Token{}, fmt.Errorf("unexpected token (%s) in grub config linux command", + return 0, fmt.Errorf("unexpected token (%s) in grub config linux command", grub.TokenTypeString(argToken.Type)) } if len(argToken.SubWords) == 1 && argToken.SubWords[0].Type == grub.VAR_EXPANSION && - argToken.SubWords[0].Value == "kernelopts" { + argToken.SubWords[0].Value == grubKernelOpts { // Found the $kernelopts arg. // Any new args to be inserted, will be inserted immediately before this token. - insertAtToken = &argToken + insertAtTokens = append(insertAtTokens, argToken) + } + } + + if len(insertAtTokens) < 1 { + return 0, fmt.Errorf("failed to find $%s in linux command line", grubKernelOpts) + } + if len(insertAtTokens) > 1 { + return 0, fmt.Errorf("too many $%s tokens found in linux command line", grubKernelOpts) + } + + insertAtToken := insertAtTokens[0] + insertAt := insertAtToken.Loc.Start.Index + return insertAt, nil +} + +// Takes a tokenized grub.cfg file and makes a best effort to extract the kernel command-line args. +func parseCommandLineArgs(argTokens []grub.Token) ([]grubConfigLinuxArg, error) { + args := []grubConfigLinuxArg(nil) + + for i := range argTokens { + argToken := argTokens[i] + if argToken.Type != grub.WORD { + return nil, fmt.Errorf("unexpected token (%s) in grub config linux command", + grub.TokenTypeString(argToken.Type)) } hasVarExpansion := false @@ -247,11 +285,7 @@ func getLinuxCommandLineArgs(grub2Config string) ([]grubConfigLinuxArg, grub.Tok args = append(args, arg) } - if insertAtToken == nil { - return nil, grub.Token{}, fmt.Errorf("failed to find $kernelopts in linux command line") - } - - return args, *insertAtToken, nil + return args, nil } // Filters a list of kernel command-line args to only those that match the provided list of names. @@ -287,6 +321,15 @@ func findKernelCommandLineArgValue(args []grubConfigLinuxArg, name string) (stri } // Finds an existing kernel command-line arg and replaces its value. +// +// Params: +// - inputGrubCfgContent: The string contents of the grub.cfg file. +// - name: The name of the command-line arg to replace. +// - value: The value to set the command-line arg to. +// +// Returns: +// - outputGrubCfgContent: The new string contents of the grub.cfg file. +// - oldValue: The previous value of the arg. func replaceKernelCommandLineArgValue(inputGrubCfgContent string, name string, value string, ) (outputGrubCfgContent string, oldValue string, err error) { newArg := fmt.Sprintf("%s=%s", name, value) @@ -314,76 +357,36 @@ func replaceKernelCommandLineArgValue(inputGrubCfgContent string, name string, v return outputGrubCfgContent, oldValue, nil } -// Inserts new kernel command-line args into the grub config file. -func addKernelCommandLine(kernelExtraArgs imagecustomizerapi.KernelExtraArguments, - imageChroot *safechroot.Chroot, -) error { - var err error - - extraCommandLine := strings.TrimSpace(string(kernelExtraArgs)) - if extraCommandLine == "" { - // Nothing to do. - return nil - } - - logger.Log.Infof("Setting KernelCommandLine.ExtraCommandLine") - - grub2ConfigFile, err := readGrub2ConfigFile(imageChroot) - if err != nil { - return err - } - - newGrub2ConfigFile, err := appendKernelCommandLineArgs(grub2ConfigFile, extraCommandLine) - if err != nil { - return err - } - - // Update grub.cfg file. - err = writeGrub2ConfigFile(newGrub2ConfigFile, imageChroot) - if err != nil { - return err - } - - return nil -} - -// Updates the kernel command-line args with the new SELinux mode. +// Finds all the kernel command-line args that match the provided names, then insert replacement arg(s). // -// See, installutils.setGrubCfgSELinux() -func updateSELinuxCommandLine(selinuxMode imagecustomizerapi.SELinuxMode, imageChroot *safechroot.Chroot) error { - logger.Log.Infof("Updating SELinux kernel command-line args") - - grub2Config, err := readGrub2ConfigFile(imageChroot) - if err != nil { - return err - } - - newGrub2Config, err := updateSELinuxCommandLineHelper(grub2Config, selinuxMode) +// Params: +// - grub2Config: The string contents of the grub.cfg file. +// - argsToRemove: A list of arg names to remove from the command-line args. +// - newArgs: A list of new arg values to add to the command-line args. +// +// Output: +// - grub2Config: The new string contents of the grub.cfg file. +func updateKernelCommandLineArgs(grub2Config string, argsToRemove []string, newArgs []string) (string, error) { + args, insertAtToken, err := getLinuxCommandLineArgs(grub2Config) if err != nil { - return err + return "", err } - // Update grub.cfg file. - err = writeGrub2ConfigFile(newGrub2Config, imageChroot) + grub2Config, err = updateKernelCommandLineArgsHelper(grub2Config, args, insertAtToken, argsToRemove, newArgs) if err != nil { - return err + return "", err } - return nil + return grub2Config, nil } -// Finds all the kernel command-line args that match the provided names, then insert replacement arg(s). -func updateKernelCommandLineArgs(grub2Config string, argsToRemove []string, newArgs []string) (string, error) { +func updateKernelCommandLineArgsHelper(value string, args []grubConfigLinuxArg, insertAt int, + argsToRemove []string, newArgs []string, +) (string, error) { newArgsQuoted := grubArgsToString(newArgs) - - args, insertAtToken, err := getLinuxCommandLineArgs(grub2Config) - if err != nil { - return "", err - } - foundArgs := findMatchingCommandLineArgs(args, argsToRemove) - grub2ConfigBuilder := strings.Builder{} + builder := strings.Builder{} nextIndex := 0 if len(foundArgs) > 0 { @@ -391,30 +394,32 @@ func updateKernelCommandLineArgs(grub2Config string, argsToRemove []string, newA for _, arg := range foundArgs { start := arg.Token.Loc.Start.Index end := arg.Token.Loc.End.Index - grub2ConfigBuilder.WriteString(grub2Config[nextIndex:start]) + builder.WriteString(value[nextIndex:start]) nextIndex = end } // Insert the new arg at the location of the last arg. - grub2ConfigBuilder.WriteString(newArgsQuoted) + builder.WriteString(newArgsQuoted) } else { // Write out the grub config to the point where the new arg will be inserted. - insertAt := insertAtToken.Loc.Start.Index - grub2ConfigBuilder.WriteString(grub2Config[nextIndex:insertAt]) + builder.WriteString(value[nextIndex:insertAt]) nextIndex = insertAt // Insert the new arg. - grub2ConfigBuilder.WriteString(newArgsQuoted) - grub2ConfigBuilder.WriteString(" ") + builder.WriteString(" ") + builder.WriteString(newArgsQuoted) + builder.WriteString(" ") } // Write out the remainder of the grub config. - grub2ConfigBuilder.WriteString(grub2Config[nextIndex:]) + builder.WriteString(value[nextIndex:]) - grub2Config = grub2ConfigBuilder.String() - return grub2Config, nil + value = builder.String() + return value, nil } +// Takes a list of unescaped and unquoted kernel command-line args and combines them into a single string with +// appropriate quoting for a grub.cfg file. func grubArgsToString(args []string) string { builder := strings.Builder{} for i, arg := range args { @@ -430,8 +435,8 @@ func grubArgsToString(args []string) string { return combinedString } -// Update the SELinux kernel command-line args. -func updateSELinuxCommandLineHelper(grub2Config string, selinuxMode imagecustomizerapi.SELinuxMode) (string, error) { +// Converts an SELinux mode into the list of required command-line args for that mode. +func selinuxModeToArgs(selinuxMode imagecustomizerapi.SELinuxMode) ([]string, error) { newSELinuxArgs := []string(nil) switch selinuxMode { case imagecustomizerapi.SELinuxModeDisabled: @@ -445,11 +450,20 @@ func updateSELinuxCommandLineHelper(grub2Config string, selinuxMode imagecustomi newSELinuxArgs = []string{installutils.CmdlineSELinuxSecurityArg, installutils.CmdlineSELinuxEnabledArg} default: - return "", fmt.Errorf("unknown SELinux mode (%s)", selinuxMode) + return nil, fmt.Errorf("unknown SELinux mode (%s)", selinuxMode) } - grub2Config, err := updateKernelCommandLineArgs(grub2Config, []string{"security", "selinux", "enforcing"}, - newSELinuxArgs) + return newSELinuxArgs, nil +} + +// Update the SELinux kernel command-line args. +func updateSELinuxCommandLineHelper(grub2Config string, selinuxMode imagecustomizerapi.SELinuxMode) (string, error) { + newSELinuxArgs, err := selinuxModeToArgs(selinuxMode) + if err != nil { + return "", err + } + + grub2Config, err = updateKernelCommandLineArgs(grub2Config, selinuxArgNames, newSELinuxArgs) if err != nil { return "", err } @@ -525,20 +539,10 @@ func replaceSetCommandValue(grub2Config string, varName string, newValue string) return grub2Config, nil } -// Gets the current SELinux mode of an image. -func getCurrentSELinuxMode(imageChroot *safechroot.Chroot) (imagecustomizerapi.SELinuxMode, error) { - logger.Log.Debugf("Get existing SELinux mode") - - grub2Config, err := readGrub2ConfigFile(imageChroot) - if err != nil { - return imagecustomizerapi.SELinuxModeDefault, err - } - - args, _, err := getLinuxCommandLineArgs(grub2Config) - if err != nil { - return "", err - } - +// Takes a list of kernel command-line args and calculates the SELinux mode that is set. +// If the command-line args delegate the SELinux mode to the /etc/selinux/config file, then SELinuxModeDefault ("") is +// returned. +func getSELinuxModeFromLinuxArgs(args []grubConfigLinuxArg) (imagecustomizerapi.SELinuxMode, error) { // Try to find any existing SELinux args. securityValue, err := findKernelCommandLineArgValue(args, "security") if err != nil { @@ -565,14 +569,12 @@ func getCurrentSELinuxMode(imageChroot *safechroot.Chroot) (imagecustomizerapi.S return imagecustomizerapi.SELinuxModeForceEnforcing, nil } - selinuxMode, err := getSELinuxModeFromConfigFile(imageChroot) - if err != nil { - return imagecustomizerapi.SELinuxModeDefault, err - } - - return selinuxMode, nil + // The SELinux mode has been left up to the /etc/selinux/config file. + // Signal this by returning the default ("") value. + return imagecustomizerapi.SELinuxModeDefault, nil } +// Gets the SELinux mode set by the /etc/selinux/config file. func getSELinuxModeFromConfigFile(imageChroot *safechroot.Chroot) (imagecustomizerapi.SELinuxMode, error) { selinuxConfigFilePath := filepath.Join(imageChroot.RootDir(), installutils.SELinuxConfigFile) @@ -607,6 +609,7 @@ func getSELinuxModeFromConfigFile(imageChroot *safechroot.Chroot) (imagecustomiz } } +// Reads the /boot/grub2/grub.cfg file. func readGrub2ConfigFile(imageChroot *safechroot.Chroot) (string, error) { logger.Log.Debugf("Reading grub.cfg file") @@ -615,21 +618,22 @@ func readGrub2ConfigFile(imageChroot *safechroot.Chroot) (string, error) { // Read the existing grub.cfg file. grub2Config, err := file.Read(grub2ConfigFilePath) if err != nil { - return "", fmt.Errorf("failed to read existing grub2 config file (%s):\n%w", installutils.GrubCfgFile, err) + return "", fmt.Errorf("failed to read grub2 config file (%s):\n%w", installutils.GrubCfgFile, err) } return grub2Config, nil } +// Writes the /boot/grub2/grub.cfg file. func writeGrub2ConfigFile(grub2Config string, imageChroot *safechroot.Chroot) error { logger.Log.Debugf("Writing grub.cfg file") grub2ConfigFilePath := getGrub2ConfigFilePath(imageChroot) // Update grub.cfg file. - err := os.WriteFile(grub2ConfigFilePath, []byte(grub2Config), 0) + err := file.Write(grub2Config, grub2ConfigFilePath) if err != nil { - return fmt.Errorf("failed to write new grub2 config file (%s):\n%w", installutils.GrubCfgFile, err) + return fmt.Errorf("failed to write grub2 config file (%s):\n%w", installutils.GrubCfgFile, err) } return nil @@ -639,6 +643,7 @@ func getGrub2ConfigFilePath(imageChroot *safechroot.Chroot) string { return filepath.Join(imageChroot.RootDir(), installutils.GrubCfgFile) } +// Regenerates the initramfs file. func regenerateInitrd(imageChroot *safechroot.Chroot) error { logger.Log.Infof("Regenerate initramfs file") diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeoverlay.go b/toolkit/tools/pkg/imagecustomizerlib/customizeoverlay.go index 2a3289b1d4d..c649a2ab475 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeoverlay.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeoverlay.go @@ -5,11 +5,9 @@ package imagecustomizerlib import ( "fmt" - "path/filepath" "strings" "github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi" - "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" ) @@ -35,7 +33,7 @@ func enableOverlays(overlays *[]imagecustomizerapi.Overlay, imageChroot *safechr overlaysDereference := *overlays err = updateGrubConfigForOverlay(imageChroot, overlaysDereference) if err != nil { - return false, err + return false, fmt.Errorf("failed to update grub config for filesystem overlays:\n%w", err) } return true, nil @@ -71,21 +69,20 @@ func updateGrubConfigForOverlay(imageChroot *safechroot.Chroot, overlays []image fmt.Sprintf("rd.overlayfs=%s", concatenatedOverlays), } - grubCfgPath := filepath.Join(imageChroot.RootDir(), "boot/grub2/grub.cfg") - - grub2Config, err := file.Read(grubCfgPath) + bootCustomizer, err := NewBootCustomizer(imageChroot) if err != nil { - return fmt.Errorf("failed to read grub config:\n%w", err) + return err } - grub2Config, err = updateKernelCommandLineArgs(grub2Config, []string{"rd.overlayfs"}, newArgs) + err = bootCustomizer.UpdateKernelCommandLineArgs(defaultGrubFileVarNameCmdlineLinux, []string{"rd.overlayfs"}, + newArgs) if err != nil { - return fmt.Errorf("failed to set overlay kernel command line args:\n%w", err) + return err } - err = file.Write(grub2Config, grubCfgPath) + err = bootCustomizer.WriteToFile(imageChroot) if err != nil { - return fmt.Errorf("failed to write updated grub config:\n%w", err) + return err } return nil diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go b/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go index 011b80fa03b..676e149aa51 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go @@ -445,7 +445,12 @@ func addCustomizerRelease(imageChroot *safechroot.Chroot, toolVersion string, bu func handleBootLoader(baseConfigPath string, config *imagecustomizerapi.Config, imageConnection *ImageConnection, ) error { - currentSelinuxMode, err := getCurrentSELinuxMode(imageConnection.Chroot()) + bootCustomizer, err := NewBootCustomizer(imageConnection.Chroot()) + if err != nil { + return err + } + + currentSelinuxMode, err := bootCustomizer.GetSELinuxMode(imageConnection.Chroot()) if err != nil { return fmt.Errorf("failed to get existing SELinux mode:\n%w", err) } @@ -475,19 +480,59 @@ func handleBootLoader(baseConfigPath string, config *imagecustomizerapi.Config, return nil } +// Inserts new kernel command-line args into the grub config file. +func addKernelCommandLine(kernelExtraArguments imagecustomizerapi.KernelExtraArguments, + imageChroot *safechroot.Chroot, +) error { + var err error + + if kernelExtraArguments == "" { + // Nothing to do. + return nil + } + + logger.Log.Infof("Setting KernelCommandLine.ExtraCommandLine") + + bootCustomizer, err := NewBootCustomizer(imageChroot) + if err != nil { + return err + } + + err = bootCustomizer.AddKernelCommandLine(string(kernelExtraArguments)) + if err != nil { + return err + } + + err = bootCustomizer.WriteToFile(imageChroot) + if err != nil { + return err + } + + return nil +} + func handleSELinux(selinuxMode imagecustomizerapi.SELinuxMode, resetBootLoaderType imagecustomizerapi.ResetBootLoaderType, imageChroot *safechroot.Chroot, ) (imagecustomizerapi.SELinuxMode, error) { var err error - // Resolve the default SELinux mode. - if selinuxMode == imagecustomizerapi.SELinuxModeDefault { - selinuxMode, err = getCurrentSELinuxMode(imageChroot) - if err != nil { - return selinuxMode, fmt.Errorf("failed to get current SELinux mode:\n%w", err) - } + bootCustomizer, err := NewBootCustomizer(imageChroot) + if err != nil { + return selinuxMode, err + } + + currentSELinuxMode, err := bootCustomizer.GetSELinuxMode(imageChroot) + if err != nil { + return selinuxMode, fmt.Errorf("failed to get current SELinux mode:\n%w", err) } + if selinuxMode == imagecustomizerapi.SELinuxModeDefault || selinuxMode == currentSELinuxMode { + // Don't need to change the configured SELinux mode. + return currentSELinuxMode, nil + } + + logger.Log.Infof("Configuring SELinux mode") + switch resetBootLoaderType { case imagecustomizerapi.ResetBootLoaderTypeHard: // The grub.cfg file has been recreated from scratch and therefore the SELinux args will already be correct and @@ -495,14 +540,19 @@ func handleSELinux(selinuxMode imagecustomizerapi.SELinuxMode, resetBootLoaderTy default: // Update the SELinux kernel command-line args. - err := updateSELinuxCommandLine(selinuxMode, imageChroot) + err := bootCustomizer.UpdateSELinuxCommandLine(selinuxMode) if err != nil { - return selinuxMode, fmt.Errorf("failed to update SELinux args in grub.cfg:\n%w", err) + return selinuxMode, err + } + + err = bootCustomizer.WriteToFile(imageChroot) + if err != nil { + return selinuxMode, err } } if selinuxMode != imagecustomizerapi.SELinuxModeDisabled { - err = updateSELinuxMode(selinuxMode, imageChroot) + err = updateSELinuxModeInConfigFile(selinuxMode, imageChroot) if err != nil { return selinuxMode, err } @@ -511,7 +561,7 @@ func handleSELinux(selinuxMode imagecustomizerapi.SELinuxMode, resetBootLoaderTy return selinuxMode, nil } -func updateSELinuxMode(selinuxMode imagecustomizerapi.SELinuxMode, imageChroot *safechroot.Chroot) error { +func updateSELinuxModeInConfigFile(selinuxMode imagecustomizerapi.SELinuxMode, imageChroot *safechroot.Chroot) error { if selinuxMode == imagecustomizerapi.SELinuxModeDisabled { // SELinux is disabled in the kernel command line. // So, no need to update the SELinux config file. diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeverity.go b/toolkit/tools/pkg/imagecustomizerlib/customizeverity.go index e244ed381b6..a07895e7690 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeverity.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeverity.go @@ -122,6 +122,11 @@ func updateGrubConfig(dataPartitionIdType imagecustomizerapi.IdType, dataPartiti return fmt.Errorf("failed to read grub config:\n%w", err) } + grubMkconfigEnabled := isGrubMkconfigConfig(grub2Config) + if grubMkconfigEnabled { + return fmt.Errorf("grub-mkconfig enabled images not yet supported for verity") + } + grub2Config, err = updateKernelCommandLineArgs(grub2Config, []string{"rd.systemd.verity", "roothash", "systemd.verity_root_data", "systemd.verity_root_hash", "systemd.verity_root_options"}, newArgs) if err != nil { diff --git a/toolkit/tools/pkg/imagecustomizerlib/defaultgrubutils.go b/toolkit/tools/pkg/imagecustomizerlib/defaultgrubutils.go new file mode 100644 index 00000000000..0ef7cbf7859 --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/defaultgrubutils.go @@ -0,0 +1,300 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerlib + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/microsoft/azurelinux/toolkit/tools/imagegen/installutils" + "github.com/microsoft/azurelinux/toolkit/tools/internal/file" + "github.com/microsoft/azurelinux/toolkit/tools/internal/grub" + "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" + "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" +) + +const ( + // A string that is present in the /boot/grub2/grub.cfg when it is generated by grub-mkconfig. + grubMkconfigHeader = "automatically generated by grub2-mkconfig" + + grubKernelOpts = "kernelopts" +) + +type defaultGrubFileVarName string + +const ( + defaultGrubFileVarNameCmdlineLinux defaultGrubFileVarName = "GRUB_CMDLINE_LINUX" + defaultGrubFileVarNameCmdlineLinuxDefault defaultGrubFileVarName = "GRUB_CMDLINE_LINUX_DEFAULT" + + // The variable in the /etc/default/grub file that contains the SELinux args. + defaultGrubFileVarNameCmdlineForSELinux = defaultGrubFileVarNameCmdlineLinux +) + +type defaultGrubFileVarAssign struct { + Token grub.Token + Name string + Value string +} + +// Takes the string contents of a /etc/default/grub file and looks for all the variable assignment statements. +func findDefaultGrubFileVarAssigns(defaultGrubFileContent string) ([]defaultGrubFileVarAssign, error) { + // So, technically the /etc/default/grub file is a Bash file not a grub config file. + // While these are very similar formats, they do have differences. + // But, we are just going to hope that the user isn't annoying enough to Bash specifc features within + // /etc/default/grub file. (e.g. <<< strings.) + grubTokens, err := grub.TokenizeConfig(defaultGrubFileContent) + if err != nil { + return nil, err + } + + varAssigns := []defaultGrubFileVarAssign(nil) + + lines := grub.SplitTokensIntoLines(grubTokens) + for _, line := range lines { + if len(line) != 1 { + // Normal variable assignments only have 1 value. + // Export variable assignments have 2 values. But we are ignoring those. + continue + } + + argToken := line[0] + + isVarAssign := len(argToken.SubWords) >= 1 && + argToken.SubWords[0].Type == grub.KEYWORD_STRING && + strings.Contains(argToken.SubWords[0].Value, "=") + + if !isVarAssign { + continue + } + + argStringBuilder := strings.Builder{} + for _, subword := range argToken.SubWords { + switch subword.Type { + case grub.KEYWORD_STRING, grub.STRING: + argStringBuilder.WriteString(subword.Value) + + case grub.QUOTED_VAR_EXPANSION: + if subword.Value != grubKernelOpts { + return nil, fmt.Errorf("unexpected quoted variable expansion (%s)", subword.Value) + } + + // There is a bug in Azure Linux 2.0 and early builds of Azure Linux 3.0 where "$kernelopts" is not + // properly escaped. + // So, be nice and fix it up. + argStringBuilder.WriteString("$") + argStringBuilder.WriteString(grubKernelOpts) + + case grub.VAR_EXPANSION: + return nil, fmt.Errorf("unexpected variable expansion (%s)", subword.Value) + } + } + + argValue := argStringBuilder.String() + name, value, foundEqSymbol := strings.Cut(argValue, "=") + if !foundEqSymbol { + // Not a variable assignment. + continue + } + + varAssign := defaultGrubFileVarAssign{ + Token: argToken, + Name: name, + Value: value, + } + varAssigns = append(varAssigns, varAssign) + } + + return varAssigns, nil +} + +// Takes the list of variable assignments in a /etc/defaukt/grub file and looks for the assignment to the variables that +// matches the provided name. +func findDefaultGrubFileVarAssign(varAssigns []defaultGrubFileVarAssign, name defaultGrubFileVarName, +) (defaultGrubFileVarAssign, error) { + for _, varAssign := range varAssigns { + if varAssign.Name == string(name) { + return varAssign, nil + } + } + + err := fmt.Errorf("failed to find %s variable assignment (%s)", installutils.GrubDefFile, name) + return defaultGrubFileVarAssign{}, err +} + +// Takes the string contents of a /etc/default/grub file and the name of the command-line args variable (either +// "GRUB_CMDLINE_LINUX" or "GRUB_CMDLINE_LINUX_DEFAULT") and returns a list of kernel command-line args. +// +// Params: +// - defaultGrubFileContent: The string contents of the /etc/default/grub file. +// - varName: The name of variable that contains kernel command-line args. Either "GRUB_CMDLINE_LINUX" or +// "GRUB_CMDLINE_LINUX_DEFAULT". +// +// Returns: +// - cmdLineVarAssign: The variable assignment that matches 'varName'. +// - args: The list of kernel command-line args. +// - insertAt: An index that new kernel command-line args can be inserted at. +func getDefaultGrubFileLinuxArgs(defaultGrubFileContent string, varName defaultGrubFileVarName, +) (defaultGrubFileVarAssign, []grubConfigLinuxArg, int, error) { + varAssigns, err := findDefaultGrubFileVarAssigns(defaultGrubFileContent) + if err != nil { + err = fmt.Errorf("failed to parse %s file:\n%w", installutils.GrubDefFile, err) + return defaultGrubFileVarAssign{}, nil, 0, err + } + + // Find the variable's (e.g. GRUB_CMDLINE_LINUX) line. + cmdLineVarAssign, err := findDefaultGrubFileVarAssign(varAssigns, varName) + if err != nil { + return defaultGrubFileVarAssign{}, nil, 0, err + } + + // The (parsed) string value of the variable is copied verbatim into the grub.cfg file. + // So, args can have quotes if needed, but those quotes will be double escaped. + argsString := cmdLineVarAssign.Value + grubTokens, err := grub.TokenizeConfig(argsString) + if err != nil { + err = fmt.Errorf("failed to parse %s's value:\n%w", varName, err) + return defaultGrubFileVarAssign{}, nil, 0, err + } + + var insertAt int + if varName == defaultGrubFileVarNameCmdlineLinuxDefault { + // GRUB_CMDLINE_LINUX_DEFAULT variable has the $kernelopts arg. + // Any args inserted should be inserted before $kernelopts. + insertAt, err = findCommandLineInsertAt(grubTokens) + if err != nil { + err = fmt.Errorf("failed to parse %s's value args:\n%w", varName, err) + return defaultGrubFileVarAssign{}, nil, 0, err + } + } else { + // Insert args at the end of the string. + insertAt = len(argsString) + } + + args, err := parseCommandLineArgs(grubTokens) + if err != nil { + err = fmt.Errorf("failed to parse %s's value args:\n%w", varName, err) + return defaultGrubFileVarAssign{}, nil, 0, err + } + + return cmdLineVarAssign, args, insertAt, nil +} + +// Takes the string contents of /etc/default/grub file and inserts the provided command-line args. +func addExtraCommandLineToDefaultGrubFile(defaultGrubFileContent string, extraCommandLine string) (string, error) { + cmdLineVarAssign, _, insertAt, err := getDefaultGrubFileLinuxArgs(defaultGrubFileContent, + defaultGrubFileVarNameCmdlineLinuxDefault) + if err != nil { + return "", err + } + + argsString := cmdLineVarAssign.Value + + // Add the extra command-line args. + argsString = argsString[:insertAt] + " " + extraCommandLine + " " + argsString[insertAt:] + + // Rewrite GRUB_CMDLINE_LINUX_DEFAULT line. + defaultGrubFileContent = replaceDefaultGrubFileVarAssign(defaultGrubFileContent, cmdLineVarAssign, argsString) + return defaultGrubFileContent, nil +} + +// Takes the string contents of the /etc/default/grub file and replaces a set of command-line args. +// +// Params: +// - defaultGrubContent: The string contents of the /etc/default/grub file. +// - varName: The name of the variable assignment to modify. Either "GRUB_CMDLINE_LINUX" or +// "GRUB_CMDLINE_LINUX_DEFAULT". +// - argsToRemove: A list of arg names to remove from the kernel command-line. +// - newArgs: A list of new arg values to add to the kernel command-line. +// +// Returns: +// - defaultGrubContent: The new string contents of the /etc/default/grub file. +func updateDefaultGrubFileKernelCommandLineArgs(defaultGrubFileContent string, varName defaultGrubFileVarName, + argsToRemove []string, newArgs []string, +) (string, error) { + cmdLineVarAssign, args, insertAt, err := getDefaultGrubFileLinuxArgs(defaultGrubFileContent, varName) + if err != nil { + return "", err + } + + value := cmdLineVarAssign.Value + value, err = updateKernelCommandLineArgsHelper(value, args, insertAt, argsToRemove, newArgs) + if err != nil { + return "", err + } + + // Rewrite GRUB_CMDLINE_LINUX line. + defaultGrubFileContent = replaceDefaultGrubFileVarAssign(defaultGrubFileContent, cmdLineVarAssign, value) + return defaultGrubFileContent, nil +} + +// Takes the string contents of the /etc/default/grub file and rewrites one of the variable assignments lines. +// +// Params: +// - defaultGrubContent: The string contents of the /etc/default/grub file. +// - varAssign: The variable assignment statement to replace. +// - newValue: The string value to assign to the variable. +func replaceDefaultGrubFileVarAssign(defaultGrubFileContent string, varAssign defaultGrubFileVarAssign, newValue string, +) string { + // Rewrite the GRUB_CMDLINE_LINUX_DEFAULT line. + cmdLineString := fmt.Sprintf("%s=%s", varAssign.Name, grub.ForceQuoteString(newValue)) + + start := varAssign.Token.Loc.Start.Index + end := varAssign.Token.Loc.End.Index + + // Rewrite the /etc/default/grub file. + defaultGrubFileContent = defaultGrubFileContent[:start] + cmdLineString + defaultGrubFileContent[end:] + return defaultGrubFileContent +} + +// Checks if the image uses grub-mkconfig. +func isGrubMkconfigEnabled(imageChroot *safechroot.Chroot) (bool, error) { + grub2ConfigFile, err := readGrub2ConfigFile(imageChroot) + if err != nil { + return false, err + } + + grubMkconfigEnabled := isGrubMkconfigConfig(grub2ConfigFile) + return grubMkconfigEnabled, nil +} + +// Takes the string contents of the grub.cfg file and checks if it was generated by the grub-mkconfig tool. +func isGrubMkconfigConfig(grub2Config string) bool { + grubMkconfigEnabled := strings.Contains(grub2Config, grubMkconfigHeader) + return grubMkconfigEnabled +} + +// Reads the string contents of the /etc/default/grub file. +func readDefaultGrubFile(imageChroot *safechroot.Chroot) (string, error) { + logger.Log.Debugf("Reading %s file", installutils.GrubDefFile) + + grub2ConfigFilePath := getDefaultGrubFilePath(imageChroot) + + // Read the existing grub.cfg file. + grub2Config, err := file.Read(grub2ConfigFilePath) + if err != nil { + return "", fmt.Errorf("failed to read grub file (%s):\n%w", installutils.GrubDefFile, err) + } + + return grub2Config, nil +} + +// Writes the string contents of the /etc/default/grub file. +func writeDefaultGrubFile(grub2Config string, imageChroot *safechroot.Chroot) error { + logger.Log.Debugf("Writing %s file", installutils.GrubDefFile) + + grub2ConfigFilePath := getDefaultGrubFilePath(imageChroot) + + // Update grub.cfg file. + err := file.Write(grub2Config, grub2ConfigFilePath) + if err != nil { + return fmt.Errorf("failed to write grub file (%s):\n%w", installutils.GrubDefFile, err) + } + + return nil +} + +func getDefaultGrubFilePath(imageChroot *safechroot.Chroot) string { + return filepath.Join(imageChroot.RootDir(), installutils.GrubDefFile) +} diff --git a/toolkit/tools/pkg/imagecustomizerlib/imageutils.go b/toolkit/tools/pkg/imagecustomizerlib/imageutils.go index 078a09777af..820510218ed 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imageutils.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imageutils.go @@ -138,6 +138,11 @@ func configureDiskBootLoader(imageConnection *ImageConnection, fileSystems []ima return err } + grubMkconfigEnabled, err := isGrubMkconfigEnabled(imageConnection.Chroot()) + if err != nil { + return err + } + mountPointMap := make(map[string]string) for _, mountPoint := range imageConnection.Chroot().GetMountPoints() { mountPointMap[mountPoint.GetTarget()] = mountPoint.GetSource() @@ -146,7 +151,8 @@ func configureDiskBootLoader(imageConnection *ImageConnection, fileSystems []ima // Configure the boot loader. err = installutils.ConfigureDiskBootloader(imagerBootType, false, false, imagerPartitionSettings, imagerKernelCommandLine, imageConnection.Chroot(), imageConnection.Loopback().DevicePath(), - mountPointMap, diskutils.EncryptedRootDevice{}, diskutils.VerityDevice{}, false /*enableGrubMkconfig*/, true) + mountPointMap, diskutils.EncryptedRootDevice{}, diskutils.VerityDevice{}, grubMkconfigEnabled, + !grubMkconfigEnabled) if err != nil { return fmt.Errorf("failed to install bootloader:\n%w", err) } diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go index d6d9e5d8e3f..98372d918a8 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go @@ -238,6 +238,11 @@ func (b *LiveOSIsoBuilder) updateGrubCfg(grubCfgFileName string, extraCommandLin return err } + grubMkconfigEnabled := isGrubMkconfigConfig(inputContentString) + if grubMkconfigEnabled { + return fmt.Errorf("grub-mkconfig enabled images not yet supported for ISO output") + } + searchCommand := fmt.Sprintf(searchCommandTemplate, isomakerlib.DefaultVolumeId) rootValue := fmt.Sprintf(rootValueTemplate, isomakerlib.DefaultVolumeId) From 509c3465bed7c438c5ea69b872ed6e8aeea74a2c Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Wed, 15 May 2024 13:42:57 -0700 Subject: [PATCH 14/51] Image Customizer: Fix ISO gen without changes. (#9116) --- .../tools/pkg/imagecustomizerlib/imagecustomizer.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go index 18d9eaf3c77..50acf7c08d7 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go @@ -272,7 +272,6 @@ func convertInputImageToWriteableFormat(ic *ImageCustomizerParameters) (*LiveOSI logger.Log.Infof("Converting input image to a writeable format") if ic.inputIsIso { - inputIsoArtifacts, err := createIsoBuilderFromIsoImage(ic.buildDir, ic.buildDirAbs, ic.inputImageFile) if err != nil { return nil, fmt.Errorf("failed to load input iso artifacts:\n%w", err) @@ -288,17 +287,17 @@ func convertInputImageToWriteableFormat(ic *ImageCustomizerParameters) (*LiveOSI return nil, fmt.Errorf("failed to create writeable image:\n%w", err) } } - return inputIsoArtifacts, nil + return inputIsoArtifacts, nil } else { logger.Log.Infof("Creating raw base image: %s", ic.rawImageFile) err := shell.ExecuteLiveWithErr(1, "qemu-img", "convert", "-O", "raw", ic.inputImageFile, ic.rawImageFile) if err != nil { return nil, fmt.Errorf("failed to convert image file to raw format:\n%w", err) } - } - return nil, nil + return nil, nil + } } func customizeOSContents(ic *ImageCustomizerParameters) error { @@ -383,7 +382,6 @@ func customizeOSContents(ic *ImageCustomizerParameters) error { } func convertWriteableFormatToOutputImage(ic *ImageCustomizerParameters, inputIsoArtifacts *LiveOSIsoBuilder) error { - logger.Log.Infof("Converting customized OS partitions into the final image") // Create final output image file if requested. @@ -395,8 +393,9 @@ func convertWriteableFormatToOutputImage(ic *ImageCustomizerParameters, inputIso if err != nil { return fmt.Errorf("failed to convert image file to format: %s:\n%w", ic.outputImageFormat, err) } + case ImageFormatIso: - if ic.customizeOSPartitions { + if ic.customizeOSPartitions || inputIsoArtifacts == nil { err := createLiveOSIsoImage(ic.buildDir, ic.configPath, inputIsoArtifacts, ic.config.Iso, ic.rawImageFile, ic.outputImageDir, ic.outputImageBase) if err != nil { return fmt.Errorf("failed to create LiveOS iso image:\n%w", err) From 1c794aeec02079c087bc69d784f30e225046cde6 Mon Sep 17 00:00:00 2001 From: Betty <38226164+BettyRain@users.noreply.github.com> Date: Wed, 15 May 2024 14:53:09 -0700 Subject: [PATCH 15/51] python-constantly: upgrade to 23.10.4; python-versioneer: add package version 0.29 (#9103) Co-authored-by: Betty Lakes --- LICENSES-AND-NOTICES/SPECS/LICENSES-MAP.md | 2 +- LICENSES-AND-NOTICES/SPECS/data/licenses.json | 1 + .../python-constantly.signatures.json | 4 +- .../python-constantly/python-constantly.spec | 20 ++- .../python-versioneer.signatures.json | 5 + .../python-versioneer/python-versioneer.spec | 129 ++++++++++++++++++ cgmanifest.json | 14 +- 7 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 SPECS/python-versioneer/python-versioneer.signatures.json create mode 100644 SPECS/python-versioneer/python-versioneer.spec diff --git a/LICENSES-AND-NOTICES/SPECS/LICENSES-MAP.md b/LICENSES-AND-NOTICES/SPECS/LICENSES-MAP.md index 816c1155bea..affacecc1d9 100644 --- a/LICENSES-AND-NOTICES/SPECS/LICENSES-MAP.md +++ b/LICENSES-AND-NOTICES/SPECS/LICENSES-MAP.md @@ -5,7 +5,7 @@ The Azure Linux SPEC files originated from a variety of sources with varying lic | CentOS | [MIT](https://www.centos.org/legal/#licensing-policy) | crash-ptdump-command
delve
fstrm
nodejs-nodemon
rhnlib
rt-setup
rt-tests
rtctl
tuned | | Ceph source | [LGPL2.1](https://github.com/ceph/ceph/blob/master/COPYING-LGPL2.1) | ceph | | Debian | [MIT](https://opensource.org/licenses/MIT) | prometheus-process-exporter | -| Fedora | [Fedora MIT License Declaration](https://fedoraproject.org/wiki/Licensing:Main?rd=Licensing#License_of_Fedora_SPEC_Files) | a52dec
abseil-cpp
accountsservice
acpica-tools
acpid
adcli
adobe-mappings-cmap
adobe-mappings-pdf
advancecomp
adwaita-icon-theme
afflib
aide
alsa-firmware
alsa-plugins
amtk
amtterm
annobin
ansible-freeipa
archivemount
argparse-manpage
arptables
arpwatch
asio
aspell
aspell-en
at
at-spi2-atk
at-spi2-core
atf
atk
atop
attr
audiofile
augeas
authbind
authd
authselect
autoconf213
avahi
babeltrace
babeltrace2
babl
baekmuk-ttf-fonts
bats
bcache-tools
biosdevname
blosc
bluez
bmake
bogofilter
bolt
boom-boot
booth
botan2
breezy
brotli
buildah
busybox
bwidget
byacc
ca-certificates
cachefilesd
cairomm
calamares
capstone
catatonit
catch
catch1
cdrdao
celt051
cereal
certmonger
cfitsio
cgdcbxd
chan
CharLS
checkpolicy
checksec
chrony
cim-schema
cjkuni-uming-fonts
cjose
ck
cldr-emoji-annotation
clucene
clutter
clutter-gst3
clutter-gtk
cmocka
cogl
collectd
colm
color-filesystem
colord
colorize
compat-lua
compiler-rt
conda
conmon
conntrack-tools
console-setup
container-exception-logger
containernetworking-plugins
convmv
corosync
corosync-qdevice
cpp-hocon
cppcheck
cpprest
cpptest
cpuid
criu
crypto-policies
cryptsetup
cscope
ctags
CUnit
cups
custodia
Cython
dbus-c++
dbus-python
dbxtool
dconf
dcraw
debootstrap
deltarpm
desktop-file-utils
device-mapper-persistent-data
dietlibc
diffstat
ding-libs
discount
distribution-gpg-keys
dleyna-connector-dbus
dleyna-core
dmraid
dnf
dnf-plugins-core
docbook-dtds
docbook-simple
docbook-slides
docbook-style-dsssl
docbook-utils
docbook2X
docbook5-schemas
docbook5-style-xsl
dogtail
dos2unix
dotconf
dovecot
dpdk
dpkg
driverctl
dropwatch
drpm
duktape
dumpet
dvd+rw-tools
dwarves
dwz
dyninst
ebtables
edac-utils
edk2
efax
efi-rpm-macros
egl-wayland
eglexternalplatform
elinks
enca
enchant
enchant2
enscript
environment-modules
evemu
execstack
exempi
exiv2
extra-cmake-modules
fabtests
facter
fakechroot
fakeroot
fapolicyd
fdk-aac-free
fdupes
fence-virt
fetchmail
fftw
filebench
fio
fipscheck
firewalld
flac
flatbuffers
flite
fltk
fmt
fontawesome-fonts
fontawesome4-fonts
fontpackages
fonts-rpm-macros
foomatic-db
freeglut
freeipmi
freeradius
freetds
freexl
fribidi
fros
frr
fsverity-utils
fuse-overlayfs
fuse-sshfs
fuse-zip
fuse3
future
fxload
gavl
gbenchmark
gconf-editor
GConf2
gcovr
gcr
gdal
gdisk
gdk-pixbuf2
generic-logos
genwqe-tools
geoclue2
GeoIP
GeoIP-GeoLite-data
geolite2
geos
gfs2-utils
ghc-srpm-macros
giflib
gl-manpages
glew
glm
glog
glslang
glusterfs
gnome-desktop-testing
gnome-doc-utils
gnome-icon-theme
gnome-keyring
gnu-efi
go-rpm-macros
gom
google-api-python-client
google-crosextra-caladea-fonts
google-crosextra-carlito-fonts
google-guice
google-noto-cjk-fonts
google-noto-emoji-fonts
google-roboto-slab-fonts
gphoto2
gpm
gpsbabel
graphene
graphite2
graphviz
grubby
gsettings-desktop-schemas
gsl
gsm
gspell
gssdp
gssntlmssp
gstreamer1
gstreamer1-plugins-base
gtk-vnc
gtk2
gtk3
gtkspell
gupnp
gupnp-av
gupnp-dlna
gupnp-igd
hardening-check
hdf
hdf5
heimdal
help2man
hexedit
hicolor-icon-theme
hiera
highlight
hivex
hostname
hping3
hsakmt
htop
hunspell
hunspell-af
hunspell-ar
hunspell-as
hunspell-ast
hunspell-az
hunspell-be
hunspell-bg
hunspell-bn
hunspell-br
hunspell-ca
hunspell-cop
hunspell-csb
hunspell-cv
hunspell-cy
hunspell-da
hunspell-de
hunspell-dsb
hunspell-el
hunspell-en
hunspell-eo
hunspell-es
hunspell-et
hunspell-eu
hunspell-fa
hunspell-fj
hunspell-fo
hunspell-fr
hunspell-fur
hunspell-fy
hunspell-ga
hunspell-gd
hunspell-gl
hunspell-grc
hunspell-gu
hunspell-gv
hunspell-haw
hunspell-hi
hunspell-hil
hunspell-hr
hunspell-hsb
hunspell-ht
hunspell-hu
hunspell-hy
hunspell-ia
hunspell-id
hunspell-is
hunspell-it
hunspell-kk
hunspell-km
hunspell-kn
hunspell-ko
hunspell-ku
hunspell-ky
hunspell-la
hunspell-lb
hunspell-ln
hunspell-mai
hunspell-mg
hunspell-mi
hunspell-mk
hunspell-ml
hunspell-mn
hunspell-mos
hunspell-mr
hunspell-ms
hunspell-mt
hunspell-nds
hunspell-ne
hunspell-nl
hunspell-no
hunspell-nr
hunspell-nso
hunspell-ny
hunspell-om
hunspell-or
hunspell-pa
hunspell-pl
hunspell-pt
hunspell-quh
hunspell-ro
hunspell-ru
hunspell-rw
hunspell-se
hunspell-shs
hunspell-si
hunspell-sk
hunspell-sl
hunspell-smj
hunspell-so
hunspell-sq
hunspell-sr
hunspell-sv
hunspell-sw
hunspell-ta
hunspell-te
hunspell-tet
hunspell-th
hunspell-tk
hunspell-tl
hunspell-tn
hunspell-tpi
hunspell-ts
hunspell-uk
hunspell-uz
hunspell-ve
hunspell-vi
hunspell-wa
hunspell-xh
hunspell-yi
hwdata
hwloc
hyperscan
hyperv-daemons
hyphen
hyphen-as
hyphen-bg
hyphen-bn
hyphen-ca
hyphen-da
hyphen-de
hyphen-el
hyphen-es
hyphen-fa
hyphen-fo
hyphen-fr
hyphen-ga
hyphen-gl
hyphen-grc
hyphen-gu
hyphen-hi
hyphen-hsb
hyphen-hu
hyphen-ia
hyphen-id
hyphen-is
hyphen-it
hyphen-kn
hyphen-ku
hyphen-lt
hyphen-mi
hyphen-ml
hyphen-mn
hyphen-mr
hyphen-nl
hyphen-or
hyphen-pa
hyphen-pl
hyphen-pt
hyphen-ro
hyphen-ru
hyphen-sa
hyphen-sk
hyphen-sl
hyphen-sv
hyphen-ta
hyphen-te
hyphen-tk
hyphen-uk
ibus
ibus-chewing
ibus-hangul
ibus-kkc
ibus-libzhuyin
ibus-m17n
ibus-rawcode
ibus-sayura
ibus-table
ibus-table-chinese
icc-profiles-openicc
icon-naming-utils
icoutils
iftop
iio-sensor-proxy
ilmbase
im-chooser
imaptest
imsettings
indent
infinipath-psm
inih
iniparser
intel-cmt-cat
intel-ipsec-mb
ioping
IP2Location
ipa-pgothic-fonts
ipcalc
ipmitool
iprutils
iptraf-ng
iptstate
irssi
iscsi-initiator-utils
isns-utils
iso-codes
isomd5sum
iw
iwd
jabberpy
jakarta-servlet
jasper
javapackages-bootstrap
javapackages-tools
jbigkit
jdom2
jemalloc
jfsutils
jimtcl
jose
js-jquery
jsoncpp
Judy
jurand
kata-containers
kde-filesystem
kde-settings
kexec-tools
keybinder3
keycloak-httpd-client-install
kf
kf-kconfig
kf-kcoreaddons
kf-ki18n
kf-kwidgetsaddons
kpmcore
kronosnet
ksh
kyotocabinet
kyua
ladspa
lame
langtable
lapack
lasso
latencytop
lato-fonts
lcms2
lcov
ldns
leatherman
ledmon
lensfun
leveldb
lftp
libabw
libaec
libao
libappstream-glib
libarrow
libart_lgpl
libasyncns
libatasmart
libavc1394
libblockdev
libbpf
libbsd
libburn
libbytesize
libcacard
libcanberra
libcdio
libcdio-paranoia
libcdr
libcgroup
libchewing
libcli
libcmis
libcmpiutil
libcomps
libcroco
libdaemon
libdap
libdatrie
libdazzle
libdbi
libdbi-drivers
libdbusmenu
libdc1394
libdeflate
libdmx
libdnf
libdrm
libdvdnav
libdvdread
libdwarf
libeasyfc
libecap
libecb
libell
libEMF
libeot
libepoxy
libepubgen
libesmtp
libetonyek
libev
libevdev
libexif
libexttextcat
libfabric
libfontenc
libfreehand
libftdi
libgadu
libgdither
libgee
libgee06
libgeotiff
libgexiv2
libgit2
libgit2-glib
libglade2
libglvnd
libgovirt
libgphoto2
libgsf
libgta
libguestfs
libgusb
libgxim
libgxps
libhangul
libhugetlbfs
libibcommon
libical
libICE
libicns
libid3tag
libIDL
libidn2
libiec61883
libieee1284
libimobiledevice
libindicator
libinput
libiodbc
libipt
libiptcdata
libiscsi
libisoburn
libisofs
libjcat
libkcapi
libkeepalive
libkkc
libkkc-data
libkml
liblangtag
libldb
libldm
liblerc
liblockfile
liblognorm
liblouis
liblqr-1
liblzf
libmad
libmediaart
libmicrohttpd
libmikmod
libmodman
libmodplug
libmodulemd1
libmpcdec
libmspub
libmtp
libmusicbrainz5
libmwaw
libnbd
libnet
libnetfilter_log
libnfs
libnotify
libntlm
libnumbertext
libnvme
liboauth
libodfgen
libofa
libogg
liboggz
liboil
libomxil-bellagio
libopenraw
liboping
libosinfo
libotf
libotr
libpagemaker
libpaper
libpciaccess
libpeas
libpfm
libpinyin
libplist
libpmemobj-cpp
libpng12
libpng15
libproxy
libpsm2
libpwquality
libqb
libqxp
libraqm
LibRaw
libraw1394
libreport
libreswan
librevenge
librsvg2
librx
libsamplerate
libsass
libsecret
libsemanage
libsigc++20
libsigsegv
libslirp
libSM
libsmbios
libsmi
libsndfile
libsodium
libspiro
libsrtp
libssh
libstaroffice
libstemmer
libstoragemgmt
libtdb
libteam
libtevent
libthai
libtnc
libtomcrypt
libtommath
libtpms
libtracecmd
libtraceevent
libtracefs
libtranslit
libucil
libunicap
libuninameslist
liburing
libusbmuxd
libuser
libutempter
libvarlink
libverto
libvirt-dbus
libvirt-glib
libvirt-java
libvirt-python
libvisio
libvisual
libvoikko
libvorbis
libvpx
libwacom
libwnck3
libwpd
libwpe
libwpg
libwps
libwvstreams
libX11
libXau
libXaw
libxcb
libXcomposite
libxcrypt
libXcursor
libXdamage
libXdmcp
libXext
libxfce4util
libXfixes
libXfont2
libXft
libXi
libXinerama
libxkbcommon
libxkbfile
libxklavier
libxmlb
libXmu
libXpm
libXrandr
libXrender
libXres
libXScrnSaver
libxshmfence
libXt
libXtst
libXv
libXxf86vm
libyami
libyang
libyubikey
libzip
libzmf
lilv
linuxconsoletools
linuxptp
lksctp-tools
lldpd
lockdev
logwatch
lpsolve
lrzsz
lua
lua-expat
lua-filesystem
lua-json
lua-lpeg
lua-lunit
lua-rpm-macros
lua-term
luajit
lujavrite
luksmeta
lutok
lv2
lzip
lzop
m17n-db
m17n-lib
mac-robber
mailcap
mailx
malaga
malaga-suomi-voikko
mallard-rng
man-pages-cs
man-pages-es
man-pages-it
man-pages-ja
man-pages-ko
man-pages-pl
man-pages-ru
man-pages-zh-CN
mandoc
mariadb-connector-c
mariadb-connector-odbc
marisa
maven-compiler-plugin
maven-jar-plugin
maven-resolver
maven-resources-plugin
maven-surefire
maven-wagon
mcelog
mcpp
mcstrans
mdadm
mdds
mdevctl
meanwhile
mecab
mecab-ipadic
media-player-info
memcached
memkind
mesa
mesa-libGLU
metis
microcode_ctl
microdnf
minicom
minizip
mksh
mobile-broadband-provider-info
mock
mock-core-configs
mod_auth_gssapi
mod_auth_mellon
mod_auth_openidc
mod_authnz_pam
mod_fcgid
mod_http2
mod_intercept_form_submit
mod_lookup_identity
mod_md
mod_security
mod_security_crs
mod_wsgi
mokutil
mozjs
mpage
mrtg
mstflint
mt-st
mtdev
mtools
mtr
mtx
munge
mutt
mythes
mythes-bg
mythes-ca
mythes-cs
mythes-da
mythes-de
mythes-el
mythes-en
mythes-eo
mythes-es
mythes-fr
mythes-ga
mythes-hu
mythes-mi
mythes-ne
mythes-nl
mythes-pl
mythes-pt
mythes-ro
mythes-ru
mythes-sk
mythes-sl
mythes-sv
mythes-uk
nbd
nbdkit
neon
netavark
netcdf
netcf
netlabel_tools
netpbm
netsniff-ng
nfs4-acl-tools
nftables
nilfs-utils
nkf
nload
nlopt
nodejs-packaging
nss-mdns
nss-pam-ldapd
nss_nis
nss_wrapper
ntfs-3g
ntfs-3g-system-compression
numad
numatop
numpy
nvmetcli
nvml
oath-toolkit
ocaml
ocaml-alcotest
ocaml-astring
ocaml-augeas
ocaml-base
ocaml-bigarray-compat
ocaml-bisect-ppx
ocaml-calendar
ocaml-camlp5
ocaml-camomile
ocaml-cinaps
ocaml-cmdliner
ocaml-compiler-libs-janestreet
ocaml-cppo
ocaml-csexp
ocaml-csv
ocaml-ctypes
ocaml-curses
ocaml-dune
ocaml-extlib
ocaml-fileutils
ocaml-findlib
ocaml-fmt
ocaml-fpath
ocaml-gettext
ocaml-integers
ocaml-libvirt
ocaml-luv
ocaml-lwt
ocaml-markup
ocaml-migrate-parsetree
ocaml-mmap
ocaml-num
ocaml-ocamlbuild
ocaml-ocplib-endian
ocaml-ounit
ocaml-parsexp
ocaml-ppx-derivers
ocaml-ppxlib
ocaml-re
ocaml-react
ocaml-result
ocaml-seq
ocaml-sexplib
ocaml-sexplib0
ocaml-stdio
ocaml-topkg
ocaml-tyxml
ocaml-uuidm
ocaml-uutf
ocaml-xml-light
ocaml-zarith
ocl-icd
oddjob
ogdi
omping
opa
opal
open-vm-tools
openblas
opencc
opencl-filesystem
opencl-headers
opencryptoki
opencsd
opendnssec
OpenEXR
openjade
openjpeg2
openmpi
openobex
openoffice-lv
openrdate
opensc
openslp
opensm
opensp
openssl
openssl-ibmpkcs11
openssl-pkcs11
openwsman
optipng
opus
opusfile
orangefs
ORBit2
orc
os-prober
osinfo-db
osinfo-db-tools
overpass-fonts
p11-kit
p7zip
pacemaker
pacrunner
pakchois
pam_krb5
pam_wrapper
papi
paps
parallel
patchelf
patchutils
pbzip2
pcp
pcsc-lite
pcsc-lite-ccid
PEGTL
perl
perl-Algorithm-C3
perl-Algorithm-Diff
perl-Alien-Build
perl-Alien-pkgconf
perl-AnyEvent
perl-AnyEvent-AIO
perl-AnyEvent-BDB
perl-App-cpanminus
perl-App-FatPacker
perl-AppConfig
perl-Archive-Extract
perl-Archive-Zip
perl-Authen-SASL
perl-B-COW
perl-B-Debug
perl-B-Hooks-EndOfScope
perl-B-Hooks-OP-Check
perl-B-Keywords
perl-B-Lint
perl-bareword-filehandles
perl-BDB
perl-Bit-Vector
perl-boolean
perl-Browser-Open
perl-BSD-Resource
perl-Business-ISBN
perl-Business-ISBN-Data
perl-Bytes-Random-Secure
perl-Capture-Tiny
perl-Carp-Clan
perl-CBOR-XS
perl-Class-Accessor
perl-Class-C3
perl-Class-C3-XS
perl-Class-Data-Inheritable
perl-Class-Factory-Util
perl-Class-Inspector
perl-Class-ISA
perl-Class-Load
perl-Class-Load-XS
perl-Class-Method-Modifiers
perl-Class-Singleton
perl-Class-Tiny
perl-Class-XSAccessor
perl-Clone
perl-Color-ANSI-Util
perl-Color-RGB-Util
perl-ColorThemeBase-Static
perl-ColorThemeRole-ANSI
perl-ColorThemes-Standard
perl-ColorThemeUtil-ANSI
perl-Compress-Bzip2
perl-Compress-LZF
perl-Compress-Raw-Lzma
perl-Config-AutoConf
perl-Config-INI
perl-Config-INI-Reader-Multiline
perl-Config-IniFiles
perl-Config-Simple
perl-Config-Tiny
perl-Const-Fast
perl-Convert-ASN1
perl-Convert-Bencode
perl-Coro
perl-Coro-Multicore
perl-CPAN-Changes
perl-CPAN-DistnameInfo
perl-CPAN-Meta-Check
perl-Cpanel-JSON-XS
perl-Crypt-CBC
perl-Crypt-DES
perl-Crypt-IDEA
perl-Crypt-OpenSSL-Bignum
perl-Crypt-OpenSSL-Guess
perl-Crypt-OpenSSL-Random
perl-Crypt-OpenSSL-RSA
perl-Crypt-PasswdMD5
perl-Crypt-Random-Seed
perl-CSS-Tiny
perl-Data-Dump
perl-Data-Munge
perl-Data-OptList
perl-Data-Peek
perl-Data-Section
perl-Data-UUID
perl-Date-Calc
perl-Date-ISO8601
perl-Date-Manip
perl-DateTime
perl-DateTime-Format-Builder
perl-DateTime-Format-DateParse
perl-DateTime-Format-HTTP
perl-DateTime-Format-IBeat
perl-DateTime-Format-ISO8601
perl-DateTime-Format-Mail
perl-DateTime-Format-Strptime
perl-DateTime-Locale
perl-DateTime-TimeZone
perl-DateTime-TimeZone-SystemV
perl-DateTime-TimeZone-Tzfile
perl-DBD-MySQL
perl-Devel-CallChecker
perl-Devel-Caller
perl-Devel-CheckBin
perl-Devel-CheckLib
perl-Devel-Cycle
perl-Devel-EnforceEncapsulation
perl-Devel-GlobalDestruction
perl-Devel-GlobalDestruction-XS
perl-Devel-Hide
perl-Devel-Leak
perl-Devel-LexAlias
perl-Devel-Size
perl-Devel-StackTrace
perl-Devel-Symdump
perl-Digest-BubbleBabble
perl-Digest-CRC
perl-Digest-HMAC
perl-Digest-SHA1
perl-Dist-CheckConflicts
perl-DynaLoader-Functions
perl-Email-Address
perl-Email-Date-Format
perl-Encode-Detect
perl-Encode-EUCJPASCII
perl-Encode-IMAPUTF7
perl-Encode-Locale
perl-Env-ShellWords
perl-Error
perl-EV
perl-Eval-Closure
perl-Event
perl-Exception-Class
perl-Expect
perl-ExtUtils-Config
perl-ExtUtils-Depends
perl-ExtUtils-Helpers
perl-ExtUtils-InstallPaths
perl-ExtUtils-PkgConfig
perl-FCGI
perl-Fedora-VSP
perl-FFI-CheckLib
perl-File-BaseDir
perl-File-BOM
perl-File-chdir
perl-File-CheckTree
perl-File-Copy-Recursive
perl-File-DesktopEntry
perl-File-Find-Object
perl-File-Find-Object-Rule
perl-File-Find-Rule
perl-File-Find-Rule-Perl
perl-File-Inplace
perl-File-Listing
perl-File-MimeInfo
perl-File-pushd
perl-File-ReadBackwards
perl-File-Remove
perl-File-ShareDir
perl-File-ShareDir-Install
perl-File-Slurp
perl-File-Slurp-Tiny
perl-File-Slurper
perl-File-Type
perl-Font-TTF
perl-FreezeThaw
perl-GD
perl-GD-Barcode
perl-generators
perl-Getopt-ArgvFile
perl-gettext
perl-Graphics-ColorNamesLite-WWW
perl-GSSAPI
perl-Guard
perl-Hook-LexWrap
perl-HTML-Parser
perl-HTML-Tagset
perl-HTML-Tree
perl-HTTP-Cookies
perl-HTTP-Daemon
perl-HTTP-Date
perl-HTTP-Message
perl-HTTP-Negotiate
perl-Image-Base
perl-Image-Info
perl-Image-Xbm
perl-Image-Xpm
perl-Import-Into
perl-Importer
perl-inc-latest
perl-indirect
perl-Inline-Files
perl-IO-AIO
perl-IO-All
perl-IO-CaptureOutput
perl-IO-Compress-Lzma
perl-IO-HTML
perl-IO-Multiplex
perl-IO-SessionData
perl-IO-Socket-INET6
perl-IO-String
perl-IO-stringy
perl-IO-Tty
perl-IPC-Run
perl-IPC-Run3
perl-IPC-System-Simple
perl-JSON
perl-JSON-Color
perl-JSON-MaybeXS
perl-LDAP
perl-libnet
perl-libwww-perl
perl-libxml-perl
perl-Lingua-EN-Inflect
perl-List-MoreUtils-XS
perl-local-lib
perl-Locale-Codes
perl-Locale-Maketext-Gettext
perl-Locale-Msgfmt
perl-Locale-PO
perl-Log-Message
perl-Log-Message-Simple
perl-LWP-MediaTypes
perl-LWP-Protocol-https
perl-Mail-AuthenticationResults
perl-Mail-DKIM
perl-Mail-IMAPTalk
perl-Mail-SPF
perl-MailTools
perl-Math-Int64
perl-Math-Random-ISAAC
perl-MIME-Charset
perl-MIME-Lite
perl-MIME-Types
perl-Mixin-Linewise
perl-MLDBM
perl-Mock-Config
perl-Module-Build-Tiny
perl-Module-CPANfile
perl-Module-Implementation
perl-Module-Install-AuthorRequires
perl-Module-Install-AuthorTests
perl-Module-Install-AutoLicense
perl-Module-Install-GithubMeta
perl-Module-Install-ManifestSkip
perl-Module-Install-ReadmeFromPod
perl-Module-Install-ReadmeMarkdownFromPod
perl-Module-Install-Repository
perl-Module-Install-TestBase
perl-Module-Load-Util
perl-Module-Manifest
perl-Module-Manifest-Skip
perl-Module-Package
perl-Module-Package-Au
perl-Module-Pluggable
perl-Module-Runtime
perl-Module-Signature
perl-Mojolicious
perl-Moo
perl-Mozilla-CA
perl-Mozilla-LDAP
perl-MRO-Compat
perl-multidimensional
perl-namespace-autoclean
perl-namespace-clean
perl-Net-CIDR-Lite
perl-Net-Daemon
perl-Net-DNS
perl-Net-DNS-Resolver-Mock
perl-Net-DNS-Resolver-Programmable
perl-Net-HTTP
perl-Net-IMAP-Simple
perl-Net-IMAP-Simple-SSL
perl-Net-IP
perl-Net-LibIDN2
perl-Net-Patricia
perl-Net-SMTP-SSL
perl-Net-SNMP
perl-Net-Telnet
perl-Newt
perl-NNTPClient
perl-NTLM
perl-Number-Compare
perl-Object-Deadly
perl-Object-HashBase
perl-Package-Anon
perl-Package-Constants
perl-Package-DeprecationManager
perl-Package-Generator
perl-Package-Stash
perl-Package-Stash-XS
perl-PadWalker
perl-Paper-Specs
perl-PAR-Dist
perl-Parallel-Iterator
perl-Params-Classify
perl-Params-Util
perl-Params-Validate
perl-Params-ValidationCompiler
perl-Parse-PMFile
perl-Parse-RecDescent
perl-Parse-Yapp
perl-Path-Tiny
perl-Perl-Critic
perl-Perl-Critic-More
perl-Perl-Destruct-Level
perl-Perl-MinimumVersion
perl-Perl4-CoreLibs
perl-PerlIO-gzip
perl-PerlIO-utf8_strict
perl-PkgConfig-LibPkgConf
perl-Pod-Coverage
perl-Pod-Coverage-TrustPod
perl-Pod-Escapes
perl-Pod-Eventual
perl-Pod-LaTeX
perl-Pod-Markdown
perl-Pod-Parser
perl-Pod-Plainer
perl-Pod-POM
perl-Pod-Spell
perl-PPI
perl-PPI-HTML
perl-PPIx-QuoteLike
perl-PPIx-Regexp
perl-PPIx-Utilities
perl-prefork
perl-Probe-Perl
perl-Razor-Agent
perl-Readonly
perl-Readonly-XS
perl-Ref-Util
perl-Ref-Util-XS
perl-Regexp-Pattern-Perl
perl-Return-MultiLevel
perl-Role-Tiny
perl-Scope-Guard
perl-Scope-Upper
perl-SGMLSpm
perl-SNMP_Session
perl-Socket6
perl-Software-License
perl-Sort-Versions
perl-Specio
perl-Spiffy
perl-strictures
perl-String-CRC32
perl-String-Format
perl-String-ShellQuote
perl-String-Similarity
perl-Sub-Exporter
perl-Sub-Exporter-Progressive
perl-Sub-Identify
perl-Sub-Info
perl-Sub-Install
perl-Sub-Name
perl-Sub-Quote
perl-Sub-Uplevel
perl-SUPER
perl-Switch
perl-Syntax-Highlight-Engine-Kate
perl-Sys-CPU
perl-Sys-MemInfo
perl-Sys-Virt
perl-Taint-Runtime
perl-Task-Weaken
perl-Term-Size-Any
perl-Term-Size-Perl
perl-Term-Table
perl-Term-UI
perl-TermReadKey
perl-Test-Base
perl-Test-ClassAPI
perl-Test-CPAN-Meta
perl-Test-CPAN-Meta-JSON
perl-Test-Deep
perl-Test-Differences
perl-Test-DistManifest
perl-Test-Distribution
perl-Test-EOL
perl-Test-Exception
perl-Test-Exit
perl-Test-FailWarnings
perl-Test-Fatal
perl-Test-File
perl-Test-File-ShareDir
perl-Test-Harness
perl-Test-HasVersion
perl-Test-InDistDir
perl-Test-Inter
perl-Test-LeakTrace
perl-Test-LongString
perl-Test-Manifest
perl-Test-Memory-Cycle
perl-Test-MinimumVersion
perl-Test-MockObject
perl-Test-MockRandom
perl-Test-Needs
perl-Test-NoTabs
perl-Test-NoWarnings
perl-Test-Object
perl-Test-Output
perl-Test-Pod
perl-Test-Pod-Coverage
perl-Test-Portability-Files
perl-Test-Requires
perl-Test-RequiresInternet
perl-Test-Script
perl-Test-Simple
perl-Test-SubCalls
perl-Test-Synopsis
perl-Test-Taint
perl-Test-TrailingSpace
perl-Test-utf8
perl-Test-Vars
perl-Test-Warn
perl-Test-Without-Module
perl-Test2-Plugin-NoWarnings
perl-Test2-Suite
perl-Test2-Tools-Explain
perl-Text-CharWidth
perl-Text-CSV_XS
perl-Text-Diff
perl-Text-Glob
perl-Text-Iconv
perl-Text-Soundex
perl-Text-Unidecode
perl-Text-WrapI18N
perl-Tie-IxHash
perl-TimeDate
perl-Tree-DAG_Node
perl-Unicode-EastAsianWidth
perl-Unicode-LineBreak
perl-Unicode-Map8
perl-Unicode-String
perl-Unicode-UTF8
perl-UNIVERSAL-can
perl-UNIVERSAL-isa
perl-Unix-Syslog
perl-URI
perl-Variable-Magic
perl-Version-Requirements
perl-WWW-RobotRules
perl-XML-Catalog
perl-XML-DOM
perl-XML-Dumper
perl-XML-Filter-BufferText
perl-XML-Generator
perl-XML-Grove
perl-XML-Handler-YAWriter
perl-XML-LibXML
perl-XML-LibXSLT
perl-XML-NamespaceSupport
perl-XML-Parser-Lite
perl-XML-RegExp
perl-XML-SAX
perl-XML-SAX-Base
perl-XML-SAX-Writer
perl-XML-Simple
perl-XML-TokeParser
perl-XML-TreeBuilder
perl-XML-Twig
perl-XML-Writer
perl-XML-XPath
perl-XML-XPathEngine
perl-XString
perl-YAML-LibYAML
perl-YAML-PP
perl-YAML-Syck
perltidy
pesign
phodav
php
php-pear
php-pecl-zip
physfs
picosat
pinfo
pipewire
pixman
pkcs11-helper
pkgconf
plexus-cipher
plexus-containers
plexus-sec-dispatcher
plotutils
pmdk-convert
pmix
pngcrush
pngnq
po4a
podman
poetry
policycoreutils
polkit-pkla-compat
polkit-qt-1
portreserve
postfix
potrace
powertop
ppp
pps-tools
pptp
priv_wrapper
procmail
prometheus
prometheus-node-exporter
ps_mem
psacct
psutils
ptlib
publicsuffix-list
pugixml
pulseaudio
puppet
pwgen
pyatspi
pybind11
pycairo
pyelftools
pyflakes
pygobject3
PyGreSQL
pykickstart
pylint
pyparted
pyproject-rpm-macros
pyserial
python-absl-py
python-aiodns
python-aiohttp
python-alsa
python-argcomplete
python-astroid
python-astunparse
python-async-generator
python-augeas
python-azure-sdk
python-beautifulsoup4
python-betamax
python-blinker
python-blivet
python-cached_property
python-charset-normalizer
python-cheetah
python-click
python-cmd2
python-colorama
python-CommonMark
python-conda-package-handling
python-configshell
python-cpuinfo
python-cups
python-curio
python-cytoolz
python-d2to1
python-dbus-client-gen
python-dbus-python-client-gen
python-dbus-signature-pyparsing
python-dbusmock
python-ddt
python-debtcollector
python-decorator
python-distlib
python-dmidecode
python-dns
python-dtopt
python-dulwich
python-editables
python-enchant
python-entrypoints
python-ethtool
python-evdev
python-extras
python-faker
python-fasteners
python-fastjsonschema
python-fields
python-filelock
python-fixtures
python-flake8
python-flask
python-flit
python-flit-core
python-fluidity-sm
python-frozendict
python-funcsigs
python-gast
python-genshi
python-google-auth
python-google-auth-oauthlib
python-greenlet
python-gssapi
python-h5py
python-hatch-fancy-pypi-readme
python-hatch-vcs
python-hatchling
python-hs-dbus-signature
python-html5lib
python-httplib2
python-humanize
python-hwdata
python-importlib-metadata
python-inotify
python-into-dbus-python
python-IPy
python-iso8601
python-isodate
python-isort
python-itsdangerous
python-justbases
python-justbytes
python-jwcrypto
python-jwt
python-kdcproxy
python-kerberos
python-kmod
python-kubernetes
python-lark
python-lazy-object-proxy
python-ldap
python-linux-procfs
python-lit
python-looseversion
python-markdown
python-markdown-it-py
python-mccabe
python-mdurl
python-memcached
python-mimeparse
python-mock
python-monotonic
python-more-itertools
python-mpmath
python-msal
python-msrestazure
python-mutagen
python-networkx
python-nose2
python-ntlm-auth
python-oauth2client
python-openpyxl
python-openstackdocstheme
python-oslo-i18n
python-oslo-sphinx
python-paramiko
python-pathspec
python-pefile
python-pexpect
python-pkgconfig
python-platformdirs
python-pluggy
python-podman-api
python-poetry-core
python-process-tests
python-productmd
python-ptyprocess
python-pycares
python-pycosat
python-pydbus
python-pymongo
python-PyMySQL
python-pyperclip
python-pyroute2
python-pyrsistent
python-pysocks
python-pytest-benchmark
python-pytest-cov
python-pytest-expect
python-pytest-flake8
python-pytest-flakes
python-pytest-forked
python-pytest-mock
python-pytest-relaxed
python-pytest-runner
python-pytest-subtests
python-pytest-timeout
python-pytest-xdist
python-pytoml
python-pyudev
python-pywbem
python-qrcode
python-rdflib
python-recommonmark
python-requests-file
python-requests-ftp
python-requests-kerberos
python-requests-mock
python-requests-oauthlib
python-requests-toolbelt
python-requests_ntlm
python-responses
python-retrying
python-rfc3986
python-rich
python-rpm-generators
python-rpmfluff
python-rtslib
python-ruamel-yaml
python-ruamel-yaml-clib
python-s3transfer
python-schedutils
python-semantic_version
python-should_dsl
python-simpleline
python-slip
python-sniffio
python-sortedcontainers
python-soupsieve
python-sphinx
python-sphinx-epytext
python-sphinx-theme-py3doc-enhanced
python-sphinx_rtd_theme
python-sphinxcontrib-apidoc
python-sphinxcontrib-applehelp
python-sphinxcontrib-devhelp
python-sphinxcontrib-htmlhelp
python-sphinxcontrib-httpdomain
python-sphinxcontrib-jsmath
python-sphinxcontrib-qthelp
python-sphinxcontrib-serializinghtml
python-sqlalchemy
python-suds
python-systemd
python-tempita
python-templated-dictionary
python-termcolor
python-testpath
python-testresources
python-testscenarios
python-testtools
python-tidy
python-toml
python-tomli
python-toolz
python-tornado
python-tox
python-tox-current-env
python-tqdm
python-trio
python-trove-classifiers
python-typing-extensions
python-uamqp
python-unittest2
python-uritemplate
python-urwid
python-varlink
python-virt-firmware
python-voluptuous
python-waitress
python-webencodings
python-webtest
python-wheel
python-whoosh
python-winrm
python-wrapt
python-xmltodict
python-yubico
python-zipp
python-zmq
python-zstd
python3-mallard-ducktype
python3-pytest-asyncio
python3-typed_ast
pyusb
pywbem
pyxattr
qemu
qhull
qpdf
qperf
qr-code-generator
qt-rpm-macros
qt5-qtconnectivity
qt5-qtsensors
qt5-qtserialport
qtbase
qtdeclarative
qtsvg
qttools
quagga
quota
radvd
ragel
raptor2
rarian
rasdaemon
rasqal
rcs
rdist
rdma-core
re2
re2c
realmd
rear
recode
redland
resource-agents
rest
rhash
rlwrap
rp-pppoe
rpm-mpi-hooks
rpmdevtools
rpmlint
rtkit
rtl-sdr
ruby-augeas
rubygem-bson
rubygem-coderay
rubygem-diff-lcs
rubygem-flexmock
rubygem-hpricot
rubygem-introspection
rubygem-liquid
rubygem-maruku
rubygem-metaclass
rubygem-mongo
rubygem-mustache
rubygem-mysql2
rubygem-pkg-config
rubygem-rake
rubygem-rake-compiler
rubygem-ronn
rubygem-rouge
rubygem-rspec
rubygem-rspec-expectations
rubygem-rspec-mocks
rubygem-rspec-support
rubygem-thread_order
rusers
rust-cbindgen
samba
sanlock
sassist
satyr
sbc
sblim-cim-client2
sblim-cmpi-base
sblim-cmpi-devel
sblim-cmpi-fsvol
sblim-cmpi-network
sblim-cmpi-nfsv3
sblim-cmpi-nfsv4
sblim-cmpi-params
sblim-cmpi-sysfs
sblim-cmpi-syslog
sblim-indication_helper
sblim-sfcb
sblim-sfcc
sblim-sfcCommon
sblim-testsuite
sblim-wbemcli
scl-utils
scotch
screen
scrub
SDL
SDL2
SDL_sound
sdparm
seabios
secilc
selinux-policy
sendmail
serd
setools
setserial
setuptool
sgabios
sgml-common
sgpio
shared-mime-info
sharutils
sip
sisu
skkdic
sleuthkit
slirp4netns
smartmontools
smc-tools
socket_wrapper
softhsm
sombok
sord
sos
sound-theme-freedesktop
soundtouch
sox
soxr
sparsehash
spausedd
speex
speexdsp
spice-protocol
spice-vdagent
spirv-headers
spirv-tools
splix
squashfs-tools
squid
sratom
sscg
star
startup-notification
stunnel
subscription-manager
suitesparse
SuperLU
supermin
switcheroo-control
swtpm
symlinks
sympy
sysfsutils
systemd
systemd-bootchart
t1lib
t1utils
taglib
tang
targetcli
tbb
tcl-pgtcl
tclx
teckit
telnet
thrift
tidy
time
tini
tinycdb
tix
tk
tlog
tmpwatch
tn5250
tofrodos
tokyocabinet
trace-cmd
tss2
ttembed
ttmkfdir
tuna
twolame
uchardet
uclibc-ng
ucpp
ucs-miscfixed-fonts
ucx
udftools
udica
udisks2
uglify-js
uid_wrapper
unicode-emoji
unicode-ucd
unique3
units
upower
uriparser
urlview
usb_modeswitch
usb_modeswitch-data
usbguard
usbip
usbmuxd
usbredir
usermode
ustr
uthash
uuid
uw-imap
v4l-utils
vhostmd
vino
virglrenderer
virt-p2v
virt-top
virt-what
virt-who
vitess
vmem
volume_key
vorbis-tools
vte291
vulkan-headers
vulkan-loader
watchdog
wavpack
wayland
wayland-protocols
web-assets
webrtc-audio-processing
websocketpp
wget
whois
wireguard-tools
wireless-regdb
wireshark
woff2
wordnet
words
wpebackend-fdo
wsmancli
wvdial
x3270
xapian-core
Xaw3d
xcb-proto
xcb-util
xcb-util-image
xcb-util-keysyms
xcb-util-renderutil
xcb-util-wm
xdelta
xdg-dbus-proxy
xdg-utils
xdp-tools
xerces-c
xfconf
xfsdump
xhtml1-dtds
xkeyboard-config
xmlstarlet
xmltoman
xmvn
xorg-x11-apps
xorg-x11-drv-libinput
xorg-x11-font-utils
xorg-x11-fonts
xorg-x11-proto-devel
xorg-x11-server
xorg-x11-server-utils
xorg-x11-util-macros
xorg-x11-utils
xorg-x11-xauth
xorg-x11-xbitmaps
xorg-x11-xinit
xorg-x11-xkb-utils
xorg-x11-xtrans-devel
xrestop
xterm
xxhash
yajl
yaml-cpp
yasm
yelp-tools
yelp-xsl
ykclient
yp-tools
ypbind
ypserv
z3
zenity
zerofree
zfs-fuse
zipper
zopfli
zziplib | +| Fedora | [Fedora MIT License Declaration](https://fedoraproject.org/wiki/Licensing:Main?rd=Licensing#License_of_Fedora_SPEC_Files) | a52dec
abseil-cpp
accountsservice
acpica-tools
acpid
adcli
adobe-mappings-cmap
adobe-mappings-pdf
advancecomp
adwaita-icon-theme
afflib
aide
alsa-firmware
alsa-plugins
amtk
amtterm
annobin
ansible-freeipa
archivemount
argparse-manpage
arptables
arpwatch
asio
aspell
aspell-en
at
at-spi2-atk
at-spi2-core
atf
atk
atop
attr
audiofile
augeas
authbind
authd
authselect
autoconf213
avahi
babeltrace
babeltrace2
babl
baekmuk-ttf-fonts
bats
bcache-tools
biosdevname
blosc
bluez
bmake
bogofilter
bolt
boom-boot
booth
botan2
breezy
brotli
buildah
busybox
bwidget
byacc
ca-certificates
cachefilesd
cairomm
calamares
capstone
catatonit
catch
catch1
cdrdao
celt051
cereal
certmonger
cfitsio
cgdcbxd
chan
CharLS
checkpolicy
checksec
chrony
cim-schema
cjkuni-uming-fonts
cjose
ck
cldr-emoji-annotation
clucene
clutter
clutter-gst3
clutter-gtk
cmocka
cogl
collectd
colm
color-filesystem
colord
colorize
compat-lua
compiler-rt
conda
conmon
conntrack-tools
console-setup
container-exception-logger
containernetworking-plugins
convmv
corosync
corosync-qdevice
cpp-hocon
cppcheck
cpprest
cpptest
cpuid
criu
crypto-policies
cryptsetup
cscope
ctags
CUnit
cups
custodia
Cython
dbus-c++
dbus-python
dbxtool
dconf
dcraw
debootstrap
deltarpm
desktop-file-utils
device-mapper-persistent-data
dietlibc
diffstat
ding-libs
discount
distribution-gpg-keys
dleyna-connector-dbus
dleyna-core
dmraid
dnf
dnf-plugins-core
docbook-dtds
docbook-simple
docbook-slides
docbook-style-dsssl
docbook-utils
docbook2X
docbook5-schemas
docbook5-style-xsl
dogtail
dos2unix
dotconf
dovecot
dpdk
dpkg
driverctl
dropwatch
drpm
duktape
dumpet
dvd+rw-tools
dwarves
dwz
dyninst
ebtables
edac-utils
edk2
efax
efi-rpm-macros
egl-wayland
eglexternalplatform
elinks
enca
enchant
enchant2
enscript
environment-modules
evemu
execstack
exempi
exiv2
extra-cmake-modules
fabtests
facter
fakechroot
fakeroot
fapolicyd
fdk-aac-free
fdupes
fence-virt
fetchmail
fftw
filebench
fio
fipscheck
firewalld
flac
flatbuffers
flite
fltk
fmt
fontawesome-fonts
fontawesome4-fonts
fontpackages
fonts-rpm-macros
foomatic-db
freeglut
freeipmi
freeradius
freetds
freexl
fribidi
fros
frr
fsverity-utils
fuse-overlayfs
fuse-sshfs
fuse-zip
fuse3
future
fxload
gavl
gbenchmark
gconf-editor
GConf2
gcovr
gcr
gdal
gdisk
gdk-pixbuf2
generic-logos
genwqe-tools
geoclue2
GeoIP
GeoIP-GeoLite-data
geolite2
geos
gfs2-utils
ghc-srpm-macros
giflib
gl-manpages
glew
glm
glog
glslang
glusterfs
gnome-desktop-testing
gnome-doc-utils
gnome-icon-theme
gnome-keyring
gnu-efi
go-rpm-macros
gom
google-api-python-client
google-crosextra-caladea-fonts
google-crosextra-carlito-fonts
google-guice
google-noto-cjk-fonts
google-noto-emoji-fonts
google-roboto-slab-fonts
gphoto2
gpm
gpsbabel
graphene
graphite2
graphviz
grubby
gsettings-desktop-schemas
gsl
gsm
gspell
gssdp
gssntlmssp
gstreamer1
gstreamer1-plugins-base
gtk-vnc
gtk2
gtk3
gtkspell
gupnp
gupnp-av
gupnp-dlna
gupnp-igd
hardening-check
hdf
hdf5
heimdal
help2man
hexedit
hicolor-icon-theme
hiera
highlight
hivex
hostname
hping3
hsakmt
htop
hunspell
hunspell-af
hunspell-ar
hunspell-as
hunspell-ast
hunspell-az
hunspell-be
hunspell-bg
hunspell-bn
hunspell-br
hunspell-ca
hunspell-cop
hunspell-csb
hunspell-cv
hunspell-cy
hunspell-da
hunspell-de
hunspell-dsb
hunspell-el
hunspell-en
hunspell-eo
hunspell-es
hunspell-et
hunspell-eu
hunspell-fa
hunspell-fj
hunspell-fo
hunspell-fr
hunspell-fur
hunspell-fy
hunspell-ga
hunspell-gd
hunspell-gl
hunspell-grc
hunspell-gu
hunspell-gv
hunspell-haw
hunspell-hi
hunspell-hil
hunspell-hr
hunspell-hsb
hunspell-ht
hunspell-hu
hunspell-hy
hunspell-ia
hunspell-id
hunspell-is
hunspell-it
hunspell-kk
hunspell-km
hunspell-kn
hunspell-ko
hunspell-ku
hunspell-ky
hunspell-la
hunspell-lb
hunspell-ln
hunspell-mai
hunspell-mg
hunspell-mi
hunspell-mk
hunspell-ml
hunspell-mn
hunspell-mos
hunspell-mr
hunspell-ms
hunspell-mt
hunspell-nds
hunspell-ne
hunspell-nl
hunspell-no
hunspell-nr
hunspell-nso
hunspell-ny
hunspell-om
hunspell-or
hunspell-pa
hunspell-pl
hunspell-pt
hunspell-quh
hunspell-ro
hunspell-ru
hunspell-rw
hunspell-se
hunspell-shs
hunspell-si
hunspell-sk
hunspell-sl
hunspell-smj
hunspell-so
hunspell-sq
hunspell-sr
hunspell-sv
hunspell-sw
hunspell-ta
hunspell-te
hunspell-tet
hunspell-th
hunspell-tk
hunspell-tl
hunspell-tn
hunspell-tpi
hunspell-ts
hunspell-uk
hunspell-uz
hunspell-ve
hunspell-vi
hunspell-wa
hunspell-xh
hunspell-yi
hwdata
hwloc
hyperscan
hyperv-daemons
hyphen
hyphen-as
hyphen-bg
hyphen-bn
hyphen-ca
hyphen-da
hyphen-de
hyphen-el
hyphen-es
hyphen-fa
hyphen-fo
hyphen-fr
hyphen-ga
hyphen-gl
hyphen-grc
hyphen-gu
hyphen-hi
hyphen-hsb
hyphen-hu
hyphen-ia
hyphen-id
hyphen-is
hyphen-it
hyphen-kn
hyphen-ku
hyphen-lt
hyphen-mi
hyphen-ml
hyphen-mn
hyphen-mr
hyphen-nl
hyphen-or
hyphen-pa
hyphen-pl
hyphen-pt
hyphen-ro
hyphen-ru
hyphen-sa
hyphen-sk
hyphen-sl
hyphen-sv
hyphen-ta
hyphen-te
hyphen-tk
hyphen-uk
ibus
ibus-chewing
ibus-hangul
ibus-kkc
ibus-libzhuyin
ibus-m17n
ibus-rawcode
ibus-sayura
ibus-table
ibus-table-chinese
icc-profiles-openicc
icon-naming-utils
icoutils
iftop
iio-sensor-proxy
ilmbase
im-chooser
imaptest
imsettings
indent
infinipath-psm
inih
iniparser
intel-cmt-cat
intel-ipsec-mb
ioping
IP2Location
ipa-pgothic-fonts
ipcalc
ipmitool
iprutils
iptraf-ng
iptstate
irssi
iscsi-initiator-utils
isns-utils
iso-codes
isomd5sum
iw
iwd
jabberpy
jakarta-servlet
jasper
javapackages-bootstrap
javapackages-tools
jbigkit
jdom2
jemalloc
jfsutils
jimtcl
jose
js-jquery
jsoncpp
Judy
jurand
kata-containers
kde-filesystem
kde-settings
kexec-tools
keybinder3
keycloak-httpd-client-install
kf
kf-kconfig
kf-kcoreaddons
kf-ki18n
kf-kwidgetsaddons
kpmcore
kronosnet
ksh
kyotocabinet
kyua
ladspa
lame
langtable
lapack
lasso
latencytop
lato-fonts
lcms2
lcov
ldns
leatherman
ledmon
lensfun
leveldb
lftp
libabw
libaec
libao
libappstream-glib
libarrow
libart_lgpl
libasyncns
libatasmart
libavc1394
libblockdev
libbpf
libbsd
libburn
libbytesize
libcacard
libcanberra
libcdio
libcdio-paranoia
libcdr
libcgroup
libchewing
libcli
libcmis
libcmpiutil
libcomps
libcroco
libdaemon
libdap
libdatrie
libdazzle
libdbi
libdbi-drivers
libdbusmenu
libdc1394
libdeflate
libdmx
libdnf
libdrm
libdvdnav
libdvdread
libdwarf
libeasyfc
libecap
libecb
libell
libEMF
libeot
libepoxy
libepubgen
libesmtp
libetonyek
libev
libevdev
libexif
libexttextcat
libfabric
libfontenc
libfreehand
libftdi
libgadu
libgdither
libgee
libgee06
libgeotiff
libgexiv2
libgit2
libgit2-glib
libglade2
libglvnd
libgovirt
libgphoto2
libgsf
libgta
libguestfs
libgusb
libgxim
libgxps
libhangul
libhugetlbfs
libibcommon
libical
libICE
libicns
libid3tag
libIDL
libidn2
libiec61883
libieee1284
libimobiledevice
libindicator
libinput
libiodbc
libipt
libiptcdata
libiscsi
libisoburn
libisofs
libjcat
libkcapi
libkeepalive
libkkc
libkkc-data
libkml
liblangtag
libldb
libldm
liblerc
liblockfile
liblognorm
liblouis
liblqr-1
liblzf
libmad
libmediaart
libmicrohttpd
libmikmod
libmodman
libmodplug
libmodulemd1
libmpcdec
libmspub
libmtp
libmusicbrainz5
libmwaw
libnbd
libnet
libnetfilter_log
libnfs
libnotify
libntlm
libnumbertext
libnvme
liboauth
libodfgen
libofa
libogg
liboggz
liboil
libomxil-bellagio
libopenraw
liboping
libosinfo
libotf
libotr
libpagemaker
libpaper
libpciaccess
libpeas
libpfm
libpinyin
libplist
libpmemobj-cpp
libpng12
libpng15
libproxy
libpsm2
libpwquality
libqb
libqxp
libraqm
LibRaw
libraw1394
libreport
libreswan
librevenge
librsvg2
librx
libsamplerate
libsass
libsecret
libsemanage
libsigc++20
libsigsegv
libslirp
libSM
libsmbios
libsmi
libsndfile
libsodium
libspiro
libsrtp
libssh
libstaroffice
libstemmer
libstoragemgmt
libtdb
libteam
libtevent
libthai
libtnc
libtomcrypt
libtommath
libtpms
libtracecmd
libtraceevent
libtracefs
libtranslit
libucil
libunicap
libuninameslist
liburing
libusbmuxd
libuser
libutempter
libvarlink
libverto
libvirt-dbus
libvirt-glib
libvirt-java
libvirt-python
libvisio
libvisual
libvoikko
libvorbis
libvpx
libwacom
libwnck3
libwpd
libwpe
libwpg
libwps
libwvstreams
libX11
libXau
libXaw
libxcb
libXcomposite
libxcrypt
libXcursor
libXdamage
libXdmcp
libXext
libxfce4util
libXfixes
libXfont2
libXft
libXi
libXinerama
libxkbcommon
libxkbfile
libxklavier
libxmlb
libXmu
libXpm
libXrandr
libXrender
libXres
libXScrnSaver
libxshmfence
libXt
libXtst
libXv
libXxf86vm
libyami
libyang
libyubikey
libzip
libzmf
lilv
linuxconsoletools
linuxptp
lksctp-tools
lldpd
lockdev
logwatch
lpsolve
lrzsz
lua
lua-expat
lua-filesystem
lua-json
lua-lpeg
lua-lunit
lua-rpm-macros
lua-term
luajit
lujavrite
luksmeta
lutok
lv2
lzip
lzop
m17n-db
m17n-lib
mac-robber
mailcap
mailx
malaga
malaga-suomi-voikko
mallard-rng
man-pages-cs
man-pages-es
man-pages-it
man-pages-ja
man-pages-ko
man-pages-pl
man-pages-ru
man-pages-zh-CN
mandoc
mariadb-connector-c
mariadb-connector-odbc
marisa
maven-compiler-plugin
maven-jar-plugin
maven-resolver
maven-resources-plugin
maven-surefire
maven-wagon
mcelog
mcpp
mcstrans
mdadm
mdds
mdevctl
meanwhile
mecab
mecab-ipadic
media-player-info
memcached
memkind
mesa
mesa-libGLU
metis
microcode_ctl
microdnf
minicom
minizip
mksh
mobile-broadband-provider-info
mock
mock-core-configs
mod_auth_gssapi
mod_auth_mellon
mod_auth_openidc
mod_authnz_pam
mod_fcgid
mod_http2
mod_intercept_form_submit
mod_lookup_identity
mod_md
mod_security
mod_security_crs
mod_wsgi
mokutil
mozjs
mpage
mrtg
mstflint
mt-st
mtdev
mtools
mtr
mtx
munge
mutt
mythes
mythes-bg
mythes-ca
mythes-cs
mythes-da
mythes-de
mythes-el
mythes-en
mythes-eo
mythes-es
mythes-fr
mythes-ga
mythes-hu
mythes-mi
mythes-ne
mythes-nl
mythes-pl
mythes-pt
mythes-ro
mythes-ru
mythes-sk
mythes-sl
mythes-sv
mythes-uk
nbd
nbdkit
neon
netavark
netcdf
netcf
netlabel_tools
netpbm
netsniff-ng
nfs4-acl-tools
nftables
nilfs-utils
nkf
nload
nlopt
nodejs-packaging
nss-mdns
nss-pam-ldapd
nss_nis
nss_wrapper
ntfs-3g
ntfs-3g-system-compression
numad
numatop
numpy
nvmetcli
nvml
oath-toolkit
ocaml
ocaml-alcotest
ocaml-astring
ocaml-augeas
ocaml-base
ocaml-bigarray-compat
ocaml-bisect-ppx
ocaml-calendar
ocaml-camlp5
ocaml-camomile
ocaml-cinaps
ocaml-cmdliner
ocaml-compiler-libs-janestreet
ocaml-cppo
ocaml-csexp
ocaml-csv
ocaml-ctypes
ocaml-curses
ocaml-dune
ocaml-extlib
ocaml-fileutils
ocaml-findlib
ocaml-fmt
ocaml-fpath
ocaml-gettext
ocaml-integers
ocaml-libvirt
ocaml-luv
ocaml-lwt
ocaml-markup
ocaml-migrate-parsetree
ocaml-mmap
ocaml-num
ocaml-ocamlbuild
ocaml-ocplib-endian
ocaml-ounit
ocaml-parsexp
ocaml-ppx-derivers
ocaml-ppxlib
ocaml-re
ocaml-react
ocaml-result
ocaml-seq
ocaml-sexplib
ocaml-sexplib0
ocaml-stdio
ocaml-topkg
ocaml-tyxml
ocaml-uuidm
ocaml-uutf
ocaml-xml-light
ocaml-zarith
ocl-icd
oddjob
ogdi
omping
opa
opal
open-vm-tools
openblas
opencc
opencl-filesystem
opencl-headers
opencryptoki
opencsd
opendnssec
OpenEXR
openjade
openjpeg2
openmpi
openobex
openoffice-lv
openrdate
opensc
openslp
opensm
opensp
openssl
openssl-ibmpkcs11
openssl-pkcs11
openwsman
optipng
opus
opusfile
orangefs
ORBit2
orc
os-prober
osinfo-db
osinfo-db-tools
overpass-fonts
p11-kit
p7zip
pacemaker
pacrunner
pakchois
pam_krb5
pam_wrapper
papi
paps
parallel
patchelf
patchutils
pbzip2
pcp
pcsc-lite
pcsc-lite-ccid
PEGTL
perl
perl-Algorithm-C3
perl-Algorithm-Diff
perl-Alien-Build
perl-Alien-pkgconf
perl-AnyEvent
perl-AnyEvent-AIO
perl-AnyEvent-BDB
perl-App-cpanminus
perl-App-FatPacker
perl-AppConfig
perl-Archive-Extract
perl-Archive-Zip
perl-Authen-SASL
perl-B-COW
perl-B-Debug
perl-B-Hooks-EndOfScope
perl-B-Hooks-OP-Check
perl-B-Keywords
perl-B-Lint
perl-bareword-filehandles
perl-BDB
perl-Bit-Vector
perl-boolean
perl-Browser-Open
perl-BSD-Resource
perl-Business-ISBN
perl-Business-ISBN-Data
perl-Bytes-Random-Secure
perl-Capture-Tiny
perl-Carp-Clan
perl-CBOR-XS
perl-Class-Accessor
perl-Class-C3
perl-Class-C3-XS
perl-Class-Data-Inheritable
perl-Class-Factory-Util
perl-Class-Inspector
perl-Class-ISA
perl-Class-Load
perl-Class-Load-XS
perl-Class-Method-Modifiers
perl-Class-Singleton
perl-Class-Tiny
perl-Class-XSAccessor
perl-Clone
perl-Color-ANSI-Util
perl-Color-RGB-Util
perl-ColorThemeBase-Static
perl-ColorThemeRole-ANSI
perl-ColorThemes-Standard
perl-ColorThemeUtil-ANSI
perl-Compress-Bzip2
perl-Compress-LZF
perl-Compress-Raw-Lzma
perl-Config-AutoConf
perl-Config-INI
perl-Config-INI-Reader-Multiline
perl-Config-IniFiles
perl-Config-Simple
perl-Config-Tiny
perl-Const-Fast
perl-Convert-ASN1
perl-Convert-Bencode
perl-Coro
perl-Coro-Multicore
perl-CPAN-Changes
perl-CPAN-DistnameInfo
perl-CPAN-Meta-Check
perl-Cpanel-JSON-XS
perl-Crypt-CBC
perl-Crypt-DES
perl-Crypt-IDEA
perl-Crypt-OpenSSL-Bignum
perl-Crypt-OpenSSL-Guess
perl-Crypt-OpenSSL-Random
perl-Crypt-OpenSSL-RSA
perl-Crypt-PasswdMD5
perl-Crypt-Random-Seed
perl-CSS-Tiny
perl-Data-Dump
perl-Data-Munge
perl-Data-OptList
perl-Data-Peek
perl-Data-Section
perl-Data-UUID
perl-Date-Calc
perl-Date-ISO8601
perl-Date-Manip
perl-DateTime
perl-DateTime-Format-Builder
perl-DateTime-Format-DateParse
perl-DateTime-Format-HTTP
perl-DateTime-Format-IBeat
perl-DateTime-Format-ISO8601
perl-DateTime-Format-Mail
perl-DateTime-Format-Strptime
perl-DateTime-Locale
perl-DateTime-TimeZone
perl-DateTime-TimeZone-SystemV
perl-DateTime-TimeZone-Tzfile
perl-DBD-MySQL
perl-Devel-CallChecker
perl-Devel-Caller
perl-Devel-CheckBin
perl-Devel-CheckLib
perl-Devel-Cycle
perl-Devel-EnforceEncapsulation
perl-Devel-GlobalDestruction
perl-Devel-GlobalDestruction-XS
perl-Devel-Hide
perl-Devel-Leak
perl-Devel-LexAlias
perl-Devel-Size
perl-Devel-StackTrace
perl-Devel-Symdump
perl-Digest-BubbleBabble
perl-Digest-CRC
perl-Digest-HMAC
perl-Digest-SHA1
perl-Dist-CheckConflicts
perl-DynaLoader-Functions
perl-Email-Address
perl-Email-Date-Format
perl-Encode-Detect
perl-Encode-EUCJPASCII
perl-Encode-IMAPUTF7
perl-Encode-Locale
perl-Env-ShellWords
perl-Error
perl-EV
perl-Eval-Closure
perl-Event
perl-Exception-Class
perl-Expect
perl-ExtUtils-Config
perl-ExtUtils-Depends
perl-ExtUtils-Helpers
perl-ExtUtils-InstallPaths
perl-ExtUtils-PkgConfig
perl-FCGI
perl-Fedora-VSP
perl-FFI-CheckLib
perl-File-BaseDir
perl-File-BOM
perl-File-chdir
perl-File-CheckTree
perl-File-Copy-Recursive
perl-File-DesktopEntry
perl-File-Find-Object
perl-File-Find-Object-Rule
perl-File-Find-Rule
perl-File-Find-Rule-Perl
perl-File-Inplace
perl-File-Listing
perl-File-MimeInfo
perl-File-pushd
perl-File-ReadBackwards
perl-File-Remove
perl-File-ShareDir
perl-File-ShareDir-Install
perl-File-Slurp
perl-File-Slurp-Tiny
perl-File-Slurper
perl-File-Type
perl-Font-TTF
perl-FreezeThaw
perl-GD
perl-GD-Barcode
perl-generators
perl-Getopt-ArgvFile
perl-gettext
perl-Graphics-ColorNamesLite-WWW
perl-GSSAPI
perl-Guard
perl-Hook-LexWrap
perl-HTML-Parser
perl-HTML-Tagset
perl-HTML-Tree
perl-HTTP-Cookies
perl-HTTP-Daemon
perl-HTTP-Date
perl-HTTP-Message
perl-HTTP-Negotiate
perl-Image-Base
perl-Image-Info
perl-Image-Xbm
perl-Image-Xpm
perl-Import-Into
perl-Importer
perl-inc-latest
perl-indirect
perl-Inline-Files
perl-IO-AIO
perl-IO-All
perl-IO-CaptureOutput
perl-IO-Compress-Lzma
perl-IO-HTML
perl-IO-Multiplex
perl-IO-SessionData
perl-IO-Socket-INET6
perl-IO-String
perl-IO-stringy
perl-IO-Tty
perl-IPC-Run
perl-IPC-Run3
perl-IPC-System-Simple
perl-JSON
perl-JSON-Color
perl-JSON-MaybeXS
perl-LDAP
perl-libnet
perl-libwww-perl
perl-libxml-perl
perl-Lingua-EN-Inflect
perl-List-MoreUtils-XS
perl-local-lib
perl-Locale-Codes
perl-Locale-Maketext-Gettext
perl-Locale-Msgfmt
perl-Locale-PO
perl-Log-Message
perl-Log-Message-Simple
perl-LWP-MediaTypes
perl-LWP-Protocol-https
perl-Mail-AuthenticationResults
perl-Mail-DKIM
perl-Mail-IMAPTalk
perl-Mail-SPF
perl-MailTools
perl-Math-Int64
perl-Math-Random-ISAAC
perl-MIME-Charset
perl-MIME-Lite
perl-MIME-Types
perl-Mixin-Linewise
perl-MLDBM
perl-Mock-Config
perl-Module-Build-Tiny
perl-Module-CPANfile
perl-Module-Implementation
perl-Module-Install-AuthorRequires
perl-Module-Install-AuthorTests
perl-Module-Install-AutoLicense
perl-Module-Install-GithubMeta
perl-Module-Install-ManifestSkip
perl-Module-Install-ReadmeFromPod
perl-Module-Install-ReadmeMarkdownFromPod
perl-Module-Install-Repository
perl-Module-Install-TestBase
perl-Module-Load-Util
perl-Module-Manifest
perl-Module-Manifest-Skip
perl-Module-Package
perl-Module-Package-Au
perl-Module-Pluggable
perl-Module-Runtime
perl-Module-Signature
perl-Mojolicious
perl-Moo
perl-Mozilla-CA
perl-Mozilla-LDAP
perl-MRO-Compat
perl-multidimensional
perl-namespace-autoclean
perl-namespace-clean
perl-Net-CIDR-Lite
perl-Net-Daemon
perl-Net-DNS
perl-Net-DNS-Resolver-Mock
perl-Net-DNS-Resolver-Programmable
perl-Net-HTTP
perl-Net-IMAP-Simple
perl-Net-IMAP-Simple-SSL
perl-Net-IP
perl-Net-LibIDN2
perl-Net-Patricia
perl-Net-SMTP-SSL
perl-Net-SNMP
perl-Net-Telnet
perl-Newt
perl-NNTPClient
perl-NTLM
perl-Number-Compare
perl-Object-Deadly
perl-Object-HashBase
perl-Package-Anon
perl-Package-Constants
perl-Package-DeprecationManager
perl-Package-Generator
perl-Package-Stash
perl-Package-Stash-XS
perl-PadWalker
perl-Paper-Specs
perl-PAR-Dist
perl-Parallel-Iterator
perl-Params-Classify
perl-Params-Util
perl-Params-Validate
perl-Params-ValidationCompiler
perl-Parse-PMFile
perl-Parse-RecDescent
perl-Parse-Yapp
perl-Path-Tiny
perl-Perl-Critic
perl-Perl-Critic-More
perl-Perl-Destruct-Level
perl-Perl-MinimumVersion
perl-Perl4-CoreLibs
perl-PerlIO-gzip
perl-PerlIO-utf8_strict
perl-PkgConfig-LibPkgConf
perl-Pod-Coverage
perl-Pod-Coverage-TrustPod
perl-Pod-Escapes
perl-Pod-Eventual
perl-Pod-LaTeX
perl-Pod-Markdown
perl-Pod-Parser
perl-Pod-Plainer
perl-Pod-POM
perl-Pod-Spell
perl-PPI
perl-PPI-HTML
perl-PPIx-QuoteLike
perl-PPIx-Regexp
perl-PPIx-Utilities
perl-prefork
perl-Probe-Perl
perl-Razor-Agent
perl-Readonly
perl-Readonly-XS
perl-Ref-Util
perl-Ref-Util-XS
perl-Regexp-Pattern-Perl
perl-Return-MultiLevel
perl-Role-Tiny
perl-Scope-Guard
perl-Scope-Upper
perl-SGMLSpm
perl-SNMP_Session
perl-Socket6
perl-Software-License
perl-Sort-Versions
perl-Specio
perl-Spiffy
perl-strictures
perl-String-CRC32
perl-String-Format
perl-String-ShellQuote
perl-String-Similarity
perl-Sub-Exporter
perl-Sub-Exporter-Progressive
perl-Sub-Identify
perl-Sub-Info
perl-Sub-Install
perl-Sub-Name
perl-Sub-Quote
perl-Sub-Uplevel
perl-SUPER
perl-Switch
perl-Syntax-Highlight-Engine-Kate
perl-Sys-CPU
perl-Sys-MemInfo
perl-Sys-Virt
perl-Taint-Runtime
perl-Task-Weaken
perl-Term-Size-Any
perl-Term-Size-Perl
perl-Term-Table
perl-Term-UI
perl-TermReadKey
perl-Test-Base
perl-Test-ClassAPI
perl-Test-CPAN-Meta
perl-Test-CPAN-Meta-JSON
perl-Test-Deep
perl-Test-Differences
perl-Test-DistManifest
perl-Test-Distribution
perl-Test-EOL
perl-Test-Exception
perl-Test-Exit
perl-Test-FailWarnings
perl-Test-Fatal
perl-Test-File
perl-Test-File-ShareDir
perl-Test-Harness
perl-Test-HasVersion
perl-Test-InDistDir
perl-Test-Inter
perl-Test-LeakTrace
perl-Test-LongString
perl-Test-Manifest
perl-Test-Memory-Cycle
perl-Test-MinimumVersion
perl-Test-MockObject
perl-Test-MockRandom
perl-Test-Needs
perl-Test-NoTabs
perl-Test-NoWarnings
perl-Test-Object
perl-Test-Output
perl-Test-Pod
perl-Test-Pod-Coverage
perl-Test-Portability-Files
perl-Test-Requires
perl-Test-RequiresInternet
perl-Test-Script
perl-Test-Simple
perl-Test-SubCalls
perl-Test-Synopsis
perl-Test-Taint
perl-Test-TrailingSpace
perl-Test-utf8
perl-Test-Vars
perl-Test-Warn
perl-Test-Without-Module
perl-Test2-Plugin-NoWarnings
perl-Test2-Suite
perl-Test2-Tools-Explain
perl-Text-CharWidth
perl-Text-CSV_XS
perl-Text-Diff
perl-Text-Glob
perl-Text-Iconv
perl-Text-Soundex
perl-Text-Unidecode
perl-Text-WrapI18N
perl-Tie-IxHash
perl-TimeDate
perl-Tree-DAG_Node
perl-Unicode-EastAsianWidth
perl-Unicode-LineBreak
perl-Unicode-Map8
perl-Unicode-String
perl-Unicode-UTF8
perl-UNIVERSAL-can
perl-UNIVERSAL-isa
perl-Unix-Syslog
perl-URI
perl-Variable-Magic
perl-Version-Requirements
perl-WWW-RobotRules
perl-XML-Catalog
perl-XML-DOM
perl-XML-Dumper
perl-XML-Filter-BufferText
perl-XML-Generator
perl-XML-Grove
perl-XML-Handler-YAWriter
perl-XML-LibXML
perl-XML-LibXSLT
perl-XML-NamespaceSupport
perl-XML-Parser-Lite
perl-XML-RegExp
perl-XML-SAX
perl-XML-SAX-Base
perl-XML-SAX-Writer
perl-XML-Simple
perl-XML-TokeParser
perl-XML-TreeBuilder
perl-XML-Twig
perl-XML-Writer
perl-XML-XPath
perl-XML-XPathEngine
perl-XString
perl-YAML-LibYAML
perl-YAML-PP
perl-YAML-Syck
perltidy
pesign
phodav
php
php-pear
php-pecl-zip
physfs
picosat
pinfo
pipewire
pixman
pkcs11-helper
pkgconf
plexus-cipher
plexus-containers
plexus-sec-dispatcher
plotutils
pmdk-convert
pmix
pngcrush
pngnq
po4a
podman
poetry
policycoreutils
polkit-pkla-compat
polkit-qt-1
portreserve
postfix
potrace
powertop
ppp
pps-tools
pptp
priv_wrapper
procmail
prometheus
prometheus-node-exporter
ps_mem
psacct
psutils
ptlib
publicsuffix-list
pugixml
pulseaudio
puppet
pwgen
pyatspi
pybind11
pycairo
pyelftools
pyflakes
pygobject3
PyGreSQL
pykickstart
pylint
pyparted
pyproject-rpm-macros
pyserial
python-absl-py
python-aiodns
python-aiohttp
python-alsa
python-argcomplete
python-astroid
python-astunparse
python-async-generator
python-augeas
python-azure-sdk
python-beautifulsoup4
python-betamax
python-blinker
python-blivet
python-cached_property
python-charset-normalizer
python-cheetah
python-click
python-cmd2
python-colorama
python-CommonMark
python-conda-package-handling
python-configshell
python-cpuinfo
python-cups
python-curio
python-cytoolz
python-d2to1
python-dbus-client-gen
python-dbus-python-client-gen
python-dbus-signature-pyparsing
python-dbusmock
python-ddt
python-debtcollector
python-decorator
python-distlib
python-dmidecode
python-dns
python-dtopt
python-dulwich
python-editables
python-enchant
python-entrypoints
python-ethtool
python-evdev
python-extras
python-faker
python-fasteners
python-fastjsonschema
python-fields
python-filelock
python-fixtures
python-flake8
python-flask
python-flit
python-flit-core
python-fluidity-sm
python-frozendict
python-funcsigs
python-gast
python-genshi
python-google-auth
python-google-auth-oauthlib
python-greenlet
python-gssapi
python-h5py
python-hatch-fancy-pypi-readme
python-hatch-vcs
python-hatchling
python-hs-dbus-signature
python-html5lib
python-httplib2
python-humanize
python-hwdata
python-importlib-metadata
python-inotify
python-into-dbus-python
python-IPy
python-iso8601
python-isodate
python-isort
python-itsdangerous
python-justbases
python-justbytes
python-jwcrypto
python-jwt
python-kdcproxy
python-kerberos
python-kmod
python-kubernetes
python-lark
python-lazy-object-proxy
python-ldap
python-linux-procfs
python-lit
python-looseversion
python-markdown
python-markdown-it-py
python-mccabe
python-mdurl
python-memcached
python-mimeparse
python-mock
python-monotonic
python-more-itertools
python-mpmath
python-msal
python-msrestazure
python-mutagen
python-networkx
python-nose2
python-ntlm-auth
python-oauth2client
python-openpyxl
python-openstackdocstheme
python-oslo-i18n
python-oslo-sphinx
python-paramiko
python-pathspec
python-pefile
python-pexpect
python-pkgconfig
python-platformdirs
python-pluggy
python-podman-api
python-poetry-core
python-process-tests
python-productmd
python-ptyprocess
python-pycares
python-pycosat
python-pydbus
python-pymongo
python-PyMySQL
python-pyperclip
python-pyroute2
python-pyrsistent
python-pysocks
python-pytest-benchmark
python-pytest-cov
python-pytest-expect
python-pytest-flake8
python-pytest-flakes
python-pytest-forked
python-pytest-mock
python-pytest-relaxed
python-pytest-runner
python-pytest-subtests
python-pytest-timeout
python-pytest-xdist
python-pytoml
python-pyudev
python-pywbem
python-qrcode
python-rdflib
python-recommonmark
python-requests-file
python-requests-ftp
python-requests-kerberos
python-requests-mock
python-requests-oauthlib
python-requests-toolbelt
python-requests_ntlm
python-responses
python-retrying
python-rfc3986
python-rich
python-rpm-generators
python-rpmfluff
python-rtslib
python-ruamel-yaml
python-ruamel-yaml-clib
python-s3transfer
python-schedutils
python-semantic_version
python-should_dsl
python-simpleline
python-slip
python-sniffio
python-sortedcontainers
python-soupsieve
python-sphinx
python-sphinx-epytext
python-sphinx-theme-py3doc-enhanced
python-sphinx_rtd_theme
python-sphinxcontrib-apidoc
python-sphinxcontrib-applehelp
python-sphinxcontrib-devhelp
python-sphinxcontrib-htmlhelp
python-sphinxcontrib-httpdomain
python-sphinxcontrib-jsmath
python-sphinxcontrib-qthelp
python-sphinxcontrib-serializinghtml
python-sqlalchemy
python-suds
python-systemd
python-tempita
python-templated-dictionary
python-termcolor
python-testpath
python-testresources
python-testscenarios
python-testtools
python-tidy
python-toml
python-tomli
python-toolz
python-tornado
python-tox
python-tox-current-env
python-tqdm
python-trio
python-trove-classifiers
python-typing-extensions
python-uamqp
python-unittest2
python-uritemplate
python-urwid
python-varlink
python-versioneer
python-virt-firmware
python-voluptuous
python-waitress
python-webencodings
python-webtest
python-wheel
python-whoosh
python-winrm
python-wrapt
python-xmltodict
python-yubico
python-zipp
python-zmq
python-zstd
python3-mallard-ducktype
python3-pytest-asyncio
python3-typed_ast
pyusb
pywbem
pyxattr
qemu
qhull
qpdf
qperf
qr-code-generator
qt-rpm-macros
qt5-qtconnectivity
qt5-qtsensors
qt5-qtserialport
qtbase
qtdeclarative
qtsvg
qttools
quagga
quota
radvd
ragel
raptor2
rarian
rasdaemon
rasqal
rcs
rdist
rdma-core
re2
re2c
realmd
rear
recode
redland
resource-agents
rest
rhash
rlwrap
rp-pppoe
rpm-mpi-hooks
rpmdevtools
rpmlint
rtkit
rtl-sdr
ruby-augeas
rubygem-bson
rubygem-coderay
rubygem-diff-lcs
rubygem-flexmock
rubygem-hpricot
rubygem-introspection
rubygem-liquid
rubygem-maruku
rubygem-metaclass
rubygem-mongo
rubygem-mustache
rubygem-mysql2
rubygem-pkg-config
rubygem-rake
rubygem-rake-compiler
rubygem-ronn
rubygem-rouge
rubygem-rspec
rubygem-rspec-expectations
rubygem-rspec-mocks
rubygem-rspec-support
rubygem-thread_order
rusers
rust-cbindgen
samba
sanlock
sassist
satyr
sbc
sblim-cim-client2
sblim-cmpi-base
sblim-cmpi-devel
sblim-cmpi-fsvol
sblim-cmpi-network
sblim-cmpi-nfsv3
sblim-cmpi-nfsv4
sblim-cmpi-params
sblim-cmpi-sysfs
sblim-cmpi-syslog
sblim-indication_helper
sblim-sfcb
sblim-sfcc
sblim-sfcCommon
sblim-testsuite
sblim-wbemcli
scl-utils
scotch
screen
scrub
SDL
SDL2
SDL_sound
sdparm
seabios
secilc
selinux-policy
sendmail
serd
setools
setserial
setuptool
sgabios
sgml-common
sgpio
shared-mime-info
sharutils
sip
sisu
skkdic
sleuthkit
slirp4netns
smartmontools
smc-tools
socket_wrapper
softhsm
sombok
sord
sos
sound-theme-freedesktop
soundtouch
sox
soxr
sparsehash
spausedd
speex
speexdsp
spice-protocol
spice-vdagent
spirv-headers
spirv-tools
splix
squashfs-tools
squid
sratom
sscg
star
startup-notification
stunnel
subscription-manager
suitesparse
SuperLU
supermin
switcheroo-control
swtpm
symlinks
sympy
sysfsutils
systemd
systemd-bootchart
t1lib
t1utils
taglib
tang
targetcli
tbb
tcl-pgtcl
tclx
teckit
telnet
thrift
tidy
time
tini
tinycdb
tix
tk
tlog
tmpwatch
tn5250
tofrodos
tokyocabinet
trace-cmd
tss2
ttembed
ttmkfdir
tuna
twolame
uchardet
uclibc-ng
ucpp
ucs-miscfixed-fonts
ucx
udftools
udica
udisks2
uglify-js
uid_wrapper
unicode-emoji
unicode-ucd
unique3
units
upower
uriparser
urlview
usb_modeswitch
usb_modeswitch-data
usbguard
usbip
usbmuxd
usbredir
usermode
ustr
uthash
uuid
uw-imap
v4l-utils
vhostmd
vino
virglrenderer
virt-p2v
virt-top
virt-what
virt-who
vitess
vmem
volume_key
vorbis-tools
vte291
vulkan-headers
vulkan-loader
watchdog
wavpack
wayland
wayland-protocols
web-assets
webrtc-audio-processing
websocketpp
wget
whois
wireguard-tools
wireless-regdb
wireshark
woff2
wordnet
words
wpebackend-fdo
wsmancli
wvdial
x3270
xapian-core
Xaw3d
xcb-proto
xcb-util
xcb-util-image
xcb-util-keysyms
xcb-util-renderutil
xcb-util-wm
xdelta
xdg-dbus-proxy
xdg-utils
xdp-tools
xerces-c
xfconf
xfsdump
xhtml1-dtds
xkeyboard-config
xmlstarlet
xmltoman
xmvn
xorg-x11-apps
xorg-x11-drv-libinput
xorg-x11-font-utils
xorg-x11-fonts
xorg-x11-proto-devel
xorg-x11-server
xorg-x11-server-utils
xorg-x11-util-macros
xorg-x11-utils
xorg-x11-xauth
xorg-x11-xbitmaps
xorg-x11-xinit
xorg-x11-xkb-utils
xorg-x11-xtrans-devel
xrestop
xterm
xxhash
yajl
yaml-cpp
yasm
yelp-tools
yelp-xsl
ykclient
yp-tools
ypbind
ypserv
z3
zenity
zerofree
zfs-fuse
zipper
zopfli
zziplib | | Fedora (Copyright Remi Collet) | [CC-BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/legalcode) | libmemcached-awesome
librabbitmq | | Fedora (ISC) | [ISC License](https://github.com/sarugaku/resolvelib/blob/main/LICENSE) | python-resolvelib | | Magnus Edenhill Open Source | [Magnus Edenhill Open Source BSD License](https://github.com/jemalloc/jemalloc/blob/dev/COPYING) | librdkafka | diff --git a/LICENSES-AND-NOTICES/SPECS/data/licenses.json b/LICENSES-AND-NOTICES/SPECS/data/licenses.json index 010cac6b250..7e6e8f17947 100644 --- a/LICENSES-AND-NOTICES/SPECS/data/licenses.json +++ b/LICENSES-AND-NOTICES/SPECS/data/licenses.json @@ -1838,6 +1838,7 @@ "python-uritemplate", "python-urwid", "python-varlink", + "python-versioneer", "python-virt-firmware", "python-voluptuous", "python-waitress", diff --git a/SPECS/python-constantly/python-constantly.signatures.json b/SPECS/python-constantly/python-constantly.signatures.json index 410df8f848e..bda9c0a07c2 100644 --- a/SPECS/python-constantly/python-constantly.signatures.json +++ b/SPECS/python-constantly/python-constantly.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "constantly-15.1.0.tar.gz": "586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35" + "constantly-23.10.4.tar.gz": "aa92b70a33e2ac0bb33cd745eb61776594dc48764b06c35e0efd050b7f1c7cbd" } -} +} \ No newline at end of file diff --git a/SPECS/python-constantly/python-constantly.spec b/SPECS/python-constantly/python-constantly.spec index a5f96d63262..ac273ff3373 100644 --- a/SPECS/python-constantly/python-constantly.spec +++ b/SPECS/python-constantly/python-constantly.spec @@ -1,14 +1,14 @@ Summary: Symbolic constants in Python Name: python-constantly -Version: 15.1.0 -Release: 7%{?dist} +Version: 23.10.4 +Release: 1%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux Group: Development/Languages/Python URL: https://github.com/twisted/constantly # Source to be fixed as part of https://microsoft.visualstudio.com/OS/_workitems/edit/25936171. -Source0: https://files.pythonhosted.org/packages/95/f1/207a0a478c4bb34b1b49d5915e2db574cadc415c9ac3a7ef17e29b2e8951/constantly-%{version}.tar.gz +Source0: https://files.pythonhosted.org/packages/4d/6f/cb2a94494ff74aa9528a36c5b1422756330a75a8367bf20bd63171fc324d/constantly-%{version}.tar.gz BuildArch: noarch %description @@ -19,6 +19,12 @@ Summary: Symbolic constants in Python BuildRequires: python3-devel BuildRequires: python3-setuptools BuildRequires: python3-xml +BuildRequires: python3-pip +BuildRequires: python3-versioneer +%if 0%{?with_check} +Buildrequires: python3-twisted +Buildrequires: python3-typing-extensions +%endif Requires: python3 %description -n python3-constantly @@ -26,9 +32,6 @@ A library that provides symbolic constant support. It includes collections and c %prep %autosetup -n constantly-%{version} -# SafeConfigParser class is renamed to ConfigParser in python 3.12. See: python/cpython#89336 -sed -i 's/SafeConfigParser/ConfigParser/g' ./versioneer.py -sed -i 's/readfp/read_file/g' ./versioneer.py %build %py3_build @@ -45,6 +48,9 @@ sed -i 's/readfp/read_file/g' ./versioneer.py %{python3_sitelib}/* %changelog +* Tue May 14 2024 Betty Lakes - 23.10.4-1 +- Upgrade to 23.10.4 + * Fri Feb 16 2024 Andrew Phelps - 15.1.0-7 - Fix build for python 3.12 @@ -56,7 +62,7 @@ sed -i 's/readfp/read_file/g' ./versioneer.py * Sat May 09 2020 Nick Samson - 15.1.0-5 - Added %%license line automatically -* Wed Apr 09 2020 Pawel Winogrodzki - 15.1.0-4 +* Thu Apr 09 2020 Pawel Winogrodzki - 15.1.0-4 - Fixed "Source0" tag. - License verified. - Removed "%%define sha1". diff --git a/SPECS/python-versioneer/python-versioneer.signatures.json b/SPECS/python-versioneer/python-versioneer.signatures.json new file mode 100644 index 00000000000..48154142a74 --- /dev/null +++ b/SPECS/python-versioneer/python-versioneer.signatures.json @@ -0,0 +1,5 @@ +{ + "Signatures": { + "versioneer-0.29.tar.gz": "5ab283b9857211d61b53318b7c792cf68e798e765ee17c27ade9f6c924235731" + } +} \ No newline at end of file diff --git a/SPECS/python-versioneer/python-versioneer.spec b/SPECS/python-versioneer/python-versioneer.spec new file mode 100644 index 00000000000..a80acba077a --- /dev/null +++ b/SPECS/python-versioneer/python-versioneer.spec @@ -0,0 +1,129 @@ +%global _description %{expand: +Versioneer is a tool to automatically update version strings (in setup.py and +the conventional 'from PROJECT import _version' pattern) by asking your +version-control system about the current tree.} +%global pypi_name versioneer + +Name: python-versioneer +Version: 0.29 +Release: 1%{?dist} +Summary: Easy VCS-based management of project version strings +Vendor: Microsoft Corporation +Distribution: Azure Linux +License: Unlicense +URL: https://github.com/warner/python-versioneer +Source0: https://files.pythonhosted.org/packages/32/d7/854e45d2b03e1a8ee2aa6429dd396d002ce71e5d88b77551b2fb249cb382/versioneer-0.29.tar.gz +BuildArch: noarch + +BuildRequires: python3-devel +BuildRequires: python3-pip +BuildRequires: python3-wheel + +%description %_description + +%package -n python3-versioneer +Summary: %{summary} + +%description -n python3-versioneer %_description + +%pyproject_extras_subpkg -n python3-versioneer toml + +%prep +%autosetup -n versioneer-%{version} + +%generate_buildrequires +%pyproject_buildrequires -x toml + +%build +%pyproject_wheel + +%install +%pyproject_install +# Remove the unwanted shebang from the amalgamated versioneer.py file: +sed -r -i '1{/^#!/d}' %{buildroot}%{python3_sitelib}/versioneer.py +%pyproject_save_files versioneer + +%check +# Based on tox.ini; but we do not use tox, because tox.ini has too many linting +# tests and other unwanted dependencies. +%{python3} setup.py make_versioneer +%{python3} -m unittest discover test +# Some of these do not work; it is not clear that this indicates a real +# problem. We would need at least “BuildRequires: git-core” if they did work. +#{python3} test/git/test_git.py -v +# These generally require python3dist(virtualenv) and network access. +#{python3} test/git/test_invocations.py -v + +%files -n python3-versioneer -f %{pyproject_files} +%doc README.md +%doc details.md +%{_bindir}/versioneer + +%changelog +* Tue May 14 2024 Betty Lakes - 0.29-1 +- Cleaning-up spec. License verified. +- Initial Azure Linux import from Fedora 40 (license: MIT). + +* Fri Jan 26 2024 Fedora Release Engineering - 0.28-5 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Mon Jan 22 2024 Fedora Release Engineering - 0.28-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Fri Jul 21 2023 Fedora Release Engineering - 0.28-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild + +* Tue Jun 13 2023 Python Maint - 0.28-2 +- Rebuilt for Python 3.12 + +* Fri Mar 03 2023 Benjamin A. Beasley - 0.28-1 +- Update to 0.28 +- Port to pyproject-rpm-macros +- Add metapackage for “toml” extra +- License has changed from CC0-1.0 (spec file had Public Domain) to Unlicense +- Remove obsolete posttrans scriptlet +- Add details.md to documentation + +* Fri Jan 20 2023 Fedora Release Engineering - 0.21-5 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild + +* Fri Jul 22 2022 Fedora Release Engineering - 0.21-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_37_Mass_Rebuild + +* Mon Jun 13 2022 Python Maint - 0.21-3 +- Rebuilt for Python 3.11 + +* Fri Jan 21 2022 Fedora Release Engineering - 0.21-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_36_Mass_Rebuild + +* Thu Dec 02 2021 Orion Poplawski - 0.21-1 +- Update to 0.21 +- Own egg-info directory and cleanup previous egg-info directory + +* Tue Jul 27 2021 Fedora Release Engineering - 0.18-9 +- Second attempt - Rebuilt for + https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild + +* Fri Jun 04 2021 Python Maint - 0.18-8 +- Rebuilt for Python 3.10 + +* Wed Jan 27 2021 Fedora Release Engineering - 0.18-7 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_34_Mass_Rebuild + +* Wed Jul 29 2020 Fedora Release Engineering - 0.18-6 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_33_Mass_Rebuild + +* Tue May 26 2020 Miro Hrončok - 0.18-5 +- Rebuilt for Python 3.9 + +* Thu Jan 30 2020 Fedora Release Engineering - 0.18-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_32_Mass_Rebuild + +* Tue Dec 31 2019 Mukundan Ragavan - 0.18-3 +- rebuild + +* Mon Dec 30 2019 Mukundan Ragavan - 0.18-2 +- Address changes from package review + +* Sun Dec 22 2019 Mukundan Ragavan - 0.18-1 +- Initial package. diff --git a/cgmanifest.json b/cgmanifest.json index d5d135e3fa5..8b5ce87e638 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -21953,8 +21953,8 @@ "type": "other", "other": { "name": "python-constantly", - "version": "15.1.0", - "downloadUrl": "https://files.pythonhosted.org/packages/95/f1/207a0a478c4bb34b1b49d5915e2db574cadc415c9ac3a7ef17e29b2e8951/constantly-15.1.0.tar.gz" + "version": "23.10.4", + "downloadUrl": "https://files.pythonhosted.org/packages/4d/6f/cb2a94494ff74aa9528a36c5b1422756330a75a8367bf20bd63171fc324d/constantly-23.10.4.tar.gz" } } }, @@ -24768,6 +24768,16 @@ } } }, + { + "component": { + "type": "other", + "other": { + "name": "python-versioneer", + "version": "0.29", + "downloadUrl": "https://files.pythonhosted.org/packages/32/d7/854e45d2b03e1a8ee2aa6429dd396d002ce71e5d88b77551b2fb249cb382/versioneer-0.29.tar.gz" + } + } + }, { "component": { "type": "other", From f6d2453e77cae0b7c5475b27b39f624eb913b73e Mon Sep 17 00:00:00 2001 From: Betty <38226164+BettyRain@users.noreply.github.com> Date: Wed, 15 May 2024 14:53:20 -0700 Subject: [PATCH 16/51] python-wheel: upgrade to 0.43.0 (#9083) Co-authored-by: Betty Lakes --- .../python-wheel/python-wheel.signatures.json | 4 +- SPECS/python-wheel/python-wheel.spec | 62 ++++++++++++------- cgmanifest.json | 4 +- .../manifests/package/toolchain_aarch64.txt | 4 +- .../manifests/package/toolchain_x86_64.txt | 4 +- .../build_official_toolchain_rpms.sh | 4 +- 6 files changed, 49 insertions(+), 33 deletions(-) diff --git a/SPECS/python-wheel/python-wheel.signatures.json b/SPECS/python-wheel/python-wheel.signatures.json index ef0b72b4cf1..204e84b55c9 100644 --- a/SPECS/python-wheel/python-wheel.signatures.json +++ b/SPECS/python-wheel/python-wheel.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "wheel-0.33.6.tar.gz": "ce9b3af862796aeb7af75c72e54fe974e4bb9a95502f49b1da25fdbf4c53e742" + "wheel-0.43.0.tar.gz": "23060d7cc8afafc2930554624b4bae7d58031830672048622c926675ab91e3b0" } -} +} \ No newline at end of file diff --git a/SPECS/python-wheel/python-wheel.spec b/SPECS/python-wheel/python-wheel.spec index 8af856eac3f..c6880ac23ce 100644 --- a/SPECS/python-wheel/python-wheel.spec +++ b/SPECS/python-wheel/python-wheel.spec @@ -1,16 +1,17 @@ # The function of bootstrap is that it disables the wheel subpackage %bcond_with bootstrap +%bcond main_python 1 Summary: Built-package format for Python Name: python-%{pypi_name} -Version: 0.33.6 -Release: 7%{?dist} +Version: 0.43.0 +Release: 1%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux URL: https://github.com/pypa/wheel Source0: %{url}/archive/%{version}/%{pypi_name}-%{version}.tar.gz %global pypi_name wheel -%global python_wheelname %{pypi_name}-%{version}-py2.py3-none-any.whl +%global python_wheel_name %{pypi_name}-%{version}-py3-none-any.whl %global python_wheeldir %{_datadir}/python-wheels %global _description \ A built-package format for Python.\ @@ -33,13 +34,14 @@ Summary: %{summary} %{?python_provide:%python_provide python3-%{pypi_name}} BuildRequires: python3-devel BuildRequires: python3-setuptools +BuildRequires: python3-pip %if 0%{?with_check} BuildRequires: python3-atomicwrites BuildRequires: python3-attrs -BuildRequires: python3-pip BuildRequires: python3-pytest BuildRequires: python3-six %endif +BuildRequires: python3-flit-core %description -n python3-%{pypi_name} %{_description} @@ -53,55 +55,69 @@ Summary: The Python wheel module packaged as a wheel A Python wheel of wheel to use with virtualenv. %endif - %prep %autosetup -n %{pypi_name}-%{version} -p1 -# Empty files make rpmlint sad -test -s wheel/cli/install.py || echo "# empty" > wheel/cli/install.py +%generate_buildrequires +%pyproject_buildrequires %build -%py3_build - -%if %{without bootstrap} -%{py3_build_wheel} -%endif +%global _pyproject_wheeldir dist +%python3 -m flit_core.wheel +%pyproject_wheel %install -%py3_install +# pip is not available when bootstrapping, so we need to unpack the wheel and +# create the entrypoints manually. +%if %{with bootstrap} +mkdir -p %{buildroot}%{python3_sitelib} +unzip %{_pyproject_wheeldir}/%{python_wheel_name} \ + -d %{buildroot}%{python3_sitelib} -x wheel-%{version}.dist-info/RECORD +install -Dpm 0755 %{SOURCE1} %{buildroot}%{_bindir}/wheel +%py3_shebang_fix %{buildroot}%{_bindir}/wheel +%else +%pyproject_install +%endif + mv %{buildroot}%{_bindir}/%{pypi_name}{,-%{python3_version}} +%if %{with main_python} ln -s %{pypi_name}-%{python3_version} %{buildroot}%{_bindir}/%{pypi_name}-3 ln -s %{pypi_name}-3 %{buildroot}%{_bindir}/%{pypi_name} - -%if %{without bootstrap} -mkdir -p %{buildroot}%{python_wheeldir} -install -p dist/%{python_wheelname} -t %{buildroot}%{python_wheeldir} %endif +mkdir -p %{buildroot}%{python_wheel_dir} +install -p %{_pyproject_wheeldir}/%{python_wheel_name} -t %{buildroot}%{python_wheel_dir} %check -rm setup.cfg -%{python3} -m pip install pluggy more-itertools -PYTHONPATH=%{buildroot}%{python3_sitelib} py.test3 -v --ignore build +pip3 install iniconfig +# Smoke test +%{py3_test_envvars} wheel-%{python3_version} version +%py3_check_import wheel +%pytest -v --ignore build %files -n python3-%{pypi_name} %license LICENSE.txt %doc README.rst -%{_bindir}/%{pypi_name} -%{_bindir}/%{pypi_name}-3 %{_bindir}/%{pypi_name}-%{python3_version} %{python3_sitelib}/%{pypi_name}* +%if %{with main_python} +%{_bindir}/%{pypi_name} +%{_bindir}/%{pypi_name}-3 +%endif %if %{without bootstrap} %files wheel %license LICENSE.txt # we own the dir for simplicity %dir %{python_wheeldir}/ -%{python_wheeldir}/%{python_wheelname} +%{python_wheeldir}/%{python_wheel_name} %endif %changelog +* Fri May 10 2024 Betty Lakes - 0.43.0-1 +- Updated to 0.43.0 + * Thu Mar 03 2022 Bala - 0.33.6-7 - BR multiple python3 modules for PTest - pip3 install additional modules which not available as RPM diff --git a/cgmanifest.json b/cgmanifest.json index 8b5ce87e638..d3daa0b0837 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -24883,8 +24883,8 @@ "type": "other", "other": { "name": "python-wheel", - "version": "0.33.6", - "downloadUrl": "https://github.com/pypa/wheel/archive/0.33.6/wheel-0.33.6.tar.gz" + "version": "0.43.0", + "downloadUrl": "https://github.com/pypa/wheel/archive/0.43.0/wheel-0.43.0.tar.gz" } } }, diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index 6cb72b90d67..a4932886e63 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -518,7 +518,7 @@ procps-ng-lang-4.0.4-1.azl3.aarch64.rpm pyproject-rpm-macros-1.12.0-2.azl3.noarch.rpm pyproject-srpm-macros-1.12.0-2.azl3.noarch.rpm python-markupsafe-debuginfo-2.1.3-1.azl3.aarch64.rpm -python-wheel-wheel-0.33.6-7.azl3.noarch.rpm +python-wheel-wheel-0.43.0-1.azl3.noarch.rpm python3-3.12.0-3.azl3.aarch64.rpm python3-audit-3.1.2-1.azl3.aarch64.rpm python3-cracklib-2.9.11-1.azl3.aarch64.rpm @@ -544,7 +544,7 @@ python3-rpm-generators-14-11.azl3.noarch.rpm python3-setuptools-69.0.3-2.azl3.noarch.rpm python3-test-3.12.0-3.azl3.aarch64.rpm python3-tools-3.12.0-3.azl3.aarch64.rpm -python3-wheel-0.33.6-7.azl3.noarch.rpm +python3-wheel-0.43.0-1.azl3.noarch.rpm readline-8.2-1.azl3.aarch64.rpm readline-debuginfo-8.2-1.azl3.aarch64.rpm readline-devel-8.2-1.azl3.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index 4be665e5395..4ebdea9e795 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -524,7 +524,7 @@ procps-ng-lang-4.0.4-1.azl3.x86_64.rpm pyproject-rpm-macros-1.12.0-2.azl3.noarch.rpm pyproject-srpm-macros-1.12.0-2.azl3.noarch.rpm python-markupsafe-debuginfo-2.1.3-1.azl3.x86_64.rpm -python-wheel-wheel-0.33.6-7.azl3.noarch.rpm +python-wheel-wheel-0.43.0-1.azl3.noarch.rpm python3-3.12.0-3.azl3.x86_64.rpm python3-audit-3.1.2-1.azl3.x86_64.rpm python3-cracklib-2.9.11-1.azl3.x86_64.rpm @@ -550,7 +550,7 @@ python3-rpm-generators-14-11.azl3.noarch.rpm python3-setuptools-69.0.3-2.azl3.noarch.rpm python3-test-3.12.0-3.azl3.x86_64.rpm python3-tools-3.12.0-3.azl3.x86_64.rpm -python3-wheel-0.33.6-7.azl3.noarch.rpm +python3-wheel-0.43.0-1.azl3.noarch.rpm readline-8.2-1.azl3.x86_64.rpm readline-debuginfo-8.2-1.azl3.x86_64.rpm readline-devel-8.2-1.azl3.x86_64.rpm diff --git a/toolkit/scripts/toolchain/build_official_toolchain_rpms.sh b/toolkit/scripts/toolchain/build_official_toolchain_rpms.sh index 18df2ada342..95593e0d660 100755 --- a/toolkit/scripts/toolchain/build_official_toolchain_rpms.sh +++ b/toolkit/scripts/toolchain/build_official_toolchain_rpms.sh @@ -650,9 +650,9 @@ chroot_and_install_rpms rpm rpm-build-libs chroot_and_install_rpms rpm rpm-devel chroot_and_install_rpms rpm rpm-build -build_rpm_in_chroot_no_install python-wheel -build_rpm_in_chroot_no_install python-flit-core build_rpm_in_chroot_no_install python-pip +build_rpm_in_chroot_no_install python-flit-core +build_rpm_in_chroot_no_install python-wheel # python-jinja2 needs python3-markupsafe # python3-setuptools, python3-libs are also needed but already installed From 58532ace7bbfe1cfbcad292448ce9dec7719b56b Mon Sep 17 00:00:00 2001 From: Christopher Co <35273088+christopherco@users.noreply.github.com> Date: Wed, 15 May 2024 15:47:18 -0700 Subject: [PATCH 17/51] Add macro for enabling frame pointers during build and enable frame pointers in glibc (#9092) Enabling frame pointers during compilation brings benefits to performance profiling and tracing. See https://www.brendangregg.com/blog/2024-03-17/the-return-of-the-frame-pointers.html for more details. This change in particular adds a macro to add the -fno-omit-frame-pointer flag to CFLAGS when defining %define _include_frame_pointers 1 in the spec file. This way the packager can control enabling whether to build with frame pointers enabled or not. By default, building with frame pointers is disabled. This change also enables frame pointers by default in our glibc build, which is one of the hot-spots where fast profiling with frame pointers would be beneficial. Signed-off-by: Chris Co --- SPECS-EXTENDED/buildah/buildah.spec | 7 ++++-- SPECS-EXTENDED/catatonit/catatonit.spec | 7 ++++-- SPECS-EXTENDED/dyninst/dyninst.spec | 7 ++++-- SPECS-EXTENDED/podman/podman.spec | 7 ++++-- .../azurelinux-rpm-macros.signatures.json | 2 +- .../azurelinux-rpm-macros.spec | 5 ++++- SPECS/azurelinux-rpm-macros/macros | 6 ++++- SPECS/busybox/busybox.spec | 7 ++++-- SPECS/flannel/flannel.spec | 7 ++++-- SPECS/glibc/glibc.spec | 8 ++++++- SPECS/kubernetes/kubernetes.spec | 7 ++++-- SPECS/kubevirt/kubevirt.spec | 7 ++++-- SPECS/libguestfs/libguestfs.spec | 7 ++++-- SPECS/mdadm/mdadm.spec | 7 ++++-- SPECS/qemu/qemu.spec | 7 ++++-- SPECS/rust/rust.spec | 7 ++++-- SPECS/supermin/supermin.spec | 7 ++++-- SPECS/tini/tini.spec | 7 ++++-- .../manifests/package/pkggen_core_aarch64.txt | 18 +++++++-------- .../manifests/package/pkggen_core_x86_64.txt | 18 +++++++-------- .../manifests/package/toolchain_aarch64.txt | 22 +++++++++---------- .../manifests/package/toolchain_x86_64.txt | 22 +++++++++---------- 22 files changed, 127 insertions(+), 72 deletions(-) diff --git a/SPECS-EXTENDED/buildah/buildah.spec b/SPECS-EXTENDED/buildah/buildah.spec index 6cf2f52cd9f..03b707c81d1 100644 --- a/SPECS-EXTENDED/buildah/buildah.spec +++ b/SPECS-EXTENDED/buildah/buildah.spec @@ -21,7 +21,7 @@ Summary: A command line tool used for creating OCI Images Name: buildah Version: 1.18.0 -Release: 24%{?dist} +Release: 25%{?dist} License: ASL 2.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -32,7 +32,7 @@ BuildRequires: btrfs-progs-devel BuildRequires: device-mapper-devel BuildRequires: git BuildRequires: glib2-devel -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: go-md2man BuildRequires: go-rpm-macros BuildRequires: golang @@ -123,6 +123,9 @@ cp imgtype %{buildroot}/%{_bindir}/%{name}-imgtype %{_datadir}/%{name}/test %changelog +* Mon May 13 2024 Chris Co - 1.18.0-25 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 1.18.0-24 - update to build dep latest glibc-static version diff --git a/SPECS-EXTENDED/catatonit/catatonit.spec b/SPECS-EXTENDED/catatonit/catatonit.spec index cdbf977bf9d..dfb6e0e0aad 100644 --- a/SPECS-EXTENDED/catatonit/catatonit.spec +++ b/SPECS-EXTENDED/catatonit/catatonit.spec @@ -3,7 +3,7 @@ Distribution: Azure Linux Name: catatonit Version: 0.1.7 -Release: 12%{?dist} +Release: 13%{?dist} Summary: A signal-forwarding process manager for containers License: GPLv3+ URL: https://github.com/openSUSE/catatonit @@ -13,7 +13,7 @@ BuildRequires: automake BuildRequires: file BuildRequires: gcc BuildRequires: git -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: libtool BuildRequires: make @@ -61,6 +61,9 @@ ln -s %{_libexecdir}/%{name}/%{name} %{buildroot}%{_libexecdir}/podman/%{name} %{_libexecdir}/podman/%{name} %changelog +* Mon May 13 2024 Chris Co - 0.1.7-13 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 0.1.7-12 - update to build dep latest glibc-static version diff --git a/SPECS-EXTENDED/dyninst/dyninst.spec b/SPECS-EXTENDED/dyninst/dyninst.spec index 340f4c5a15e..f58a971155f 100644 --- a/SPECS-EXTENDED/dyninst/dyninst.spec +++ b/SPECS-EXTENDED/dyninst/dyninst.spec @@ -1,7 +1,7 @@ Summary: An API for Run-time Code Generation License: LGPLv2+ Name: dyninst -Release: 14%{?dist} +Release: 15%{?dist} Vendor: Microsoft Corporation Distribution: Azure Linux URL: http://www.dyninst.org @@ -31,7 +31,7 @@ BuildRequires: tbb tbb-devel # Extra requires just for the testsuite BuildRequires: gcc-gfortran libstdc++-static libxml2-devel -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} # Testsuite files should not provide/require anything %{?filter_setup: @@ -194,6 +194,9 @@ echo "%{_libdir}/dyninst" > %{buildroot}/etc/ld.so.conf.d/%{name}-%{_arch}.conf %attr(644,root,root) %{_libdir}/dyninst/testsuite/*.a %changelog +* Mon May 13 2024 Chris Co - 10.1.0-15 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 10.1.0-14 - update to build dep latest glibc-static version diff --git a/SPECS-EXTENDED/podman/podman.spec b/SPECS-EXTENDED/podman/podman.spec index 4ddbe088f6c..a1b3daa7095 100644 --- a/SPECS-EXTENDED/podman/podman.spec +++ b/SPECS-EXTENDED/podman/podman.spec @@ -36,7 +36,7 @@ Name: podman Version: 4.1.1 -Release: 21%{?dist} +Release: 22%{?dist} License: ASL 2.0 and BSD and ISC and MIT and MPLv2.0 Summary: Manage Pods, Containers and Container Images Vendor: Microsoft Corporation @@ -51,7 +51,7 @@ BuildRequires: go-md2man BuildRequires: golang BuildRequires: gcc BuildRequires: glib2-devel -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: git BuildRequires: go-rpm-macros BuildRequires: gpgme-devel @@ -387,6 +387,9 @@ cp -pav test/system %{buildroot}/%{_datadir}/%{name}/test/ # rhcontainerbot account currently managed by lsm5 %changelog +* Mon May 13 2024 Chris Co - 4.1.1-22 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 4.1.1-21 - update to build dep latest glibc-static version diff --git a/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.signatures.json b/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.signatures.json index 4760106511f..c26c0d7ab75 100644 --- a/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.signatures.json +++ b/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.signatures.json @@ -14,7 +14,7 @@ "generate-package-note.py": "bd76a8e88a1356fed74863c38e5cf6a20c1c26426ac94ba21dd172578e8ca2a2", "gpgverify": "db0e050f56b694497d70603a6f5c17dd60ddbcf7cee670616851cd389f6767c4", "import_all_modules.py": "543d8c90e7db902e3b4587b7c488419e2f20a6d8b34174b7303dcd70794d8fc7", - "macros": "348741813b676ad05698e1bb0c52be7940a7b3e976a5a77be0bdeb479e938868", + "macros": "a5cde7ca92f0552e15620a94dcc800db0316876239d882caa87fa2c894441e19", "macros.check": "79367176c3c7d10c0158b6e5d881e0fc3c8fd50c5957dad2f097c2d4a37833e7", "macros.fonts": "f52edc646414c5dd0f5f4cdd570f2f9dbe6fb97d4f0db360908deb56d96492f8", "macros.forge": "52cdffd48217cafe33afe47c625cd6aac6460848cd76de66fe61031c26beeaf4", diff --git a/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.spec b/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.spec index 9035f1017c5..398f1c6cf48 100644 --- a/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.spec +++ b/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.spec @@ -7,7 +7,7 @@ Summary: Azure Linux specific rpm macro files Name: azurelinux-rpm-macros Version: %{azl}.0 -Release: 2%{?dist} +Release: 3%{?dist} License: GPL+ AND MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -141,6 +141,9 @@ install -p -m 644 -t %{buildroot}%{rcluadir}/srpm python.lua %{_rpmconfigdir}/macros.d/macros.check %changelog +* Fri May 11 2024 Chris Co - 3.0-3 +- Add macros to allow enabling frame pointers on x86_64 and aarch64 + * Thu Mar 07 2024 Andrew Phelps - 3.0-2 - Update all python-related scripts from Fedora 40 version of python-rpm-macros - Sort python-related sources together diff --git a/SPECS/azurelinux-rpm-macros/macros b/SPECS/azurelinux-rpm-macros/macros index 7aac890c8b4..8270e594e89 100644 --- a/SPECS/azurelinux-rpm-macros/macros +++ b/SPECS/azurelinux-rpm-macros/macros @@ -199,8 +199,12 @@ print(result) %_ld_as_needed 1 %_ld_as_needed_flags %{?_ld_as_needed:-Wl,--as-needed} +# If enabled, include frame pointer information +# Use "%define _include_frame_pointers 1" to enable +%_frame_pointers_cflags %{?_include_frame_pointers:-fno-omit-frame-pointer} + %_legacy_common_support 1 -%__global_compiler_flags -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches %{_hardened_cflags} %{_annotated_cflags} %{?_legacy_common_support: -fcommon} +%__global_compiler_flags -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches %{_hardened_cflags} %{_annotated_cflags} %{_frame_pointers_cflags} %{?_legacy_common_support: -fcommon} # https://src.fedoraproject.org/rpms/redhat-rpm-config/blob/master/f/macros.ldconfig #%ldconfig /sbin/ldconfig diff --git a/SPECS/busybox/busybox.spec b/SPECS/busybox/busybox.spec index f3c2706fe3d..d883fdccb22 100644 --- a/SPECS/busybox/busybox.spec +++ b/SPECS/busybox/busybox.spec @@ -1,7 +1,7 @@ Summary: Statically linked binary providing simplified versions of system commands Name: busybox Version: 1.36.1 -Release: 3%{?dist} +Release: 4%{?dist} License: GPLv2 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -12,7 +12,7 @@ Source2: busybox-petitboot.config Patch0: busybox-1.31.1-stime-fix.patch Patch1: CVE-2022-28391.patch BuildRequires: gcc -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: libselinux-devel >= 1.27.7-2 BuildRequires: libsepol-devel %if 0%{?with_check} @@ -99,6 +99,9 @@ SKIP_KNOWN_BUGS=1 ./runtest %{_mandir}/man1/busybox.petitboot.1.gz %changelog +* Mon May 13 2024 Chris Co - 1.36.1-4 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 1.36.1-3 - update to build dep latest glibc-static version diff --git a/SPECS/flannel/flannel.spec b/SPECS/flannel/flannel.spec index 4095807739e..930da746886 100644 --- a/SPECS/flannel/flannel.spec +++ b/SPECS/flannel/flannel.spec @@ -3,7 +3,7 @@ Summary: Simple and easy way to configure a layer 3 network fabric designed for Kubernetes Name: flannel Version: 0.24.2 -Release: 3%{?dist} +Release: 4%{?dist} License: ASL 2.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -13,7 +13,7 @@ Source0: https://github.com/flannel-io/%{name}/archive/refs/tags/v%{versi Source1: %{name}-%{version}-vendor.tar.gz BuildRequires: gcc BuildRequires: glibc-devel -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: golang >= 1.20 BuildRequires: kernel-headers @@ -50,6 +50,9 @@ install -p -m 755 -t %{buildroot}%{_bindir} ./dist/flanneld %{_bindir}/flanneld %changelog +* Mon May 13 2024 Chris Co - 0.24.2-4 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 0.24.2-3 - update to build dep latest glibc-static version diff --git a/SPECS/glibc/glibc.spec b/SPECS/glibc/glibc.spec index aa511ea285f..90252e51cb4 100644 --- a/SPECS/glibc/glibc.spec +++ b/SPECS/glibc/glibc.spec @@ -4,10 +4,13 @@ # Don't depend on bash by default %define __requires_exclude ^/(bin|usr/bin).*$ +# Enable frame pointers for package +%define _include_frame_pointers 1 + Summary: Main C library Name: glibc Version: 2.38 -Release: 3%{?dist} +Release: 4%{?dist} License: BSD AND GPLv2+ AND Inner-Net AND ISC AND LGPLv2+ AND MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -324,6 +327,9 @@ grep "^FAIL: nptl/tst-eintr1" tests.sum >/dev/null && n=$((n+1)) ||: %defattr(-,root,root) %changelog +* Fri May 10 2024 Chris Co - 2.38-4 +- Enable frame pointers compiler flag + * Mon Mar 11 2024 Dan Streetman - 2.38-3 - provide C.utf8 locale diff --git a/SPECS/kubernetes/kubernetes.spec b/SPECS/kubernetes/kubernetes.spec index 835ea9209d0..249cfadcf86 100644 --- a/SPECS/kubernetes/kubernetes.spec +++ b/SPECS/kubernetes/kubernetes.spec @@ -10,7 +10,7 @@ Summary: Microsoft Kubernetes Name: kubernetes Version: 1.29.1 -Release: 3%{?dist} +Release: 4%{?dist} License: ASL 2.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -19,7 +19,7 @@ URL: https://kubernetes.io/ Source0: https://dl.k8s.io/v%{version}/kubernetes-src.tar.gz#/%{name}-v%{version}.tar.gz Source1: kubelet.service BuildRequires: flex-devel -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: golang BuildRequires: rsync BuildRequires: systemd-devel @@ -269,6 +269,9 @@ fi %{_exec_prefix}/local/bin/pause %changelog +* Mon May 13 2024 Chris Co - 1.29.1-4 +- Update to build dep latest glibc-static version + * Mon Mar 25 2024 Nicolas Guibourge - 1.29.1-3 - Fix build break due to golang version upgrade diff --git a/SPECS/kubevirt/kubevirt.spec b/SPECS/kubevirt/kubevirt.spec index e9af9d43798..8d704f7e3db 100644 --- a/SPECS/kubevirt/kubevirt.spec +++ b/SPECS/kubevirt/kubevirt.spec @@ -20,7 +20,7 @@ Summary: Container native virtualization Name: kubevirt Version: 1.2.0 -Release: 1%{?dist} +Release: 2%{?dist} License: ASL 2.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -34,7 +34,7 @@ Patch0: Cleanup-housekeeping-cgroup-on-vm-del.patch Patch1: Hotplug_detach_grace_period.patch %global debug_package %{nil} BuildRequires: glibc-devel -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: golang >= 1.21 BuildRequires: golang-packaging BuildRequires: pkgconfig @@ -270,6 +270,9 @@ install -p -m 0644 cmd/virt-launcher/qemu.conf %{buildroot}%{_datadir}/kube-virt %{_bindir}/virt-tests %changelog +* Mon May 13 2024 Chris Co - 1.2.0-2 +- Update to build dep latest glibc-static version + * Wed Mar 13 2024 Elaine Zhao - 1.2.0-1 - Bump package version to 1.2.0 from 0.59.0 - Referenced kubevirt.spec from https://code.opensuse.org/package/kubevirt/blob/master/f/kubevirt.spec diff --git a/SPECS/libguestfs/libguestfs.spec b/SPECS/libguestfs/libguestfs.spec index f98d10218e6..42e10e3aac5 100644 --- a/SPECS/libguestfs/libguestfs.spec +++ b/SPECS/libguestfs/libguestfs.spec @@ -25,7 +25,7 @@ Summary: Access and modify virtual machine disk images Name: libguestfs Version: 1.52.0 -Release: 1%{?dist} +Release: 2%{?dist} License: LGPLv2+ Vendor: Microsoft Corporation Distribution: Azure Linux @@ -82,7 +82,7 @@ BuildRequires: gcc-c++ BuildRequires: gdisk BuildRequires: genisoimage BuildRequires: gfs2-utils -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: gobject-introspection-devel BuildRequires: gperf BuildRequires: grep @@ -1146,6 +1146,9 @@ rm ocaml/html/.gitignore %endif %changelog +* Mon May 13 2024 Chris Co - 1.52.0-2 +- Update to build dep latest glibc-static version + * Wed Mar 27 2024 BettyLakes - 1.52.0-1 - Update to 1.52.0 - Move to pcre2 diff --git a/SPECS/mdadm/mdadm.spec b/SPECS/mdadm/mdadm.spec index 42bf6b93418..a8561a6d32b 100644 --- a/SPECS/mdadm/mdadm.spec +++ b/SPECS/mdadm/mdadm.spec @@ -2,7 +2,7 @@ Name: mdadm Version: 4.2 -Release: 2%{?dist} +Release: 3%{?dist} Summary: The mdadm program controls Linux md devices (software RAID arrays) URL: http://www.kernel.org/pub/linux/utils/raid/mdadm/ License: GPLv2+ @@ -145,7 +145,7 @@ Patch198: mdadm-2.5.2-static.patch Patch199: disable-Werror.patch BuildRequires: make -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: systemd-rpm-macros BuildRequires: binutils-devel BuildRequires: gcc @@ -219,6 +219,9 @@ install -m644 %{SOURCE5} %{buildroot}/etc/libreport/events.d %{_datadir}/mdadm/mdcheck %changelog +* Mon May 13 2024 Chris Co - 4.2-3 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 4.2-2 - update to build dep latest glibc-static version diff --git a/SPECS/qemu/qemu.spec b/SPECS/qemu/qemu.spec index a78147e9908..e8fd06e5b8e 100644 --- a/SPECS/qemu/qemu.spec +++ b/SPECS/qemu/qemu.spec @@ -438,7 +438,7 @@ Obsoletes: sgabios-bin <= 1:0.20180715git-10.fc38 Summary: QEMU is a FAST! processor emulator Name: qemu Version: 8.2.0 -Release: 5%{?dist} +Release: 6%{?dist} License: Apache-2.0 AND BSD-2-Clause AND BSD-3-Clause AND FSFAP AND GPL-1.0-or-later AND GPL-2.0-only AND GPL-2.0-or-later AND GPL-2.0-or-later WITH GCC-exception-2.0 AND LGPL-2.0-only AND LGPL-2.0-or-later AND LGPL-2.1-only AND LGPL-2.1-or-later AND MIT AND LicenseRef-Fedora-Public-Domain AND CC-BY-3.0 URL: http://www.qemu.org/ @@ -659,7 +659,7 @@ BuildRequires: rutabaga-gfx-ffi-devel %endif %if %{user_static} -BuildRequires: glibc-static >= 2.38-3 +BuildRequires: glibc-static >= 2.38-4 BuildRequires: glib2-static zlib-static BuildRequires: pcre2-static %endif @@ -3467,6 +3467,9 @@ useradd -r -u 107 -g qemu -G kvm -d / -s /sbin/nologin \ %changelog +* Mon May 13 2024 Chris Co - 8.2.0-6 +- Update to build dep latest glibc-static version + * Mon Apr 01 2024 Kanika Nema - 8.2.0-5 - Disable eventfd based migration tests as they hang when run as part of check. - Diable TLS PSK tests as they fail. diff --git a/SPECS/rust/rust.spec b/SPECS/rust/rust.spec index c9db7b2aa05..a128c3a2eb1 100644 --- a/SPECS/rust/rust.spec +++ b/SPECS/rust/rust.spec @@ -9,7 +9,7 @@ Summary: Rust Programming Language Name: rust Version: 1.75.0 -Release: 5%{?dist} +Release: 6%{?dist} License: (ASL 2.0 OR MIT) AND BSD AND CC-BY-3.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -57,7 +57,7 @@ BuildRequires: ninja-build BuildRequires: openssl-devel BuildRequires: python3 %if 0%{?with_check} -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} %endif # rustc uses a C compiler to invoke the linker, and links to glibc in most cases Requires: binutils @@ -169,6 +169,9 @@ rm %{buildroot}%{_bindir}/*.old %{_mandir}/man1/* %changelog +* Mon May 13 2024 Chris Co - 1.75.0-6 +- Update to build dep latest glibc-static version + * Mon Apr 01 2024 Muhammad Falak - 1.75.0-5 - Enable profiler support diff --git a/SPECS/supermin/supermin.spec b/SPECS/supermin/supermin.spec index bd7dac4969a..35058fa6d80 100644 --- a/SPECS/supermin/supermin.spec +++ b/SPECS/supermin/supermin.spec @@ -21,7 +21,7 @@ Summary: Tool for creating supermin appliances Name: supermin Version: 5.2.2 -Release: 3%{?dist} +Release: 4%{?dist} License: GPLv2+ Vendor: Microsoft Corporation Distribution: Azure Linux @@ -54,7 +54,7 @@ BuildRequires: systemd-udev %if %{with dietlibc} BuildRequires: dietlibc-devel %else -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} %endif %if 0%{?with_check} @@ -129,6 +129,9 @@ make check || { %{_rpmconfigdir}/supermin-find-requires %changelog +* Mon May 13 2024 Chris Co - 5.2.2-4 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 5.2.2-3 - update to build dep latest glibc-static version diff --git a/SPECS/tini/tini.spec b/SPECS/tini/tini.spec index 88a37f23b4b..4cec714a4b0 100644 --- a/SPECS/tini/tini.spec +++ b/SPECS/tini/tini.spec @@ -1,7 +1,7 @@ Summary: A tiny but valid init for containers Name: tini Version: 0.19.0 -Release: 14%{?dist} +Release: 15%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -13,7 +13,7 @@ BuildRequires: diffutils BuildRequires: file BuildRequires: gcc BuildRequires: glibc-devel -BuildRequires: glibc-static >= 2.38-3%{?dist} +BuildRequires: glibc-static >= 2.38-4%{?dist} BuildRequires: kernel-headers BuildRequires: make BuildRequires: sed @@ -66,6 +66,9 @@ ln -s %{_bindir}/tini-static %{buildroot}%{_bindir}/docker-init %{_bindir}/docker-init %changelog +* Mon May 13 2024 Chris Co - 0.19.0-15 +- Update to build dep latest glibc-static version + * Mon Mar 11 2024 Dan Streetman - 0.19.0-14 - update to build dep latest glibc-static version diff --git a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt index a1265975fd4..e6449beb676 100644 --- a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt @@ -1,12 +1,12 @@ filesystem-1.1-21.azl3.aarch64.rpm kernel-headers-6.6.29.1-3.azl3.noarch.rpm -glibc-2.38-3.azl3.aarch64.rpm -glibc-devel-2.38-3.azl3.aarch64.rpm -glibc-i18n-2.38-3.azl3.aarch64.rpm -glibc-iconv-2.38-3.azl3.aarch64.rpm -glibc-lang-2.38-3.azl3.aarch64.rpm -glibc-nscd-2.38-3.azl3.aarch64.rpm -glibc-tools-2.38-3.azl3.aarch64.rpm +glibc-2.38-4.azl3.aarch64.rpm +glibc-devel-2.38-4.azl3.aarch64.rpm +glibc-i18n-2.38-4.azl3.aarch64.rpm +glibc-iconv-2.38-4.azl3.aarch64.rpm +glibc-lang-2.38-4.azl3.aarch64.rpm +glibc-nscd-2.38-4.azl3.aarch64.rpm +glibc-tools-2.38-4.azl3.aarch64.rpm zlib-1.3.1-1.azl3.aarch64.rpm zlib-devel-1.3.1-1.azl3.aarch64.rpm file-5.45-1.azl3.aarch64.rpm @@ -204,8 +204,8 @@ libltdl-2.4.7-1.azl3.aarch64.rpm libltdl-devel-2.4.7-1.azl3.aarch64.rpm lua-5.4.6-1.azl3.aarch64.rpm lua-libs-5.4.6-1.azl3.aarch64.rpm -azurelinux-rpm-macros-3.0-2.azl3.noarch.rpm -azurelinux-check-macros-3.0-2.azl3.noarch.rpm +azurelinux-rpm-macros-3.0-3.azl3.noarch.rpm +azurelinux-check-macros-3.0-3.azl3.noarch.rpm tdnf-3.5.6-1.azl3.aarch64.rpm tdnf-cli-libs-3.5.6-1.azl3.aarch64.rpm tdnf-devel-3.5.6-1.azl3.aarch64.rpm diff --git a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt index 29cb66ffe22..06c57d569c7 100644 --- a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt @@ -1,12 +1,12 @@ filesystem-1.1-21.azl3.x86_64.rpm kernel-headers-6.6.29.1-3.azl3.noarch.rpm -glibc-2.38-3.azl3.x86_64.rpm -glibc-devel-2.38-3.azl3.x86_64.rpm -glibc-i18n-2.38-3.azl3.x86_64.rpm -glibc-iconv-2.38-3.azl3.x86_64.rpm -glibc-lang-2.38-3.azl3.x86_64.rpm -glibc-nscd-2.38-3.azl3.x86_64.rpm -glibc-tools-2.38-3.azl3.x86_64.rpm +glibc-2.38-4.azl3.x86_64.rpm +glibc-devel-2.38-4.azl3.x86_64.rpm +glibc-i18n-2.38-4.azl3.x86_64.rpm +glibc-iconv-2.38-4.azl3.x86_64.rpm +glibc-lang-2.38-4.azl3.x86_64.rpm +glibc-nscd-2.38-4.azl3.x86_64.rpm +glibc-tools-2.38-4.azl3.x86_64.rpm zlib-1.3.1-1.azl3.x86_64.rpm zlib-devel-1.3.1-1.azl3.x86_64.rpm file-5.45-1.azl3.x86_64.rpm @@ -204,8 +204,8 @@ libltdl-2.4.7-1.azl3.x86_64.rpm libltdl-devel-2.4.7-1.azl3.x86_64.rpm lua-5.4.6-1.azl3.x86_64.rpm lua-libs-5.4.6-1.azl3.x86_64.rpm -azurelinux-rpm-macros-3.0-2.azl3.noarch.rpm -azurelinux-check-macros-3.0-2.azl3.noarch.rpm +azurelinux-rpm-macros-3.0-3.azl3.noarch.rpm +azurelinux-check-macros-3.0-3.azl3.noarch.rpm tdnf-3.5.6-1.azl3.x86_64.rpm tdnf-cli-libs-3.5.6-1.azl3.x86_64.rpm tdnf-devel-3.5.6-1.azl3.x86_64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index a4932886e63..957ae3d03b2 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -5,7 +5,7 @@ audit-devel-3.1.2-1.azl3.aarch64.rpm audit-libs-3.1.2-1.azl3.aarch64.rpm autoconf-2.72-1.azl3.noarch.rpm automake-1.16.5-1.azl3.noarch.rpm -azurelinux-check-macros-3.0-2.azl3.noarch.rpm +azurelinux-check-macros-3.0-3.azl3.noarch.rpm azurelinux-repos-3.0-2.azl3.noarch.rpm azurelinux-repos-debug-3.0-2.azl3.noarch.rpm azurelinux-repos-debug-preview-3.0-2.azl3.noarch.rpm @@ -19,7 +19,7 @@ azurelinux-repos-ms-oss-3.0-2.azl3.noarch.rpm azurelinux-repos-ms-oss-preview-3.0-2.azl3.noarch.rpm azurelinux-repos-preview-3.0-2.azl3.noarch.rpm azurelinux-repos-shared-3.0-2.azl3.noarch.rpm -azurelinux-rpm-macros-3.0-2.azl3.noarch.rpm +azurelinux-rpm-macros-3.0-3.azl3.noarch.rpm bash-5.2.15-1.azl3.aarch64.rpm bash-debuginfo-5.2.15-1.azl3.aarch64.rpm bash-devel-5.2.15-1.azl3.aarch64.rpm @@ -121,15 +121,15 @@ glib-debuginfo-2.78.1-4.azl3.aarch64.rpm glib-devel-2.78.1-4.azl3.aarch64.rpm glib-doc-2.78.1-4.azl3.noarch.rpm glib-schemas-2.78.1-4.azl3.aarch64.rpm -glibc-2.38-3.azl3.aarch64.rpm -glibc-debuginfo-2.38-3.azl3.aarch64.rpm -glibc-devel-2.38-3.azl3.aarch64.rpm -glibc-i18n-2.38-3.azl3.aarch64.rpm -glibc-iconv-2.38-3.azl3.aarch64.rpm -glibc-lang-2.38-3.azl3.aarch64.rpm -glibc-nscd-2.38-3.azl3.aarch64.rpm -glibc-static-2.38-3.azl3.aarch64.rpm -glibc-tools-2.38-3.azl3.aarch64.rpm +glibc-2.38-4.azl3.aarch64.rpm +glibc-debuginfo-2.38-4.azl3.aarch64.rpm +glibc-devel-2.38-4.azl3.aarch64.rpm +glibc-i18n-2.38-4.azl3.aarch64.rpm +glibc-iconv-2.38-4.azl3.aarch64.rpm +glibc-lang-2.38-4.azl3.aarch64.rpm +glibc-nscd-2.38-4.azl3.aarch64.rpm +glibc-static-2.38-4.azl3.aarch64.rpm +glibc-tools-2.38-4.azl3.aarch64.rpm gmp-6.3.0-1.azl3.aarch64.rpm gmp-debuginfo-6.3.0-1.azl3.aarch64.rpm gmp-devel-6.3.0-1.azl3.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index 4ebdea9e795..97a81334f89 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -5,7 +5,7 @@ audit-devel-3.1.2-1.azl3.x86_64.rpm audit-libs-3.1.2-1.azl3.x86_64.rpm autoconf-2.72-1.azl3.noarch.rpm automake-1.16.5-1.azl3.noarch.rpm -azurelinux-check-macros-3.0-2.azl3.noarch.rpm +azurelinux-check-macros-3.0-3.azl3.noarch.rpm azurelinux-repos-3.0-2.azl3.noarch.rpm azurelinux-repos-debug-3.0-2.azl3.noarch.rpm azurelinux-repos-debug-preview-3.0-2.azl3.noarch.rpm @@ -19,7 +19,7 @@ azurelinux-repos-ms-oss-3.0-2.azl3.noarch.rpm azurelinux-repos-ms-oss-preview-3.0-2.azl3.noarch.rpm azurelinux-repos-preview-3.0-2.azl3.noarch.rpm azurelinux-repos-shared-3.0-2.azl3.noarch.rpm -azurelinux-rpm-macros-3.0-2.azl3.noarch.rpm +azurelinux-rpm-macros-3.0-3.azl3.noarch.rpm bash-5.2.15-1.azl3.x86_64.rpm bash-debuginfo-5.2.15-1.azl3.x86_64.rpm bash-devel-5.2.15-1.azl3.x86_64.rpm @@ -126,15 +126,15 @@ glib-debuginfo-2.78.1-4.azl3.x86_64.rpm glib-devel-2.78.1-4.azl3.x86_64.rpm glib-doc-2.78.1-4.azl3.noarch.rpm glib-schemas-2.78.1-4.azl3.x86_64.rpm -glibc-2.38-3.azl3.x86_64.rpm -glibc-debuginfo-2.38-3.azl3.x86_64.rpm -glibc-devel-2.38-3.azl3.x86_64.rpm -glibc-i18n-2.38-3.azl3.x86_64.rpm -glibc-iconv-2.38-3.azl3.x86_64.rpm -glibc-lang-2.38-3.azl3.x86_64.rpm -glibc-nscd-2.38-3.azl3.x86_64.rpm -glibc-static-2.38-3.azl3.x86_64.rpm -glibc-tools-2.38-3.azl3.x86_64.rpm +glibc-2.38-4.azl3.x86_64.rpm +glibc-debuginfo-2.38-4.azl3.x86_64.rpm +glibc-devel-2.38-4.azl3.x86_64.rpm +glibc-i18n-2.38-4.azl3.x86_64.rpm +glibc-iconv-2.38-4.azl3.x86_64.rpm +glibc-lang-2.38-4.azl3.x86_64.rpm +glibc-nscd-2.38-4.azl3.x86_64.rpm +glibc-static-2.38-4.azl3.x86_64.rpm +glibc-tools-2.38-4.azl3.x86_64.rpm gmp-6.3.0-1.azl3.x86_64.rpm gmp-debuginfo-6.3.0-1.azl3.x86_64.rpm gmp-devel-6.3.0-1.azl3.x86_64.rpm From c6787c743d4de878d36a84819d19d9d1f03e7685 Mon Sep 17 00:00:00 2001 From: nicolas guibourge Date: Thu, 16 May 2024 10:42:02 -0400 Subject: [PATCH 18/51] Upgrade azure-iot-sdk-c to 2024-03-04 CVE-2024-21646, CVE-2024-27099, CVE-2024-25110 (#9042) Co-authored-by: CBL-Mariner Servicing Account --- SPECS/azure-iot-sdk-c/azure-iot-sdk-c.signatures.json | 2 +- SPECS/azure-iot-sdk-c/azure-iot-sdk-c.spec | 7 +++++-- cgmanifest.json | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/SPECS/azure-iot-sdk-c/azure-iot-sdk-c.signatures.json b/SPECS/azure-iot-sdk-c/azure-iot-sdk-c.signatures.json index b740cc1d9fa..89030d836bb 100644 --- a/SPECS/azure-iot-sdk-c/azure-iot-sdk-c.signatures.json +++ b/SPECS/azure-iot-sdk-c/azure-iot-sdk-c.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "azure-iot-sdk-c-2023.08.07.tar.gz": "ab892d7362a8fc72cea1c599bc5682d60ad94f11cecb7acafbcd8ba81d795149" + "azure-iot-sdk-c-2024.03.04.tar.gz": "70ddeb1f638fa809bae9230a199a5672c17fdccef5a75cbbec4a48efa594c0bb" } } diff --git a/SPECS/azure-iot-sdk-c/azure-iot-sdk-c.spec b/SPECS/azure-iot-sdk-c/azure-iot-sdk-c.spec index 6215495f6c1..a47968e4549 100644 --- a/SPECS/azure-iot-sdk-c/azure-iot-sdk-c.spec +++ b/SPECS/azure-iot-sdk-c/azure-iot-sdk-c.spec @@ -6,7 +6,7 @@ Name: azure-iot-sdk-c # For apt-get packages they fix the version number to 0.2.0 and increase the release number with each release. # Since we want to control the release number as thr distribution, this scheme is not applicable for us. # They also used to use a regular versioning scheme like 1.3.7 but they did not tag their latest LTS with a version like that. -Version: 2023.08.07 +Version: 2024.03.04 Release: 1%{?dist} License: MIT Group: Applications/File @@ -23,7 +23,7 @@ Distribution: Azure Linux # --owner=0 --group=0 --numeric-owner \ # --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ # -cf %{name}-%{version}.tar.gz %{name} -Source0: %{name}-%{version}.tar.gz +Source0: https://github.com/Azure/azure-iot-sdk-c/archive/refs/tags/LTS_03_2024.tar.gz#/%{name}-%{version}.tar.gz BuildRequires: cmake BuildRequires: build-essential @@ -90,6 +90,9 @@ install -p -m 755 provisioning_client/tools/tpm_device_provision/tpm_device_prov /usr/cmake/* %changelog +* Mon May 06 2024 Nicolas Guibourge - 2024.03.04-1 +- Upgrade to LTS_03_2024 + * Thu Jan 04 2024 Karim Eldegwy - 2023.08.07-1 - Upgrate to 2023.08.07-1 diff --git a/cgmanifest.json b/cgmanifest.json index d3daa0b0837..c0e12265127 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -855,8 +855,8 @@ "type": "other", "other": { "name": "azure-iot-sdk-c", - "version": "2023.08.07", - "downloadUrl": "https://github.com/Azure/azure-iot-sdk-c/archive/refs/tags/LTS_08_2023.tar.gz" + "version": "2024.03.04", + "downloadUrl": "https://github.com/Azure/azure-iot-sdk-c/archive/refs/tags/LTS_03_2024.tar.gz" } } }, From f58ea1fa106c0a4a3c50a73d941971c34b75a36f Mon Sep 17 00:00:00 2001 From: Elaheh Dehghani <108492863+ellie-di@users.noreply.github.com> Date: Thu, 16 May 2024 13:13:38 -0700 Subject: [PATCH 19/51] Add new compress-toolchain-final-rpms target to makefile (#9127) Co-authored-by: Daniel McIlvaney --- toolkit/scripts/toolchain.mk | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/toolkit/scripts/toolchain.mk b/toolkit/scripts/toolchain.mk index f6079fee57b..e85fb75d365 100644 --- a/toolkit/scripts/toolchain.mk +++ b/toolkit/scripts/toolchain.mk @@ -110,6 +110,12 @@ compress-toolchain: tar -cvp -f $(final_toolchain) -C $(toolchain_build_dir) built_rpms_all $(if $(CACHE_DIR), cp $(raw_toolchain) $(final_toolchain) $(CACHE_DIR)) +# Creates a toolchain archive from the final toolchain rpms placed in ./build/toolchain_rpms, regardless of their source. This may be used with both locally built or downloaded toolchains. +# The archive is placed in ./out/toolchain_final_extracted_rpms.tar.gz and may be used with TOOLCHAIN_ARCHIVE= to populate a toolchain during subsequent builds. +compress-toolchain-final-rpms: + tar -I $(ARCHIVE_TOOL) -cvf $(OUT_DIR)/toolchain_final_extracted_rpms.tar.gz --transform='s`/*[^/]*/``' -C $(TOOLCHAIN_RPMS_DIR) . + echo "Created $(OUT_DIR)/toolchain_final_extracted_rpms.tar.gz" + # After hydrating the toolchain run # "sudo touch build/toolchain/toolchain_from_container.tar.gz" (should really check for existence of files in toolchain_*.txt) # "sudo make toolchain REBUILD_TOOLCHAIN=y INCREMENTAL_TOOLCHAIN=y" From 27def2843b2cb10e83af64a8141d7c0cddea022d Mon Sep 17 00:00:00 2001 From: Pawel Winogrodzki Date: Thu, 16 May 2024 13:38:31 -0700 Subject: [PATCH 20/51] Removing hard-coded 'with_check' from 'podman.spec'. (#9105) --- SPECS-EXTENDED/podman/podman.spec | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/SPECS-EXTENDED/podman/podman.spec b/SPECS-EXTENDED/podman/podman.spec index a1b3daa7095..a9a1ec47979 100644 --- a/SPECS-EXTENDED/podman/podman.spec +++ b/SPECS-EXTENDED/podman/podman.spec @@ -1,4 +1,3 @@ -%global with_check 0 %global with_debug 1 %if 0%{?with_debug} @@ -36,7 +35,7 @@ Name: podman Version: 4.1.1 -Release: 22%{?dist} +Release: 23%{?dist} License: ASL 2.0 and BSD and ISC and MIT and MPLv2.0 Summary: Manage Pods, Containers and Container Images Vendor: Microsoft Corporation @@ -387,6 +386,9 @@ cp -pav test/system %{buildroot}/%{_datadir}/%{name}/test/ # rhcontainerbot account currently managed by lsm5 %changelog +* Tue May 14 2024 Pawel Winogrodzki - 4.1.1-23 +- Cleaning up the invalid global 'with_check' macro. + * Mon May 13 2024 Chris Co - 4.1.1-22 - Update to build dep latest glibc-static version From 253b6a9ff1f1708c2a55f7fde8292340374f425f Mon Sep 17 00:00:00 2001 From: Neha Agarwal <58672330+neha170@users.noreply.github.com> Date: Thu, 16 May 2024 13:45:55 -0700 Subject: [PATCH 21/51] refactor depsearch (#9029) --- toolkit/tools/depsearch/depsearch.go | 385 +--------------------- toolkit/tools/pkg/depsearch/depsearch.go | 386 +++++++++++++++++++++++ 2 files changed, 398 insertions(+), 373 deletions(-) create mode 100644 toolkit/tools/pkg/depsearch/depsearch.go diff --git a/toolkit/tools/depsearch/depsearch.go b/toolkit/tools/depsearch/depsearch.go index ee8a94aa7fc..b58d2950b91 100644 --- a/toolkit/tools/depsearch/depsearch.go +++ b/toolkit/tools/depsearch/depsearch.go @@ -1,29 +1,22 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +// depsearch analyzes the dependency graphs from a build, and can list either the packages which depend on a given search term, +// or all the packages the searched packages depend on to build/run. + package main import ( - "fmt" "os" - "path/filepath" - "sort" "github.com/microsoft/azurelinux/toolkit/tools/internal/exe" - "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/pkggraph" - "github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils" - "github.com/microsoft/azurelinux/toolkit/tools/scheduler/schedulerutils" + "github.com/microsoft/azurelinux/toolkit/tools/pkg/depsearch" - "gonum.org/v1/gonum/graph" "gopkg.in/alecthomas/kingpin.v2" ) -const ( - defaultFilterPath = "./resources/manifests/package/toolchain_x86_64.txt" -) - var ( app = kingpin.New("depsearch", "Returns a list of everything that depends on a given package or spec") @@ -48,11 +41,6 @@ var ( ) func main() { - var ( - outputGraph *pkggraph.PkgGraph - root *pkggraph.PkgNode - ) - app.Version(exe.ToolkitVersion) kingpin.MustParse(app.Parse(os.Args[1:])) logger.InitBestEffort(logFlags) @@ -69,11 +57,13 @@ func main() { // Only one of runtimeOnlyPlusBuild or runtimeOnly can be set if *runtimeFilterLevel < -1 { - logger.Log.Fatalf("Invalid runtime filter level '%d', valid ranges are -1, >=0", *runtimeFilterLevel) + logger.Log.Fatalf("Invalid runtime filter level %d, valid ranges are -1, >=0", *runtimeFilterLevel) } // We can color the entries when using --tree, or limit the output in all modes with --rpm-filter - configureFilterFiles(filterFile, filter) + err := depsearch.ConfigureFilterFiles(filterFile, *filter) + logger.PanicOnError(err) + if len(*filterFile) > 0 && (*filter || *printTree) { logger.Log.Infof("Applying package filter from (%s)", *filterFile) } else { @@ -89,366 +79,15 @@ func main() { logger.Log.Panicf("Failed to read DOT graph with error: %s", err) } - // Generate a list of nodes to search from - nodeListPkg := searchForPkg(graph, pkgSearchList) - nodeListSpec := searchForSpec(graph, specSearchList) - nodeListGoal := searchForGoal(graph, goalSearchList) - - nodeLists := append(nodeListPkg, append(nodeListSpec, nodeListGoal...)...) - nodeSet := sliceutils.RemoveDuplicatesFromSlice(nodeLists) - - if len(nodeSet) == 0 { - logger.Log.Panicf("Could not find any nodes matching pkgs:[%s] or specs:[%s] or goals[%s]", *pkgsToSearch, *specsToSearch, *goalsToSearch) - } else { - logger.Log.Infof("Found %d nodes to consider", len(nodeSet)) - } - - if *reverseSearch { - logger.Log.Infof("Reversed search will list all the dependencies of the provided packages") - outputGraph, root, err = buildRequiresGraph(graph, nodeSet) - } else { - logger.Log.Infof("Forward search will list all dependants which rely on any of the provided packages") - outputGraph, root, err = buildDependsOnGraph(graph, nodeSet) - } + outputGraph, root, err := depsearch.GetDependencyGraph(pkgSearchList, specSearchList, goalSearchList, graph, *reverseSearch) + logger.PanicOnError(err) + err = depsearch.PrintSpecs(outputGraph, *printTree, *filter, *filterFile, *printDuplicates, *verbosity, *maxDepth, *runtimeFilterLevel, root) if err != nil { - logger.Log.Panicf("Failed to generate graph to run depsearch on: %s", err) + logger.Log.Fatalf("Failed to print:\n%s", err) } - printSpecs(outputGraph, *printTree, *filter, *filterFile, *printDuplicates, *verbosity, *maxDepth, *runtimeFilterLevel, root) - if len(*outputGraphFile) > 0 { pkggraph.WriteDOTGraphFile(outputGraph, *outputGraphFile) } } - -func configureFilterFiles(filterFile *string, filter *bool) { - setDefault := false - if len(*filterFile) == 0 { - *filterFile = defaultFilterPath - setDefault = true - } - isFile, err := file.PathExists(*filterFile) - if err != nil { - logger.Log.Panicf("Failed to query if filter file (%s) exists: %s", *filterFile, err) - } - - // If we are just trying to use the default, its fine if its missing. - if !isFile && setDefault { - logger.Log.Warnf("Default toolchain filter file (%s) not found, setting to ''", *filterFile) - *filterFile = "" - } - - if len(*filterFile) == 0 && *filter { - logger.Log.Panic("Must pass a --rpm-filter-file to use the filter function, consider './resources/manifests/package/toolchain_x86_64.txt'") - } -} - -func searchForGoal(graph *pkggraph.PkgGraph, goals []string) (list []*pkggraph.PkgNode) { - for _, goal := range goals { - n := graph.FindGoalNode(goal) - if n != nil { - list = append(list, n) - } - } - return -} - -func searchForPkg(graph *pkggraph.PkgGraph, packages []string) (list []*pkggraph.PkgNode) { - for _, n := range graph.AllPreferredRunNodes() { - nodeName := n.VersionedPkg.Name - for _, searchName := range packages { - if nodeName == searchName { - list = append(list, n) - } - } - } - return -} - -func searchForSpec(graph *pkggraph.PkgGraph, specs []string) (list []*pkggraph.PkgNode) { - for _, n := range graph.AllPreferredRunNodes() { - nodeSpec := n.SpecName() - for _, searchSpec := range specs { - if nodeSpec == searchSpec { - list = append(list, n) - } - } - } - return -} - -func buildRequiresGraph(graphIn *pkggraph.PkgGraph, nodeList []*pkggraph.PkgNode) (graphOut *pkggraph.PkgGraph, root *pkggraph.PkgNode, err error) { - // Make a copy of the graph - newGraph, err := graphIn.DeepCopy() - if err != nil { - return - } - - // Add a goal node to all the things we care about - root = newGraph.AddMetaNode(nil, nodeList) - graphOut, err = newGraph.CreateSubGraph(root) - if err != nil { - return - } - - return -} - -func buildDependsOnGraph(graphIn *pkggraph.PkgGraph, nodeList []*pkggraph.PkgNode) (graphOut *pkggraph.PkgGraph, root *pkggraph.PkgNode, err error) { - // Make a copy of the graph - reversedGraph, err := graphIn.DeepCopy() - if err != nil { - return - } - - // Then reverse every edge in the graph - for _, edge := range graph.EdgesOf(reversedGraph.Edges()) { - reversedGraph.RemoveEdge(edge.From().ID(), edge.To().ID()) - reversedGraph.SetEdge(edge.ReversedEdge()) - } - - // Add a goal node to all the things we care about - root = reversedGraph.AddMetaNode(nil, nodeList) - graphOut, err = reversedGraph.CreateSubGraph(root) - if err != nil { - return - } - - return -} - -const ( - colorReset = "\033[0m" - colorRed = "\033[31m" -) - -var ( - reservedFiles map[string]bool -) - -func formatNode(n *pkggraph.PkgNode, verbosity int) string { - switch verbosity { - case 1: - return n.SpecName() - case 2: - return filepath.Base(n.RpmPath) - case 3: - return fmt.Sprintf("'%s' from node '%s'", filepath.Base(n.RpmPath), n.FriendlyName()) - case 4: - return fmt.Sprintf("(%v)'%#v'", n.VersionedPkg, *n) - default: - logger.Log.Fatalf("Invalid verbosity level %v", verbosity) - } - return "" -} - -func isFilteredFile(path, filterFile string) bool { - if len(filterFile) > 0 { - if len(reservedFiles) == 0 { - reservedFileList, err := schedulerutils.ReadReservedFilesList(filterFile) - if err != nil { - logger.Log.Fatalf("Failed to load filter file (%s): %s", filterFile, err) - } - reservedFiles = sliceutils.SliceToSet[string](reservedFileList) - } - base := filepath.Base(path) - - return len(path) > 0 && reservedFiles[base] - } else { - return false - } -} - -type treeNode struct { - lines []string -} - -type treeSearch struct { - graph *pkggraph.PkgGraph - - // Don't print a node twice, just add '...' instead to save space - alreadyAdded map[*pkggraph.PkgNode]bool - // These nodes were caught by the filter and should be marked - filteredNodes map[*pkggraph.PkgNode]bool - // These nodes are not in the filter list - normalNodes map[*pkggraph.PkgNode]bool - - nodesVisited, nodesTotal int -} - -func createSearch(g *pkggraph.PkgGraph, root *pkggraph.PkgNode) (t *treeSearch, err error) { - newSearch := &treeSearch{ - graph: g, - alreadyAdded: make(map[*pkggraph.PkgNode]bool), - filteredNodes: make(map[*pkggraph.PkgNode]bool), - normalNodes: make(map[*pkggraph.PkgNode]bool), - nodesVisited: 0, - } - - //Calculate the number of nodes we might visit: - subGraph, err := g.CreateSubGraph(root) - if err != nil { - logger.Log.Fatalf("Failed to calculate number of nodes: %s", err) - } - - //This is the worst case possible number of searche to make - possibleEdges := subGraph.Edges().Len() * subGraph.Nodes().Len() - newSearch.nodesTotal = possibleEdges - - return newSearch, nil -} - -func (t *treeSearch) FilteredNodes() (nodes []*pkggraph.PkgNode) { - nodes = []*pkggraph.PkgNode{} - for n := range t.filteredNodes { - nodes = append(nodes, n) - } - return nodes -} - -func (t *treeSearch) NonFilteredNodes() (nodes []*pkggraph.PkgNode) { - nodes = []*pkggraph.PkgNode{} - for n := range t.normalNodes { - nodes = append(nodes, n) - } - return nodes -} - -// Call this ever time a node is processed, will print an update ever 100 nodes -func (t *treeSearch) printProgress() { - if t.nodesVisited%10000 == 0 { - logger.Log.Infof("Scanned %d nodes", t.nodesVisited) - } - t.nodesVisited++ -} - -// Run a DFS and generate a string representation of the tree. Optionally ignore all branches that only container nodes in -// -// the filter list (ie given the toolchain manifest, only print those branches which container non-toolchain packages) -func (t *treeSearch) treeNodeToString(n *pkggraph.PkgNode, depth, maxDepth int, filter bool, filterFile string, verbosity int, generateStrings, printDuplicates bool, runtimeFilterLevel int) (lines []string, hasNonToolchain bool) { - t.printProgress() - // We only care about run nodes for the purposes of detecting toolchain files - hasNonToolchain = n.Type == pkggraph.TypeLocalBuild && !isFilteredFile(n.RpmPath, filterFile) - - if !printDuplicates && t.alreadyAdded[n] { - return []string{}, false - } else { - t.alreadyAdded[n] = true - } - - thisNode := formatNode(n, verbosity) - if isFilteredFile(n.RpmPath, filterFile) { - // Highlight nodes that are in the filter file list in red, and add them to the list - if generateStrings { - lines = append(lines, "__"+colorRed+thisNode+colorReset) - } - if n.Type == pkggraph.TypeLocalRun { - // We only want to record run nodes for the purposes of listing packages in non-tree mode - t.filteredNodes[n] = true - } - } else { - // Non filtered files print normally - if generateStrings { - lines = append(lines, "__"+thisNode) - } - if n.Type == pkggraph.TypeLocalRun { - // We only want to record run nodes for the purposes of listing packages in non-tree mode - t.normalNodes[n] = true - } - } - - var childrenTreeNodes []treeNode - nodes := t.graph.From(n.ID()) - // Bail out early if we exceed max depth, maxDepth of -1 means no limit. - if depth < maxDepth || maxDepth == -1 { - var ( - childLines []string - childHasMissingToolchainPkg = false - haveDuplicatedEntry = false - ) - for nodes.Next() { - child := nodes.Node().(*pkggraph.PkgNode) - - // If we are only looking for runtime, skip build nodes (except for the root nodes: goal + each package node we care about) - if runtimeFilterLevel != -1 && depth > runtimeFilterLevel && child.Type == pkggraph.TypeLocalBuild { - continue - } - childLines, childHasMissingToolchainPkg = t.treeNodeToString(child, depth+1, maxDepth, filter, filterFile, verbosity, generateStrings, printDuplicates, runtimeFilterLevel) - hasNonToolchain = hasNonToolchain || childHasMissingToolchainPkg - - // A child will return an empty string list if it, and all its children, are either duplicates or have been filtered out - if len(childLines) > 0 { - tn := treeNode{lines: childLines} - childrenTreeNodes = append(childrenTreeNodes, tn) - } else { - haveDuplicatedEntry = true - } - } - - // If we have duplicated entires (ie empty strings), and we aren't removing all non-filtered entries, represent them with a '...' - // as the last entry instead of the node name. - if haveDuplicatedEntry && !filter { - childrenTreeNodes = append(childrenTreeNodes, treeNode{lines: []string{"..."}}) - } - - if len(childrenTreeNodes) > 0 && generateStrings { - firstN := childrenTreeNodes[:len(childrenTreeNodes)-1] - last := childrenTreeNodes[len(childrenTreeNodes)-1] - for _, tn := range firstN { - for _, l := range tn.lines { - lines = append(lines, " |"+l) - } - } - lines = append(lines, " |"+last.lines[0]) - for _, l := range last.lines[1:] { - lines = append(lines, " "+l) - } - } - } else { - lines = append(lines, " |-->") - // We are bailing out early, we don't know if this should be filtered, assume the worst - hasNonToolchain = true - } - - if !hasNonToolchain && filter && depth > 0 { - return []string{}, false - } - - return lines, hasNonToolchain -} - -func printSpecs(graph *pkggraph.PkgGraph, tree, filter bool, filterFile string, printDuplicates bool, verbosity, maxDepth, runtimeFilterLevel int, root *pkggraph.PkgNode) { - t, err := createSearch(graph, root) - if err != nil { - logger.Log.Fatalf("Failed to start search: %s", err) - } - // May as well use the tree searh to parse all the filtered packages etc, even if we are - // just printing a list - lines, _ := t.treeNodeToString(root, 0, maxDepth, filter, filterFile, verbosity, tree, printDuplicates, runtimeFilterLevel) - if tree { - for _, l := range lines { - fmt.Println(l) - } - } else { - results := make(map[string]bool) - if !filter { - // Only include toolchain packages if we aren't trying to find packages that have - // infiltrated the toolchain - for _, n := range t.FilteredNodes() { - results[formatNode(n, verbosity)] = true - } - } - // Always include normal nodes - for _, n := range t.NonFilteredNodes() { - results[formatNode(n, verbosity)] = true - } - - // Contert to list and sort - printLines := sliceutils.SetToSlice[string](results) - sort.Strings(printLines) - for _, l := range printLines { - fmt.Println(l) - } - } -} diff --git a/toolkit/tools/pkg/depsearch/depsearch.go b/toolkit/tools/pkg/depsearch/depsearch.go new file mode 100644 index 00000000000..cfdf6cdd367 --- /dev/null +++ b/toolkit/tools/pkg/depsearch/depsearch.go @@ -0,0 +1,386 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package depsearch + +import ( + "fmt" + "path/filepath" + "sort" + + "github.com/microsoft/azurelinux/toolkit/tools/internal/file" + "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" + "github.com/microsoft/azurelinux/toolkit/tools/internal/pkggraph" + "github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils" + "github.com/microsoft/azurelinux/toolkit/tools/scheduler/schedulerutils" + + "github.com/fatih/color" + "gonum.org/v1/gonum/graph" +) + +const ( + defaultFilterPath = "./resources/manifests/package/toolchain_x86_64.txt" +) + +var ( + reservedFiles map[string]bool +) + +type treeNode struct { + lines []string +} + +type treeSearch struct { + graph *pkggraph.PkgGraph + + // Don't print a node twice, just add '...' instead to save space + alreadyAdded map[*pkggraph.PkgNode]bool + // These nodes were caught by the filter and should be marked + filteredNodes map[*pkggraph.PkgNode]bool + // These nodes are not in the filter list + normalNodes map[*pkggraph.PkgNode]bool + + nodesVisited, nodesTotal int +} + +func GetDependencyGraph(pkgSearchList, specSearchList, goalSearchList []string, graph *pkggraph.PkgGraph, reverseSearch bool) (outputGraph *pkggraph.PkgGraph, root *pkggraph.PkgNode, err error) { + // Generate a list of nodes to search from + nodeListPkg := searchForPkg(graph, pkgSearchList) + nodeListSpec := searchForSpec(graph, specSearchList) + nodeListGoal := searchForGoal(graph, goalSearchList) + + nodeLists := append(nodeListPkg, append(nodeListSpec, nodeListGoal...)...) + nodeSet := sliceutils.RemoveDuplicatesFromSlice(nodeLists) + + if len(nodeSet) == 0 { + err = fmt.Errorf("failed to find nodes matching pkgs:(%v) or specs:(%v) or goals:(%v)", pkgSearchList, specSearchList, goalSearchList) + return + } else { + logger.Log.Infof("Found %d nodes to consider", len(nodeSet)) + } + + if reverseSearch { + logger.Log.Infof("Reversed search will list all the dependencies of the provided packages") + outputGraph, root, err = buildRequiresGraph(graph, nodeSet) + } else { + logger.Log.Infof("Forward search will list all dependants which rely on any of the provided packages") + outputGraph, root, err = buildDependsOnGraph(graph, nodeSet) + } + + if err != nil { + err = fmt.Errorf("failed to generate graph to run depsearch on:\n%w", err) + return + } + return +} + +func ConfigureFilterFiles(filterFile *string, filter bool) (err error) { + setDefault := false + if len(*filterFile) == 0 { + *filterFile = defaultFilterPath + setDefault = true + } + isFile, err := file.PathExists(*filterFile) + if err != nil { + err = fmt.Errorf("failed to query if filter file (%s) exists:\n%w", *filterFile, err) + return + } + + // If we are just trying to use the default, its fine if its missing. + if !isFile && setDefault { + logger.Log.Warnf("Default toolchain filter file (%s) not found, setting to ''", *filterFile) + *filterFile = "" + } + + if len(*filterFile) == 0 && filter { + err = fmt.Errorf("Must pass a --rpm-filter-file to use the filter function, consider './resources/manifests/package/toolchain_x86_64.txt'") + return + } + return +} + +func searchForGoal(graph *pkggraph.PkgGraph, goals []string) (list []*pkggraph.PkgNode) { + for _, goal := range goals { + n := graph.FindGoalNode(goal) + if n != nil { + list = append(list, n) + } + } + return +} + +func searchForPkg(graph *pkggraph.PkgGraph, packages []string) (list []*pkggraph.PkgNode) { + for _, n := range graph.AllPreferredRunNodes() { + nodeName := n.VersionedPkg.Name + for _, searchName := range packages { + if nodeName == searchName { + list = append(list, n) + } + } + } + return +} + +func searchForSpec(graph *pkggraph.PkgGraph, specs []string) (list []*pkggraph.PkgNode) { + for _, n := range graph.AllPreferredRunNodes() { + nodeSpec := n.SpecName() + for _, searchSpec := range specs { + if nodeSpec == searchSpec { + list = append(list, n) + } + } + } + return +} + +func buildRequiresGraph(graphIn *pkggraph.PkgGraph, nodeList []*pkggraph.PkgNode) (graphOut *pkggraph.PkgGraph, root *pkggraph.PkgNode, err error) { + // Make a copy of the graph + newGraph, err := graphIn.DeepCopy() + if err != nil { + return + } + + // Add a goal node to all the things we care about + root = newGraph.AddMetaNode(nil, nodeList) + graphOut, err = newGraph.CreateSubGraph(root) + if err != nil { + return + } + + return +} + +func buildDependsOnGraph(graphIn *pkggraph.PkgGraph, nodeList []*pkggraph.PkgNode) (graphOut *pkggraph.PkgGraph, root *pkggraph.PkgNode, err error) { + // Make a copy of the graph + reversedGraph, err := graphIn.DeepCopy() + if err != nil { + return + } + + // Then reverse every edge in the graph + for _, edge := range graph.EdgesOf(reversedGraph.Edges()) { + reversedGraph.RemoveEdge(edge.From().ID(), edge.To().ID()) + reversedGraph.SetEdge(edge.ReversedEdge()) + } + + // Add a goal node to all the things we care about + root = reversedGraph.AddMetaNode(nil, nodeList) + graphOut, err = reversedGraph.CreateSubGraph(root) + if err != nil { + return + } + + return +} + +func formatNode(n *pkggraph.PkgNode, verbosity int) string { + switch verbosity { + case 1: + return n.SpecName() + case 2: + return filepath.Base(n.RpmPath) + case 3: + return fmt.Sprintf("'%s' from node '%s'", filepath.Base(n.RpmPath), n.FriendlyName()) + case 4: + return fmt.Sprintf("(%v)'%#v'", n.VersionedPkg, *n) + default: + logger.Log.Fatalf("Invalid verbosity level %v", verbosity) + } + return "" +} + +func isFilteredFile(path, filterFile string) bool { + if len(filterFile) > 0 { + if len(reservedFiles) == 0 { + reservedFileList, err := schedulerutils.ReadReservedFilesList(filterFile) + if err != nil { + logger.Log.Fatalf("Failed to load filter file (%s): %s", filterFile, err) + } + reservedFiles = sliceutils.SliceToSet[string](reservedFileList) + } + base := filepath.Base(path) + + return len(path) > 0 && reservedFiles[base] + } else { + return false + } +} + +func createSearch(g *pkggraph.PkgGraph, root *pkggraph.PkgNode) (t *treeSearch, err error) { + newSearch := &treeSearch{ + graph: g, + alreadyAdded: make(map[*pkggraph.PkgNode]bool), + filteredNodes: make(map[*pkggraph.PkgNode]bool), + normalNodes: make(map[*pkggraph.PkgNode]bool), + nodesVisited: 0, + } + + //Calculate the number of nodes we might visit: + subGraph, err := g.CreateSubGraph(root) + if err != nil { + err = fmt.Errorf("failed to calculate number of nodes:\n%w", err) + return + } + + //This is the worst case possible number of searche to make + possibleEdges := subGraph.Edges().Len() * subGraph.Nodes().Len() + newSearch.nodesTotal = possibleEdges + + return newSearch, nil +} + +func (t *treeSearch) FilteredNodes() (nodes []*pkggraph.PkgNode) { + nodes = []*pkggraph.PkgNode{} + for n := range t.filteredNodes { + nodes = append(nodes, n) + } + return nodes +} + +func (t *treeSearch) NonFilteredNodes() (nodes []*pkggraph.PkgNode) { + nodes = []*pkggraph.PkgNode{} + for n := range t.normalNodes { + nodes = append(nodes, n) + } + return nodes +} + +// Call this ever time a node is processed, will print an update ever 100 nodes +func (t *treeSearch) printProgress() { + if t.nodesVisited%10000 == 0 { + logger.Log.Infof("Scanned %d nodes", t.nodesVisited) + } + t.nodesVisited++ +} + +// Run a DFS and generate a string representation of the tree. Optionally ignore all branches that only container nodes in +// +// the filter list (ie given the toolchain manifest, only print those branches which container non-toolchain packages) +func (t *treeSearch) treeNodeToString(n *pkggraph.PkgNode, depth, maxDepth int, filter bool, filterFile string, verbosity int, generateStrings, printDuplicates bool, runtimeFilterLevel int) (lines []string, hasNonToolchain bool) { + t.printProgress() + // We only care about run nodes for the purposes of detecting toolchain files + hasNonToolchain = n.Type == pkggraph.TypeLocalBuild && !isFilteredFile(n.RpmPath, filterFile) + + if !printDuplicates && t.alreadyAdded[n] { + return []string{}, false + } else { + t.alreadyAdded[n] = true + } + + thisNode := formatNode(n, verbosity) + if isFilteredFile(n.RpmPath, filterFile) { + // Highlight nodes that are in the filter file list in red, and add them to the list + if generateStrings { + lines = append(lines, fmt.Sprintf(color.RedString("%s"), thisNode)) + } + if n.Type == pkggraph.TypeLocalRun { + // We only want to record run nodes for the purposes of listing packages in non-tree mode + t.filteredNodes[n] = true + } + } else { + // Non filtered files print normally + if generateStrings { + lines = append(lines, "__"+thisNode) + } + if n.Type == pkggraph.TypeLocalRun { + // We only want to record run nodes for the purposes of listing packages in non-tree mode + t.normalNodes[n] = true + } + } + + var childrenTreeNodes []treeNode + nodes := t.graph.From(n.ID()) + // Bail out early if we exceed max depth, maxDepth of -1 means no limit. + if depth < maxDepth || maxDepth == -1 { + var ( + childLines []string + childHasMissingToolchainPkg = false + haveDuplicatedEntry = false + ) + for nodes.Next() { + child := nodes.Node().(*pkggraph.PkgNode) + + // If we are only looking for runtime, skip build nodes (except for the root nodes: goal + each package node we care about) + if runtimeFilterLevel != -1 && depth > runtimeFilterLevel && child.Type == pkggraph.TypeLocalBuild { + continue + } + childLines, childHasMissingToolchainPkg = t.treeNodeToString(child, depth+1, maxDepth, filter, filterFile, verbosity, generateStrings, printDuplicates, runtimeFilterLevel) + hasNonToolchain = hasNonToolchain || childHasMissingToolchainPkg + + // A child will return an empty string list if it, and all its children, are either duplicates or have been filtered out + if len(childLines) > 0 { + tn := treeNode{lines: childLines} + childrenTreeNodes = append(childrenTreeNodes, tn) + } else { + haveDuplicatedEntry = true + } + } + + // If we have duplicated entires (ie empty strings), and we aren't removing all non-filtered entries, represent them with a '...' + // as the last entry instead of the node name. + if haveDuplicatedEntry && !filter { + childrenTreeNodes = append(childrenTreeNodes, treeNode{lines: []string{"..."}}) + } + + if len(childrenTreeNodes) > 0 && generateStrings { + firstN := childrenTreeNodes[:len(childrenTreeNodes)-1] + last := childrenTreeNodes[len(childrenTreeNodes)-1] + for _, tn := range firstN { + for _, l := range tn.lines { + lines = append(lines, " |"+l) + } + } + lines = append(lines, " |"+last.lines[0]) + for _, l := range last.lines[1:] { + lines = append(lines, " "+l) + } + } + } else { + lines = append(lines, " |-->") + // We are bailing out early, we don't know if this should be filtered, assume the worst + hasNonToolchain = true + } + + if !hasNonToolchain && filter && depth > 0 { + return []string{}, false + } + + return lines, hasNonToolchain +} + +func PrintSpecs(graph *pkggraph.PkgGraph, tree, filter bool, filterFile string, printDuplicates bool, verbosity, maxDepth, runtimeFilterLevel int, root *pkggraph.PkgNode) (err error) { + t, err := createSearch(graph, root) + if err != nil { + err = fmt.Errorf("failed to start search:\n%w", err) + return + } + // May as well use the tree searh to parse all the filtered packages etc, even if we are + // just printing a list + lines, _ := t.treeNodeToString(root, 0, maxDepth, filter, filterFile, verbosity, tree, printDuplicates, runtimeFilterLevel) + if tree { + for _, l := range lines { + fmt.Println(l) + } + } else { + results := make(map[string]bool) + if !filter { + // Only include toolchain packages if we aren't trying to find packages that have + // infiltrated the toolchain + for _, n := range t.FilteredNodes() { + results[formatNode(n, verbosity)] = true + } + } + // Always include normal nodes + for _, n := range t.NonFilteredNodes() { + results[formatNode(n, verbosity)] = true + } + + // Contert to list and sort + printLines := sliceutils.SetToSlice[string](results) + sort.Strings(printLines) + for _, l := range printLines { + fmt.Println(l) + } + } + return +} From 5c29c71e7fed14a39da0d37befdf8c97f3f227e4 Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Thu, 16 May 2024 15:39:39 -0700 Subject: [PATCH 22/51] Image Customizer: Add tests for BootCustomizer type. (#9128) --- .../docs/building/prerequisites-mariner.md | 1 + toolkit/docs/building/prerequisites-ubuntu.md | 1 + .../pkg/imagecustomizerlib/bootcustomizer.go | 12 +- .../imagecustomizerlib/bootcustomizer_test.go | 204 ++++++++++++++++++ .../testdata/bootcfgtests/2.0-default-grub | 22 ++ .../testdata/bootcfgtests/2.0-grub.cfg | 26 +++ .../testdata/bootcfgtests/3.0-default-grub | 22 ++ .../testdata/bootcfgtests/3.0-grub.cfg | 0 8 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 toolkit/tools/pkg/imagecustomizerlib/bootcustomizer_test.go create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/2.0-default-grub create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/2.0-grub.cfg create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/3.0-default-grub create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/3.0-grub.cfg diff --git a/toolkit/docs/building/prerequisites-mariner.md b/toolkit/docs/building/prerequisites-mariner.md index cbd032f042a..b72ef4f0bbf 100644 --- a/toolkit/docs/building/prerequisites-mariner.md +++ b/toolkit/docs/building/prerequisites-mariner.md @@ -13,6 +13,7 @@ sudo tdnf -y install \ binutils \ cdrkit \ curl \ + diffutils \ dosfstools \ gawk \ glibc-devel \ diff --git a/toolkit/docs/building/prerequisites-ubuntu.md b/toolkit/docs/building/prerequisites-ubuntu.md index e5fca148fe9..e4808896427 100644 --- a/toolkit/docs/building/prerequisites-ubuntu.md +++ b/toolkit/docs/building/prerequisites-ubuntu.md @@ -13,6 +13,7 @@ sudo apt-get update sudo apt -y install \ acl \ curl \ + diffutils \ gawk \ genisoimage \ git \ diff --git a/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go index ef5d3e4617d..4341e2ea47d 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go +++ b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go @@ -72,7 +72,7 @@ func (b *BootCustomizer) AddKernelCommandLine(extraCommandLine string) error { } // Gets the image's configured SELinux mode. -func (b *BootCustomizer) GetSELinuxMode(imageChroot *safechroot.Chroot) (imagecustomizerapi.SELinuxMode, error) { +func (b *BootCustomizer) getSELinuxModeFromGrub() (imagecustomizerapi.SELinuxMode, error) { var err error var args []grubConfigLinuxArg @@ -95,6 +95,16 @@ func (b *BootCustomizer) GetSELinuxMode(imageChroot *safechroot.Chroot) (imagecu return imagecustomizerapi.SELinuxModeDefault, err } + return selinuxMode, nil +} + +func (b *BootCustomizer) GetSELinuxMode(imageChroot *safechroot.Chroot) (imagecustomizerapi.SELinuxMode, error) { + // Get the SELinux mode from the kernel command-line args. + selinuxMode, err := b.getSELinuxModeFromGrub() + if err != nil { + return imagecustomizerapi.SELinuxModeDefault, err + } + if selinuxMode == imagecustomizerapi.SELinuxModeDefault { // Get the SELinux mode from the /etc/selinux/config file. selinuxMode, err = getSELinuxModeFromConfigFile(imageChroot) diff --git a/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer_test.go b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer_test.go new file mode 100644 index 00000000000..0cdf9a2e51d --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer_test.go @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerlib + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi" + "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/stretchr/testify/assert" +) + +const ( + sampleGrubCfg20Path = "bootcfgtests/2.0-grub.cfg" + sampleDefaultGrub20Path = "bootcfgtests/2.0-default-grub" + + sampleGrubCfg30Path = "bootcfgtests/3.0-grub.cfg" + sampleDefaultGrub30Path = "bootcfgtests/3.0-default-grub" +) + +func TestBootCustomizerAddKernelCommandLine20(t *testing.T) { + b := createBootCustomizerFor20(t) + err := b.AddKernelCommandLine("console=tty0 console=ttyS0") + assert.NoError(t, err) + + expectedGrubCfdDiff := `22c22 +< linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline $kernelopts +--- +> linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline console=tty0 console=ttyS0 $kernelopts +` + checkDiffs20(t, b, expectedGrubCfdDiff, "") +} + +func TestBootCustomizerAddKernelCommandLine30(t *testing.T) { + b := createBootCustomizerFor30(t) + err := b.AddKernelCommandLine("console=tty0 console=ttyS0") + assert.NoError(t, err) + + expectedDefaultGrubFileDiff := `6c6 +< GRUB_CMDLINE_LINUX_DEFAULT=" $kernelopts" +--- +> GRUB_CMDLINE_LINUX_DEFAULT=" console=tty0 console=ttyS0 \$kernelopts" +` + checkDiffs30(t, b, "", expectedDefaultGrubFileDiff) +} + +func TestBootCustomizerSELinuxMode20(t *testing.T) { + b := createBootCustomizerFor20(t) + selinuxMode, err := b.getSELinuxModeFromGrub() + assert.NoError(t, err) + assert.Equal(t, imagecustomizerapi.SELinuxModeDisabled, selinuxMode) + + err = b.UpdateSELinuxCommandLine(imagecustomizerapi.SELinuxModePermissive) + assert.NoError(t, err) + + selinuxMode, err = b.getSELinuxModeFromGrub() + assert.NoError(t, err) + assert.Equal(t, imagecustomizerapi.SELinuxModeDefault, selinuxMode) + + expectedGrubCfgDiff := `22c22 +< linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline $kernelopts +--- +> linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline security=selinux selinux=1 $kernelopts +` + checkDiffs20(t, b, expectedGrubCfgDiff, "") + + err = b.UpdateSELinuxCommandLine(imagecustomizerapi.SELinuxModeForceEnforcing) + assert.NoError(t, err) + + selinuxMode, err = b.getSELinuxModeFromGrub() + assert.NoError(t, err) + assert.Equal(t, imagecustomizerapi.SELinuxModeForceEnforcing, selinuxMode) + + expectedGrubCfgDiff = `22c22 +< linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline $kernelopts +--- +> linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline security=selinux selinux=1 enforcing=1 $kernelopts +` + checkDiffs20(t, b, expectedGrubCfgDiff, "") + + err = b.UpdateSELinuxCommandLine(imagecustomizerapi.SELinuxModeDisabled) + assert.NoError(t, err) + + selinuxMode, err = b.getSELinuxModeFromGrub() + assert.NoError(t, err) + assert.Equal(t, imagecustomizerapi.SELinuxModeDisabled, selinuxMode) + + expectedGrubCfgDiff = `22c22 +< linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline $kernelopts +--- +> linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline $kernelopts +` + checkDiffs20(t, b, expectedGrubCfgDiff, "") +} + +func TestBootCustomizerSELinuxMode30(t *testing.T) { + b := createBootCustomizerFor30(t) + selinuxMode, err := b.getSELinuxModeFromGrub() + assert.NoError(t, err) + assert.Equal(t, imagecustomizerapi.SELinuxModeDisabled, selinuxMode) + + err = b.UpdateSELinuxCommandLine(imagecustomizerapi.SELinuxModePermissive) + assert.NoError(t, err) + + selinuxMode, err = b.getSELinuxModeFromGrub() + assert.NoError(t, err) + assert.Equal(t, imagecustomizerapi.SELinuxModeDefault, selinuxMode) + + expectedDefaultGrubFileDiff := `5c5 +< GRUB_CMDLINE_LINUX=" rd.auto=1 net.ifnames=0 lockdown=integrity " +--- +> GRUB_CMDLINE_LINUX=" rd.auto=1 net.ifnames=0 lockdown=integrity security=selinux selinux=1 " +` + checkDiffs30(t, b, "", expectedDefaultGrubFileDiff) + + err = b.UpdateSELinuxCommandLine(imagecustomizerapi.SELinuxModeForceEnforcing) + assert.NoError(t, err) + + selinuxMode, err = b.getSELinuxModeFromGrub() + assert.NoError(t, err) + assert.Equal(t, imagecustomizerapi.SELinuxModeForceEnforcing, selinuxMode) + + expectedDefaultGrubFileDiff = `5c5 +< GRUB_CMDLINE_LINUX=" rd.auto=1 net.ifnames=0 lockdown=integrity " +--- +> GRUB_CMDLINE_LINUX=" rd.auto=1 net.ifnames=0 lockdown=integrity security=selinux selinux=1 enforcing=1 " +` + checkDiffs30(t, b, "", expectedDefaultGrubFileDiff) + + err = b.UpdateSELinuxCommandLine(imagecustomizerapi.SELinuxModeDisabled) + assert.NoError(t, err) + + selinuxMode, err = b.getSELinuxModeFromGrub() + assert.NoError(t, err) + assert.Equal(t, imagecustomizerapi.SELinuxModeDisabled, selinuxMode) + + expectedDefaultGrubFileDiff = `5c5 +< GRUB_CMDLINE_LINUX=" rd.auto=1 net.ifnames=0 lockdown=integrity " +--- +> GRUB_CMDLINE_LINUX=" rd.auto=1 net.ifnames=0 lockdown=integrity " +` + checkDiffs30(t, b, "", expectedDefaultGrubFileDiff) +} + +func checkDiffs20(t *testing.T, b *BootCustomizer, expectedGrubCfgDiff string, expectedDefaultGrubFileDiff string) { + checkDiffs(t, b, filepath.Join(testDir, sampleGrubCfg20Path), filepath.Join(testDir, sampleDefaultGrub20Path), + expectedGrubCfgDiff, expectedDefaultGrubFileDiff) +} + +func checkDiffs30(t *testing.T, b *BootCustomizer, expectedGrubCfgDiff string, expectedDefaultGrubFileDiff string) { + checkDiffs(t, b, filepath.Join(testDir, sampleGrubCfg30Path), filepath.Join(testDir, sampleDefaultGrub30Path), + expectedGrubCfgDiff, expectedDefaultGrubFileDiff) +} + +func checkDiffs(t *testing.T, b *BootCustomizer, originalGrubCfgPath string, originalDefaultGrubFilePath string, + expectedGrubCfgDiff string, expectedDefaultGrubFileDiff string, +) { + grubCfgDiff := calcDiff(t, originalGrubCfgPath, b.grubCfgContent) + defaultGrubFileDiff := calcDiff(t, originalDefaultGrubFilePath, b.defaultGrubFileContent) + + assert.Equal(t, expectedGrubCfgDiff, grubCfgDiff, "diff of grub.cfg file") + assert.Equal(t, expectedDefaultGrubFileDiff, defaultGrubFileDiff, "diff of /etc/default/grub file") +} + +func calcDiff(t *testing.T, oldPath string, newContent string) string { + diff, _, err := shell.ExecuteWithStdin(newContent, "diff", oldPath, "-") + if err != nil { + exitError, isExitError := err.(*exec.ExitError) + if assert.True(t, isExitError) { + assert.Equal(t, exitError.ExitCode(), 1) + } + } + return diff +} + +func createBootCustomizerFor20(t *testing.T) *BootCustomizer { + return createBootCustomizer(t, filepath.Join(testDir, sampleGrubCfg20Path), + filepath.Join(testDir, sampleDefaultGrub20Path), false) +} + +func createBootCustomizerFor30(t *testing.T) *BootCustomizer { + return createBootCustomizer(t, filepath.Join(testDir, sampleGrubCfg30Path), + filepath.Join(testDir, sampleDefaultGrub30Path), true) +} + +func createBootCustomizer(t *testing.T, sampleGrubCfgPath string, sampleDefaultGrubFilePath string, isGrubMkconfig bool, +) *BootCustomizer { + sampleGrubCfgContent, err := os.ReadFile(sampleGrubCfgPath) + assert.NoError(t, err, "failed to read sample grub.cfg file") + + sampleDefaultGrubFileContent, err := os.ReadFile(sampleDefaultGrubFilePath) + assert.NoError(t, err, "failed to read sample /etc/default/grub file") + + b := &BootCustomizer{ + grubCfgContent: string(sampleGrubCfgContent), + defaultGrubFileContent: string(sampleDefaultGrubFileContent), + isGrubMkconfig: isGrubMkconfig, + } + return b +} diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/2.0-default-grub b/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/2.0-default-grub new file mode 100644 index 00000000000..ec12b141cf3 --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/2.0-default-grub @@ -0,0 +1,22 @@ +GRUB_TIMEOUT=0 +GRUB_DISTRIBUTOR="AzureLinux" +GRUB_DISABLE_SUBMENU=y +GRUB_TERMINAL_OUTPUT="console" +GRUB_CMDLINE_LINUX=" rd.auto=1 init=/lib/systemd/systemd net.ifnames=0 plymouth.enable=0 systemd.legacy_systemd_cgroup_controller=yes systemd.unified_cgroup_hierarchy=0 lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 loglevel=3 " +GRUB_CMDLINE_LINUX_DEFAULT=" $kernelopts" + +# =============================notice=============================== +# IMPORTANT: package and feature-specific behaviors are defined in +# /etc/default/grub.d/*.cfg. The cfg files are sourced last +# before grub2-mkconfig is called and hence have higher precedence +# than this file's GRUB_CMDLINE_LINUX. The order as it appears in the +# Linux commandline is: +# - first GRUB_CMDLINE_LINUX +# - then /etc/default/grub.d/*.cfg +# - and finally GRUB_CMDLINE_LINUX_DEFAULT +# =============================notice=============================== +for x in /etc/default/grub.d/*.cfg ; do + if [ -e "${x}" ]; then + . "${x}" + fi +done diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/2.0-grub.cfg b/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/2.0-grub.cfg new file mode 100644 index 00000000000..46f78f26aa8 --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/2.0-grub.cfg @@ -0,0 +1,26 @@ +set timeout=0 +set bootprefix=/boot +search -n -u 33beac00-b378-4b0c-b0cb-d5dcebf2cf57 -s + +load_env -f $bootprefix/mariner.cfg +if [ -f $bootprefix/mariner-mshv.cfg ]; then + load_env -f $bootprefix/mariner-mshv.cfg +fi + +if [ -f $bootprefix/systemd.cfg ]; then + load_env -f $bootprefix/systemd.cfg +else + set systemd_cmdline=net.ifnames=0 +fi +if [ -f $bootprefix/grub2/grubenv ]; then + load_env -f $bootprefix/grub2/grubenv +fi + +set rootdevice=PARTUUID=c17c558b-068b-459c-92cb-f218d14b44a1 + +menuentry "CBL-Mariner" { + linux $bootprefix/$mariner_linux rd.auto=1 root=$rootdevice $mariner_cmdline lockdown=integrity sysctl.kernel.unprivileged_bpf_disabled=1 $systemd_cmdline $kernelopts + if [ -f $bootprefix/$mariner_initrd ]; then + initrd $bootprefix/$mariner_initrd + fi +} diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/3.0-default-grub b/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/3.0-default-grub new file mode 100644 index 00000000000..a753fa8b6d6 --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/3.0-default-grub @@ -0,0 +1,22 @@ +GRUB_TIMEOUT=0 +GRUB_DISTRIBUTOR="AzureLinux" +GRUB_DISABLE_SUBMENU=y +GRUB_TERMINAL_OUTPUT="console" +GRUB_CMDLINE_LINUX=" rd.auto=1 net.ifnames=0 lockdown=integrity " +GRUB_CMDLINE_LINUX_DEFAULT=" $kernelopts" + +# =============================notice=============================== +# IMPORTANT: package and feature-specific behaviors are defined in +# /etc/default/grub.d/*.cfg. The cfg files are sourced last +# before grub2-mkconfig is called and hence have higher precedence +# than this file's GRUB_CMDLINE_LINUX. The order as it appears in the +# Linux commandline is: +# - first GRUB_CMDLINE_LINUX +# - then /etc/default/grub.d/*.cfg +# - and finally GRUB_CMDLINE_LINUX_DEFAULT +# =============================notice=============================== +for x in /etc/default/grub.d/*.cfg ; do + if [ -e "${x}" ]; then + . "${x}" || grub_warn "Received non-zero exit code from ${x}." + fi +done diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/3.0-grub.cfg b/toolkit/tools/pkg/imagecustomizerlib/testdata/bootcfgtests/3.0-grub.cfg new file mode 100644 index 00000000000..e69de29bb2d From 286ed9b540060ccb486ee82aa95c5ade6120dbdb Mon Sep 17 00:00:00 2001 From: Pawel Winogrodzki Date: Thu, 16 May 2024 17:03:44 -0700 Subject: [PATCH 23/51] Fixing epoch matching in 'InstallPackageRegex'. (CP #9131) (#9133) --- toolkit/tools/internal/tdnf/tdnf.go | 6 +- toolkit/tools/internal/tdnf/tdnf_test.go | 81 ++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/toolkit/tools/internal/tdnf/tdnf.go b/toolkit/tools/internal/tdnf/tdnf.go index b1df3d9611a..f7c36bb0675 100644 --- a/toolkit/tools/internal/tdnf/tdnf.go +++ b/toolkit/tools/internal/tdnf/tdnf.go @@ -14,14 +14,14 @@ import ( var ( // Every valid line will be of the form: . // For: - // X aarch64 1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo + // X aarch64 5:1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo // // We'd get: // - package_name: X // - architecture: aarch64 - // - version: 1.1b.8_X-22~rc1 + // - version: 5:1.1b.8_X-22~rc1 // - dist: azl3 - InstallPackageRegex = regexp.MustCompile(`^\s*([[:alnum:]_.+-]+)\s+([[:alnum:]_+-]+)\s+([[:alnum:]._+~-]+)\.([[:alpha:]]+[[:digit:]]+)`) + InstallPackageRegex = regexp.MustCompile(`^\s*([[:alnum:]_.+-]+)\s+([[:alnum:]_+-]+)\s+((?:[[:digit:]]:)?[[:alnum:]._+~-]+)\.([[:alpha:]]+[[:digit:]]+)`) // Every valid line pair will be of the form: // -. : diff --git a/toolkit/tools/internal/tdnf/tdnf_test.go b/toolkit/tools/internal/tdnf/tdnf_test.go index 6c89d93dab2..ad4d348cec6 100644 --- a/toolkit/tools/internal/tdnf/tdnf_test.go +++ b/toolkit/tools/internal/tdnf/tdnf_test.go @@ -72,3 +72,84 @@ func TestGetMajorVersionFromString_RejectTrailingDot(t *testing.T) { _, err := getMajorVersionFromString(fullVersion) assert.Error(t, err) } + +func TestInstallPackageRegex_MatchesPackageName(t *testing.T) { + const line = "X aarch64 1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo" + + matches := InstallPackageRegex.FindStringSubmatch(line) + + assert.Len(t, matches, InstallMaxMatchLen) + assert.Equal(t, "X", matches[InstallPackageName]) +} + +func TestInstallPackageRegex_FailsForMissingPackageName(t *testing.T) { + const line = " aarch64 1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo" + + assert.False(t, InstallPackageRegex.MatchString(line)) +} + +func TestInstallPackageRegex_MatchesPackageArch(t *testing.T) { + const line = "X aarch64 1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo" + + matches := InstallPackageRegex.FindStringSubmatch(line) + + assert.Len(t, matches, InstallMaxMatchLen) + assert.Equal(t, "aarch64", matches[InstallPackageArch]) +} + +func TestInstallPackageRegex_FailsForMissingArch(t *testing.T) { + const line = "X 1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo" + + assert.False(t, InstallPackageRegex.MatchString(line)) +} + +func TestInstallPackageRegex_MatchesPackageVersionNoEpoch(t *testing.T) { + const line = "X aarch64 1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo" + + matches := InstallPackageRegex.FindStringSubmatch(line) + + assert.Len(t, matches, InstallMaxMatchLen) + assert.Equal(t, "1.1b.8_X-22~rc1", matches[InstallPackageVersion]) +} + +func TestInstallPackageRegex_MatchesPackageVersionWithEpoch(t *testing.T) { + const line = "X aarch64 5:1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo" + + matches := InstallPackageRegex.FindStringSubmatch(line) + + assert.Len(t, matches, InstallMaxMatchLen) + assert.Equal(t, "5:1.1b.8_X-22~rc1", matches[InstallPackageVersion]) +} + +func TestInstallPackageRegex_FailsForMissingVersion(t *testing.T) { + const line = "X aarch64 .azl3 fetcher-cloned-repo" + + assert.False(t, InstallPackageRegex.MatchString(line)) +} + +func TestInstallPackageRegex_MatchesPackageDist(t *testing.T) { + const line = "X aarch64 1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo" + + matches := InstallPackageRegex.FindStringSubmatch(line) + + assert.Len(t, matches, InstallMaxMatchLen) + assert.Equal(t, "azl3", matches[InstallPackageDist]) +} + +func TestInstallPackageRegex_FailsForMissingDist(t *testing.T) { + const line = "X aarch64 1.1b.8_X-22~rc1 fetcher-cloned-repo" + + assert.False(t, InstallPackageRegex.MatchString(line)) +} + +func TestInstallPackageRegex_MatchesRandomWhiteSpaces(t *testing.T) { + const line = "X aarch64 1.1b.8_X-22~rc1.azl3 fetcher-cloned-repo" + + assert.True(t, InstallPackageRegex.MatchString(line)) +} + +func TestInstallPackageRegex_DoesNotMatchInvalidLine(t *testing.T) { + const line = "Invalid line" + + assert.False(t, InstallPackageRegex.MatchString(line)) +} From 059e10df550a52326143a2c332a5acc060c41b23 Mon Sep 17 00:00:00 2001 From: Maxwell McKee <66395252+mamckee@users.noreply.github.com> Date: Fri, 17 May 2024 10:24:11 -0700 Subject: [PATCH 24/51] Update SymCrypt-OpenSSL to 1.4.3 (#9139) --- SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.signatures.json | 2 +- SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.spec | 6 +++++- cgmanifest.json | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.signatures.json b/SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.signatures.json index dc22f79377a..d6207a3d5cc 100644 --- a/SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.signatures.json +++ b/SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "SymCrypt-OpenSSL-1.4.2.tar.gz": "c48ecfac71507a5497bad6ac5df3cbf5d78339f7c4aa6fb382ed8a8e05053377" + "SymCrypt-OpenSSL-1.4.3.tar.gz": "9225dd28ff03ecface28df77617f22344e144817a9556d3df909484f5661004b" } } diff --git a/SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.spec b/SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.spec index 2e265b5553b..8459cc4b497 100644 --- a/SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.spec +++ b/SPECS/SymCrypt-OpenSSL/SymCrypt-OpenSSL.spec @@ -1,6 +1,6 @@ Summary: The SymCrypt engine for OpenSSL (SCOSSL) allows the use of OpenSSL with SymCrypt as the provider for core cryptographic operations Name: SymCrypt-OpenSSL -Version: 1.4.2 +Version: 1.4.3 Release: 1%{?dist} License: MIT Vendor: Microsoft Corporation @@ -67,6 +67,10 @@ install SymCryptProvider/symcrypt_prov.cnf %{buildroot}%{_sysconfdir}/pki/tls/sy %{_sysconfdir}/pki/tls/symcrypt_prov.cnf %changelog +* Thu May 16 2024 Maxwell Moyer-McKee - 1.4.3-1 +- Additional bugfixes for TLS connections +- Add variable length GCM IV support to the SymCrypt engine + * Thu Apr 25 2024 Maxwell Moyer-McKee - 1.4.2-1 - Support additional parameters in the SymCrypt provider required for TLS connections - Various bugfixes for TLS scenarios diff --git a/cgmanifest.json b/cgmanifest.json index c0e12265127..ef4f0b12a20 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -28076,8 +28076,8 @@ "type": "other", "other": { "name": "SymCrypt-OpenSSL", - "version": "1.4.2", - "downloadUrl": "https://github.com/microsoft/SymCrypt-OpenSSL/archive/v1.4.2.tar.gz" + "version": "1.4.3", + "downloadUrl": "https://github.com/microsoft/SymCrypt-OpenSSL/archive/v1.4.3.tar.gz" } } }, From a3c0281d7b5ebf431dbc32b8384de2c917c0effc Mon Sep 17 00:00:00 2001 From: Sam Meluch <109628994+sameluch@users.noreply.github.com> Date: Fri, 17 May 2024 10:42:50 -0700 Subject: [PATCH 25/51] Fix PTests for Python Packages Initially Missing iniconfig (#9106) --- .../ptest-python-3.12-fix.patch | 57 ++++++ SPECS/python-html5lib/python-html5lib.spec | 10 +- .../python-importlib-metadata.spec | 7 +- SPECS/python-mpmath/python-mpmath.spec | 6 +- SPECS/python-pluggy/python-pluggy.spec | 8 +- .../0001-Skip-tests-for-mariner.patch | 190 ++++++++++++++++++ SPECS/python-rich/ptest-warning.patch | 89 ++++++++ SPECS/python-rich/python-rich.spec | 22 +- SPECS/python-wcwidth/python-wcwidth.spec | 10 +- .../python-websocket-client.spec | 10 +- 10 files changed, 393 insertions(+), 16 deletions(-) create mode 100644 SPECS/python-html5lib/ptest-python-3.12-fix.patch create mode 100644 SPECS/python-rich/0001-Skip-tests-for-mariner.patch create mode 100644 SPECS/python-rich/ptest-warning.patch diff --git a/SPECS/python-html5lib/ptest-python-3.12-fix.patch b/SPECS/python-html5lib/ptest-python-3.12-fix.patch new file mode 100644 index 00000000000..f934ecbd254 --- /dev/null +++ b/SPECS/python-html5lib/ptest-python-3.12-fix.patch @@ -0,0 +1,57 @@ +From 4a87368b71090f1432df6302f178c4babfcec93f Mon Sep 17 00:00:00 2001 +From: Christian Clauss +Date: Wed, 10 Jan 2024 15:13:40 +0100 +Subject: [PATCH] GitHub Actions: python: ["3.8", "3.9", "3.10", "3.11", + "3.12"] (#575) + +--- + html5lib/tests/tokenizer.py | 4 +++- + html5lib/tests/tree_construction.py | 4 +++- + requirements-test.txt | 3 ++- + 3 files changed, 8 insertions(+), 3 deletions(-) + +diff --git a/html5lib/tests/tokenizer.py b/html5lib/tests/tokenizer.py +index cc9897a4..b49d2e6e 100644 +--- a/html5lib/tests/tokenizer.py ++++ b/html5lib/tests/tokenizer.py +@@ -246,7 +246,9 @@ def runtest(self): + def repr_failure(self, excinfo): + traceback = excinfo.traceback + ntraceback = traceback.cut(path=__file__) +- excinfo.traceback = ntraceback.filter() ++ pytest_ver = getattr(pytest, "version_tuple", ()) ++ filter_args = (excinfo,) if pytest_ver >= (7, 4, 0) else () ++ excinfo.traceback = ntraceback.filter(*filter_args) + + return excinfo.getrepr(funcargs=True, + showlocals=False, +diff --git a/html5lib/tests/tree_construction.py b/html5lib/tests/tree_construction.py +index fb0657bf..363b48c2 100644 +--- a/html5lib/tests/tree_construction.py ++++ b/html5lib/tests/tree_construction.py +@@ -135,7 +135,9 @@ def runtest(self): + def repr_failure(self, excinfo): + traceback = excinfo.traceback + ntraceback = traceback.cut(path=__file__) +- excinfo.traceback = ntraceback.filter() ++ pytest_ver = getattr(pytest, "version_tuple", ()) ++ filter_args = (excinfo,) if pytest_ver >= (7, 4, 0) else () ++ excinfo.traceback = ntraceback.filter(*filter_args) + + return excinfo.getrepr(funcargs=True, + showlocals=False, +diff --git a/requirements-test.txt b/requirements-test.txt +index 27866e59..39913ee4 100644 +--- a/requirements-test.txt ++++ b/requirements-test.txt +@@ -3,8 +3,9 @@ + tox>=3.15.1,<4 + flake8>=3.8.1,<3.9 + pytest>=4.6.10,<5 ; python_version < '3' +-pytest>=5.4.2,<7 ; python_version >= '3' ++pytest>=5.4.2,<8 ; python_version >= '3' + coverage>=5.1,<6 + pytest-expect>=1.1.0,<2 + mock>=3.0.5,<4 ; python_version < '3.6' + mock>=4.0.2,<5 ; python_version >= '3.6' ++setuptools; python_version >= '3.12' diff --git a/SPECS/python-html5lib/python-html5lib.spec b/SPECS/python-html5lib/python-html5lib.spec index 842fa26e69f..cc60077d479 100644 --- a/SPECS/python-html5lib/python-html5lib.spec +++ b/SPECS/python-html5lib/python-html5lib.spec @@ -1,7 +1,7 @@ Summary: A python based HTML parser/tokenizer Name: python-html5lib Version: 1.1 -Release: 10%{?dist} +Release: 11%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -9,6 +9,8 @@ URL: https://github.com/html5lib/html5lib-python Source: %{pypi_source html5lib} # Fix compatibility with pytest 6 Patch0: %{url}/pull/506.patch +# fix compatability with python 3.12 +Patch1: ptest-python-3.12-fix.patch BuildArch: noarch @@ -20,6 +22,7 @@ BuildRequires: python3-wheel BuildRequires: python3-atomicwrites BuildRequires: python3-attrs BuildRequires: python3-docutils +BuildRequires: python3-more-itertools BuildRequires: python3-pluggy BuildRequires: python3-pygments BuildRequires: python3-pytest @@ -59,7 +62,7 @@ sed -i 's/from mock import/from unittest.mock import/' html5lib/tests/test_meta. %pyproject_save_files html5lib %check -pip3 install more-itertools umsgpack webencodings +pip3 install umsgpack webencodings iniconfig # Disabling broken tests, see: https://github.com/html5lib/html5lib-python/issues/433 %pytest -k "not test_parser_encoding and not test_prescan_encoding" @@ -67,6 +70,9 @@ pip3 install more-itertools umsgpack webencodings %doc CHANGES.rst README.rst %changelog +* Mon May 13 2024 Sam Meluch - 1.1-11 +- Add missing iniconfig dependency to check section + * Mon Mar 04 2024 Andrew Phelps - 1.1-10 - Correct usage of %%pyproject_extras_subpkg macro diff --git a/SPECS/python-importlib-metadata/python-importlib-metadata.spec b/SPECS/python-importlib-metadata/python-importlib-metadata.spec index 14982efd66f..398460ec7a7 100644 --- a/SPECS/python-importlib-metadata/python-importlib-metadata.spec +++ b/SPECS/python-importlib-metadata/python-importlib-metadata.spec @@ -1,7 +1,7 @@ Summary: Library to access the metadata for a Python package Name: python-importlib-metadata Version: 6.8.0 -Release: 2%{?dist} +Release: 3%{?dist} License: ASL 2.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -60,7 +60,7 @@ Python versions. %pyproject_save_files importlib_metadata %check -pip3 install pyfakefs more-itertools +pip3 install pyfakefs more-itertools iniconfig rm -rf .pyproject-builddir # Ignored file uses pytest_perf not available in Mariner # test_find_local tries to install setuptools from PyPI @@ -71,6 +71,9 @@ rm -rf .pyproject-builddir %doc README.rst %changelog +* Mon May 13 2024 Sam Meluch - 6.8.0-3 +- Add missing iniconfig dependency to check section + * Fri Mar 01 2024 Andrew Phelps - 6.8.0-2 - Add BR for python-setuptools_scm diff --git a/SPECS/python-mpmath/python-mpmath.spec b/SPECS/python-mpmath/python-mpmath.spec index 6deeb922f9b..6424da1a8e0 100644 --- a/SPECS/python-mpmath/python-mpmath.spec +++ b/SPECS/python-mpmath/python-mpmath.spec @@ -11,7 +11,7 @@ use gmpy to speed up high precision operations.} Summary: A pure Python library for multiprecision floating-point arithmetic Name: python-mpmath Version: 1.3.0 -Release: 3%{?dist} +Release: 4%{?dist} License: BSD Vendor: Microsoft Corporation Distribution: Azure Linux @@ -73,6 +73,7 @@ sed -i -r 's/use_scm_version=True/version="%{version}"/' setup.py %py3_install %check +pip3 install iniconfig cd build/lib/mpmath/tests/ xvfb-run -a pytest -v @@ -83,6 +84,9 @@ xvfb-run -a pytest -v %{python3_sitelib}/mpmath-%{version}-*.egg-info %changelog +* Mon May 13 2024 Sam Meluch - 1.3.0-4 +- Add missing iniconfig dependency to check section + * Thu Apr 06 2023 Riken Maharjan - 1.3.0-1 - Initial CBL-Mariner import from Fedora 38 (license: MIT) - License Verified diff --git a/SPECS/python-pluggy/python-pluggy.spec b/SPECS/python-pluggy/python-pluggy.spec index a46ddd8f741..a85e1c53378 100644 --- a/SPECS/python-pluggy/python-pluggy.spec +++ b/SPECS/python-pluggy/python-pluggy.spec @@ -5,7 +5,7 @@ The plugin manager stripped of pytest specific details. Summary: The plugin manager stripped of pytest specific details Name: python-pluggy Version: 1.3.0 -Release: 1%{?dist} +Release: 2%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -18,6 +18,7 @@ BuildRequires: python3-setuptools_scm %if 0%{?with_check} BuildRequires: python3-atomicwrites BuildRequires: python3-attrs +BuildRequires: python3-more-itertools BuildRequires: python3-pytest %endif @@ -39,7 +40,7 @@ The plugin manager stripped of pytest specific details. %py3_install %check -pip3 install more-itertools +pip3 install iniconfig # TODO investigate test_load_setuptools_instantiation failure PYTHONPATH=%{buildroot}%{python3_sitelib} python3 -m pytest testing -k "not test_load_setuptools_instantiation" @@ -51,6 +52,9 @@ PYTHONPATH=%{buildroot}%{python3_sitelib} python3 -m pytest testing -k "not test %license LICENSE %changelog +* Mon May 13 2024 Sam Meluch - 1.3.0-2 +- Add missing iniconfig dependency to check section + * Tue Jan 23 2024 Andrew Phelps - 1.3.0-1 - Upgrade to version 1.3.0 diff --git a/SPECS/python-rich/0001-Skip-tests-for-mariner.patch b/SPECS/python-rich/0001-Skip-tests-for-mariner.patch new file mode 100644 index 00000000000..0a4a8297649 --- /dev/null +++ b/SPECS/python-rich/0001-Skip-tests-for-mariner.patch @@ -0,0 +1,190 @@ +From 2aa13ad0ce14a716470996bfa18d9976156d3a22 Mon Sep 17 00:00:00 2001 +From: Sam Meluch +Date: Tue, 14 May 2024 17:03:13 -0700 +Subject: [PATCH] Skip tests for mariner + +--- + tests/test_card.py | 13 +++++++++++++ + tests/test_markdown.py | 13 +++++++++++++ + tests/test_markdown_no_hyperlinks.py | 13 +++++++++++++ + tests/test_syntax.py | 9 ++++++++- + 4 files changed, 47 insertions(+), 1 deletion(-) + +diff --git a/tests/test_card.py b/tests/test_card.py +index d578ec89..794fd5b1 100644 +--- a/tests/test_card.py ++++ b/tests/test_card.py +@@ -1,11 +1,23 @@ + import io + import re ++import sys ++import pytest ++ ++from packaging.version import Version + + from rich.__main__ import make_test_card + from rich.console import Console, RenderableType + + from ._card_render import expected + ++if sys.version_info >= (3, 8): ++ from importlib.metadata import Distribution ++else: ++ from importlib_metadata import Distribution ++ ++PYGMENTS_VERSION = Distribution.from_name("pygments").version ++OLD_PYGMENTS = Version(PYGMENTS_VERSION) <= Version("2.13.0") ++ + re_link_ids = re.compile(r"id=[\d\.\-]*?;.*?\x1b") + + +@@ -26,6 +38,7 @@ def render(renderable: RenderableType) -> str: + return output + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_card_render(): + card = make_test_card() + result = render(card) +diff --git a/tests/test_markdown.py b/tests/test_markdown.py +index 710436eb..43a53895 100644 +--- a/tests/test_markdown.py ++++ b/tests/test_markdown.py +@@ -71,10 +71,22 @@ foobar + + import io + import re ++import sys ++import pytest ++ ++from packaging.version import Version + + from rich.console import Console, RenderableType + from rich.markdown import Markdown + ++if sys.version_info >= (3, 8): ++ from importlib.metadata import Distribution ++else: ++ from importlib_metadata import Distribution ++ ++PYGMENTS_VERSION = Distribution.from_name("pygments").version ++OLD_PYGMENTS = Version(PYGMENTS_VERSION) <= Version("2.13.0") ++ + re_link_ids = re.compile(r"id=[\d\.\-]*?;.*?\x1b") + + +@@ -96,6 +108,7 @@ def render(renderable: RenderableType) -> str: + return output + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_markdown_render(): + markdown = Markdown(MARKDOWN) + rendered_markdown = render(markdown) +diff --git a/tests/test_markdown_no_hyperlinks.py b/tests/test_markdown_no_hyperlinks.py +index 4c5ff66f..289806aa 100644 +--- a/tests/test_markdown_no_hyperlinks.py ++++ b/tests/test_markdown_no_hyperlinks.py +@@ -65,10 +65,22 @@ foobar + + import io + import re ++import sys ++import pytest ++ ++from packaging.version import Version + + from rich.console import Console, RenderableType + from rich.markdown import Markdown + ++if sys.version_info >= (3, 8): ++ from importlib.metadata import Distribution ++else: ++ from importlib_metadata import Distribution ++ ++PYGMENTS_VERSION = Distribution.from_name("pygments").version ++OLD_PYGMENTS = Version(PYGMENTS_VERSION) <= Version("2.13.0") ++ + re_link_ids = re.compile(r"id=[\d\.\-]*?;.*?\x1b") + + +@@ -89,6 +101,7 @@ def render(renderable: RenderableType) -> str: + return output + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_markdown_render(): + markdown = Markdown(MARKDOWN, hyperlinks=False) + rendered_markdown = render(markdown) +diff --git a/tests/test_syntax.py b/tests/test_syntax.py +index 37cc293e..cbf9b93a 100644 +--- a/tests/test_syntax.py ++++ b/tests/test_syntax.py +@@ -5,6 +5,7 @@ import tempfile + + import pytest + from pygments.lexers import PythonLexer ++from packaging.version import Version + + from rich.measure import Measurement + from rich.panel import Panel +@@ -26,7 +27,7 @@ else: + from importlib_metadata import Distribution + + PYGMENTS_VERSION = Distribution.from_name("pygments").version +-OLD_PYGMENTS = PYGMENTS_VERSION == "2.13.0" ++OLD_PYGMENTS = Version(PYGMENTS_VERSION) <= Version("2.13.0") + + CODE = '''\ + def loop_first_last(values: Iterable[T]) -> Iterable[Tuple[bool, bool, T]]: +@@ -57,6 +58,7 @@ def test_blank_lines(): + ) + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_python_render(): + syntax = Panel.fit( + Syntax( +@@ -76,6 +78,7 @@ def test_python_render(): + assert rendered_syntax == expected + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_python_render_simple(): + syntax = Syntax( + CODE, +@@ -91,6 +94,7 @@ def test_python_render_simple(): + assert rendered_syntax == expected + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_python_render_simple_passing_lexer_instance(): + syntax = Syntax( + CODE, +@@ -141,6 +145,7 @@ def test_python_render_line_range_indent_guides(): + assert rendered_syntax == expected + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_python_render_indent_guides(): + syntax = Panel.fit( + Syntax( +@@ -234,6 +239,7 @@ def test_get_style_for_token(): + assert syntax._get_line_numbers_color() == Color.default() + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_option_no_wrap(): + syntax = Syntax( + CODE, +@@ -251,6 +257,7 @@ def test_option_no_wrap(): + assert rendered_syntax == expected + + ++@pytest.mark.skipif(OLD_PYGMENTS, reason="Pygments changed their tokenizer") + def test_syntax_highlight_ranges(): + syntax = Syntax( + CODE, +-- +2.34.1 + diff --git a/SPECS/python-rich/ptest-warning.patch b/SPECS/python-rich/ptest-warning.patch new file mode 100644 index 00000000000..5b9814844b8 --- /dev/null +++ b/SPECS/python-rich/ptest-warning.patch @@ -0,0 +1,89 @@ +From eb357884ad7fc677854396e107a1652f3c8d4aa3 Mon Sep 17 00:00:00 2001 +From: Ofek Lev +Date: Wed, 1 May 2024 09:18:57 -0400 +Subject: [PATCH] Fix warning in test suite + +--- + docs/requirements.txt | 1 + + docs/source/conf.py | 9 +++++++-- + pyproject.toml | 1 + + tests/test_syntax.py | 8 ++++++-- + 4 files changed, 15 insertions(+), 4 deletions(-) + +diff --git a/docs/requirements.txt b/docs/requirements.txt +index 4e029845b..4b7b608ef 100644 +--- a/docs/requirements.txt ++++ b/docs/requirements.txt +@@ -2,3 +2,4 @@ alabaster==0.7.12 + Sphinx==5.1.1 + sphinx-rtd-theme==1.0.0 + sphinx-copybutton==0.5.1 ++importlib-metadata; python_version < '3.8' +diff --git a/docs/source/conf.py b/docs/source/conf.py +index d1078d61e..451a13456 100644 +--- a/docs/source/conf.py ++++ b/docs/source/conf.py +@@ -17,10 +17,15 @@ + + # -- Project information ----------------------------------------------------- + ++import sys + +-import pkg_resources + import sphinx_rtd_theme + ++if sys.version_info >= (3, 8): ++ from importlib.metadata import Distribution ++else: ++ from importlib_metadata import Distribution ++ + html_theme = "sphinx_rtd_theme" + + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +@@ -30,7 +35,7 @@ + author = "Will McGugan" + + # The full version, including alpha/beta/rc tags +-release = pkg_resources.get_distribution("rich").version ++release = Distribution.from_name("rich").version + + + # -- General configuration --------------------------------------------------- +diff --git a/pyproject.toml b/pyproject.toml +index 0e9a3b74e..e589ba9dc 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -44,6 +44,7 @@ pytest-cov = "^3.0.0" + attrs = "^21.4.0" + pre-commit = "^2.17.0" + asv = "^0.5.1" ++importlib-metadata = { version = "*", python = "<3.8" } + + [build-system] + requires = ["poetry-core>=1.0.0"] +diff --git a/tests/test_syntax.py b/tests/test_syntax.py +index f3f0f41fe..37cc293eb 100644 +--- a/tests/test_syntax.py ++++ b/tests/test_syntax.py +@@ -3,7 +3,6 @@ + import sys + import tempfile + +-import pkg_resources + import pytest + from pygments.lexers import PythonLexer + +@@ -21,7 +20,12 @@ + + from .render import render + +-PYGMENTS_VERSION = pkg_resources.get_distribution("pygments").version ++if sys.version_info >= (3, 8): ++ from importlib.metadata import Distribution ++else: ++ from importlib_metadata import Distribution ++ ++PYGMENTS_VERSION = Distribution.from_name("pygments").version + OLD_PYGMENTS = PYGMENTS_VERSION == "2.13.0" + + CODE = '''\ diff --git a/SPECS/python-rich/python-rich.spec b/SPECS/python-rich/python-rich.spec index 9e3e52d991e..cb88ec2ea6a 100644 --- a/SPECS/python-rich/python-rich.spec +++ b/SPECS/python-rich/python-rich.spec @@ -1,6 +1,6 @@ Name: python-rich Version: 13.7.1 -Release: 3%{?dist} +Release: 4%{?dist} Summary: Render rich text and beautiful formatting in the terminal Vendor: Microsoft Corporation Distribution: Azure Linux @@ -10,7 +10,15 @@ Source0: %{url}/archive/v%{version}/rich-%{version}.tar.gz BuildArch: noarch -Patch0: 3229.patch +Patch0: 3229.patch +# ptest warning patch for test_syntax causing failures, can be +# removed when upgraded to a version containing the following: +# https://github.com/Textualize/rich/commit/027a4727a5b8f8407109c01df5e24604352bbe50 +Patch1: ptest-warning.patch +# This patch modifies the tests to set the OLD_PYGMENTS version +# to the current one in Azure Linux, 2.5.2. Once python-pygments +# version is updated >= 2.14.0, this patch can be removed. +Patch2: 0001-Skip-tests-for-mariner.patch BuildRequires: pyproject-rpm-macros BuildRequires: python3-attrs @@ -19,6 +27,10 @@ BuildRequires: python3-pip BuildRequires: python3-pytest BuildRequires: python3-poetry BuildRequires: python3-setuptools +%if 0%{?with_check} +BuildRequires: python3-markdown-it-py +BuildRequires: python3-packaging +%endif %description Rich is a Python library for rich text and beautiful formatting in the terminal. @@ -51,6 +63,7 @@ code, tracebacks, and more — out of the box. %check # add below to make sure initial build will catch runtime import errors +pip3 install iniconfig %pyproject_check_import %pytest -vv @@ -59,10 +72,13 @@ code, tracebacks, and more — out of the box. %doc README.md %changelog +* Mon May 13 2024 Sam Meluch - 13.7.1-4 +- Add missing iniconfig dependency to check section + * Fri May 10 2024 Riken Maharjan - 13.7.1-3 - Fix pygments name in Requires. -* Mon Mar 28 2024 Riken Maharjan - 13.7.1-2 +* Thu Mar 28 2024 Riken Maharjan - 13.7.1-2 - Initial Azure Linux import from Fedora 41 (license: MIT). - License verified. diff --git a/SPECS/python-wcwidth/python-wcwidth.spec b/SPECS/python-wcwidth/python-wcwidth.spec index 7569918eac7..85b952d1c1a 100644 --- a/SPECS/python-wcwidth/python-wcwidth.spec +++ b/SPECS/python-wcwidth/python-wcwidth.spec @@ -1,7 +1,7 @@ Summary: Measures number of Terminal column cells of wide-character codes. Name: python-wcwidth Version: 0.2.6 -Release: 1%{?dist} +Release: 2%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -24,7 +24,9 @@ BuildRequires: python3-xml BuildRequires: python3-atomicwrites BuildRequires: python3-attrs BuildRequires: python3-coverage +BuildRequires: python3-more-itertools BuildRequires: python3-pip +BuildRequires: python3-pluggy BuildRequires: python3-pytest BuildRequires: python3-pytest-cov BuildRequires: python3-six @@ -46,8 +48,7 @@ This Library is mainly for those implementing a Terminal Emulator, or programs t %check pip3 install \ - more-itertools \ - pluggy + iniconfig # note tox should have been preferred but unfortunately tox.ini is set to only support python up to 3.8 => no tests will then be executed # => stick with pytest which has a problem with test_package_version (version is correct but test stil throw an error) pytest3 -vv tests -k "not test_package_version" @@ -58,6 +59,9 @@ pytest3 -vv tests -k "not test_package_version" %{python3_sitelib}/* %changelog +* Mon May 13 2024 Sam Meluch - 0.2.6-2 +- Add missing iniconfig dependency to check section + * Fri Oct 27 2023 CBL-Mariner Servicing Account - 0.2.6-1 - Auto-upgrade to 0.2.6 - Azure Linux 3.0 - package upgrades diff --git a/SPECS/python-websocket-client/python-websocket-client.spec b/SPECS/python-websocket-client/python-websocket-client.spec index a51bd5632af..766802bba8c 100644 --- a/SPECS/python-websocket-client/python-websocket-client.spec +++ b/SPECS/python-websocket-client/python-websocket-client.spec @@ -1,7 +1,7 @@ Summary: WebSocket client for python Name: python-websocket-client Version: 1.6.2 -Release: 1%{?dist} +Release: 2%{?dist} License: ASL 2.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -25,7 +25,9 @@ BuildRequires: openssl-devel BuildRequires: python3-atomicwrites BuildRequires: python3-attrs BuildRequires: python3-coverage +BuildRequires: python3-more-itertools BuildRequires: python3-pip +BuildRequires: python3-pluggy BuildRequires: python3-pytest BuildRequires: python3-pytest-cov BuildRequires: python3-six @@ -45,8 +47,7 @@ WebSocket client for python3 %check pip3 install \ - more-itertools \ - pluggy + iniconfig # do not execute 'echo-server' test since it requires python websockets # which do not work well from a chroot pytest3 -vv websocket/tests -k "not echo-server" @@ -58,6 +59,9 @@ pytest3 -vv websocket/tests -k "not echo-server" %{_bindir}/wsdump %changelog +* Mon May 13 2024 Sam Meluch - 1.6.2-2 +- Add missing iniconfig dependency to check section + * Fri Oct 27 2023 CBL-Mariner Servicing Account - 1.6.2-1 - Auto-upgrade to 1.6.2 - Azure Linux 3.0 - package upgrades From 34daae6889df810d9e9b95176ad2be1daf5c2302 Mon Sep 17 00:00:00 2001 From: Sam Meluch <109628994+sameluch@users.noreply.github.com> Date: Fri, 17 May 2024 10:57:39 -0700 Subject: [PATCH 26/51] Fix PTest for python-repoze-lru (#9123) Co-authored-by: Daniel McIlvaney --- SPECS/python-repoze-lru/python-repoze-lru.spec | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/SPECS/python-repoze-lru/python-repoze-lru.spec b/SPECS/python-repoze-lru/python-repoze-lru.spec index a8a84eea186..817566cb833 100644 --- a/SPECS/python-repoze-lru/python-repoze-lru.spec +++ b/SPECS/python-repoze-lru/python-repoze-lru.spec @@ -1,12 +1,10 @@ -%{!?python3_sitelib: %define python3_sitelib %(python3 -c "from distutils.sysconfig import get_python_lib;print(get_python_lib())")} - %define pkgname repoze-lru %define pypiname repoze.lru Summary: A tiny LRU cache implementation and decorator Name: python-%{pkgname} Version: 0.7 -Release: 5%{?dist} +Release: 6%{?dist} License: BSD URL: https://github.com/repoze/repoze.lru Vendor: Microsoft Corporation @@ -27,7 +25,7 @@ BuildRequires: python3-devel BuildRequires: python3-xml BuildRequires: python3-setuptools Requires: python3 -%if %{with check} +%if 0%{?with_check} BuildRequires: python3-pip %endif @@ -44,8 +42,7 @@ python3 setup.py build python3 setup.py install --root=%{buildroot} %check -pip3 install tox -LANG=en_US.UTF-8 tox -e py%{python3_version_nodots} +python3 setup.py test %files -n python3-%{pkgname} %license LICENSE.txt @@ -53,6 +50,9 @@ LANG=en_US.UTF-8 tox -e py%{python3_version_nodots} %{python3_sitelib}/* %changelog +* Wed May 15 2024 Sam Meluch - 0.7-6 +- fix with_check macro, remove sitelibs redef to fix tests + * Wed Apr 27 2022 Pawel Winogrodzki - 0.7-5 - Updating source URL. From cae187699f660a9407bc96e8800d7fe84e4882d6 Mon Sep 17 00:00:00 2001 From: nicolas guibourge Date: Fri, 17 May 2024 17:23:43 -0400 Subject: [PATCH 27/51] move src tarball blob to AME - 3.0-dev branch (#8916) Co-authored-by: CBL-Mariner Servicing Account --- CONTRIBUTING.md | 2 +- .../azurelinux-rpm-macros.signatures.json | 2 +- .../azurelinux-rpm-macros.spec | 5 +- SPECS/azurelinux-rpm-macros/macros | 2 +- SPECS/maven/maven_build_caches.sh | 2 +- cgmanifest.json | 138 +++++++++--------- toolkit/Makefile | 2 +- toolkit/docs/building/add-package.md | 2 +- toolkit/docs/building/building.md | 4 +- .../manifests/package/pkggen_core_aarch64.txt | 4 +- .../manifests/package/pkggen_core_x86_64.txt | 4 +- .../manifests/package/toolchain_aarch64.txt | 4 +- .../manifests/package/toolchain_x86_64.txt | 4 +- toolkit/scripts/build_cargo_cache.sh | 2 +- toolkit/scripts/build_go_vendor_cache.sh | 2 +- 15 files changed, 91 insertions(+), 88 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82919472972..44edc638d13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,7 +155,7 @@ When creating your PR, please ensure the following: * Package tests (%check section) have been verified with RUN_CHECK=y for existing SPEC files, or added to new SPEC files. When running the check section, results will not fail a build. Check the logs for the results of this section. -* All package sources are available. The sources are either in the source server or local `SPECS` folder (`SPECS//SOURCES` or `SPECS/`). While it is possible to build packages with all sources inside the repo, our policy is generally to have the source compressed and placed on the source server. Uploading to the source server can only be accomplished by an Azure Linux developer. Please request help in your PR for uploading your sources to the source server. To check the source server see [https://cblmarinerstorage.blob.core.windows.net/sources/core/< source tar >]. +* All package sources are available. The sources are either in the source server or local `SPECS` folder (`SPECS//SOURCES` or `SPECS/`). While it is possible to build packages with all sources inside the repo, our policy is generally to have the source compressed and placed on the source server. Uploading to the source server can only be accomplished by an Azure Linux developer. Please request help in your PR for uploading your sources to the source server. To check the source server see [https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/< source tar >]. * cgmanifest files are up-to-date and alphabetically sorted. The cgmanifest files are used to record all package sources. They include the following files: diff --git a/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.signatures.json b/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.signatures.json index c26c0d7ab75..39f7255b8ca 100644 --- a/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.signatures.json +++ b/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.signatures.json @@ -14,7 +14,7 @@ "generate-package-note.py": "bd76a8e88a1356fed74863c38e5cf6a20c1c26426ac94ba21dd172578e8ca2a2", "gpgverify": "db0e050f56b694497d70603a6f5c17dd60ddbcf7cee670616851cd389f6767c4", "import_all_modules.py": "543d8c90e7db902e3b4587b7c488419e2f20a6d8b34174b7303dcd70794d8fc7", - "macros": "a5cde7ca92f0552e15620a94dcc800db0316876239d882caa87fa2c894441e19", + "macros": "d94e2ee84806c5cb996b810137e0498c2accd5ccc73cd5aff4cfc0235d760ecc", "macros.check": "79367176c3c7d10c0158b6e5d881e0fc3c8fd50c5957dad2f097c2d4a37833e7", "macros.fonts": "f52edc646414c5dd0f5f4cdd570f2f9dbe6fb97d4f0db360908deb56d96492f8", "macros.forge": "52cdffd48217cafe33afe47c625cd6aac6460848cd76de66fe61031c26beeaf4", diff --git a/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.spec b/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.spec index 398f1c6cf48..75256108e65 100644 --- a/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.spec +++ b/SPECS/azurelinux-rpm-macros/azurelinux-rpm-macros.spec @@ -7,7 +7,7 @@ Summary: Azure Linux specific rpm macro files Name: azurelinux-rpm-macros Version: %{azl}.0 -Release: 3%{?dist} +Release: 4%{?dist} License: GPL+ AND MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -141,6 +141,9 @@ install -p -m 644 -t %{buildroot}%{rcluadir}/srpm python.lua %{_rpmconfigdir}/macros.d/macros.check %changelog +* Fri May 17 2024 Nicolas Guibourge - 3.0-4 +- Move src tarball location to AME + * Fri May 11 2024 Chris Co - 3.0-3 - Add macros to allow enabling frame pointers on x86_64 and aarch64 diff --git a/SPECS/azurelinux-rpm-macros/macros b/SPECS/azurelinux-rpm-macros/macros index 8270e594e89..85b3941af31 100644 --- a/SPECS/azurelinux-rpm-macros/macros +++ b/SPECS/azurelinux-rpm-macros/macros @@ -262,4 +262,4 @@ end # Azure Linux's sources storage URL. # NOTE: only allowed to be used for source URLs when original, upstream source cannot be found. -%_distro_sources_url https://cblmarinerstorage.blob.core.windows.net/sources/core +%_distro_sources_url https://azurelinuxsrcstorage.blob.core.windows.net/sources/core diff --git a/SPECS/maven/maven_build_caches.sh b/SPECS/maven/maven_build_caches.sh index 8820c766016..487dbbb2119 100644 --- a/SPECS/maven/maven_build_caches.sh +++ b/SPECS/maven/maven_build_caches.sh @@ -45,7 +45,7 @@ BUILDARCH=$(rpm --eval '%{_arch}') SOURCEURL="https://archive.apache.org/dist/maven/maven-3/${VERSION}/source/apache-maven-${VERSION}-src.tar.gz" # Maven binary dependency to download from pmc. # NOTE: Version IN THIS IS HARDCODED. -MAVENBINARY="https://cblmarinerstorage.blob.core.windows.net/sources/core/maven-3.8.7-3.cm2.${BUILDARCH}.rpm" +MAVENBINARY="https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/maven-3.8.7-3.cm2.${BUILDARCH}.rpm" maven_m2_cache_tarball_name="apache-maven-${VERSION}-m2.tar.gz" maven_licenses_tarball_name="apache-maven-${VERSION}-licenses.tar.gz" diff --git a/cgmanifest.json b/cgmanifest.json index ef4f0b12a20..d72e4117387 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -256,7 +256,7 @@ "other": { "name": "antlr", "version": "2.7.7", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/antlr-2.7.7.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/antlr-2.7.7.tar.bz2" } } }, @@ -266,7 +266,7 @@ "other": { "name": "aopalliance", "version": "1.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/aopalliance-src.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/aopalliance-src.tar.gz" } } }, @@ -676,7 +676,7 @@ "other": { "name": "atinject", "version": "1", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/atinject-1.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/atinject-1.tar.xz" } } }, @@ -1016,7 +1016,7 @@ "other": { "name": "bcache-tools", "version": "1.1", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/bcache-tools-1.1.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/bcache-tools-1.1.tar.gz" } } }, @@ -1068,7 +1068,7 @@ "other": { "name": "bea-stax", "version": "1.2.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/stax-src-1.2.0.zip" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/stax-src-1.2.0.zip" } } }, @@ -1808,7 +1808,7 @@ "other": { "name": "clucene", "version": "2.3.3.4", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/clucene-core-2.3.3.4-e8e3d20.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/clucene-core-2.3.3.4-e8e3d20.tar.xz" } } }, @@ -2008,7 +2008,7 @@ "other": { "name": "container-exception-logger", "version": "1.0.3", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/container-exception-logger-1.0.3.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/container-exception-logger-1.0.3.tar.gz" } } }, @@ -2559,7 +2559,7 @@ "other": { "name": "device-mapper-persistent-data", "version": "0.8.5", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/device-mapper-persistent-data-0.8.5.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/device-mapper-persistent-data-0.8.5.tar.gz" } } }, @@ -2899,7 +2899,7 @@ "other": { "name": "dotconf", "version": "1.3", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/dotconf-1.3.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/dotconf-1.3.tar.gz" } } }, @@ -3289,7 +3289,7 @@ "other": { "name": "enscript", "version": "1.6.6", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/enscript-1.6.6.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/enscript-1.6.6.tar.gz" } } }, @@ -3519,7 +3519,7 @@ "other": { "name": "fence-virt", "version": "1.0.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/fence-virt-1.0.0.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/fence-virt-1.0.0.tar.bz2" } } }, @@ -3801,7 +3801,7 @@ "other": { "name": "foomatic-db", "version": "4.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/foomatic-db-4.0-20201104.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/foomatic-db-4.0-20201104.tar.gz" } } }, @@ -4001,7 +4001,7 @@ "other": { "name": "fxload", "version": "2008_10_13", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/fxload-2008_10_13-noa3load.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/fxload-2008_10_13-noa3load.tar.gz" } } }, @@ -4201,7 +4201,7 @@ "other": { "name": "GeoIP-GeoLite-data", "version": "2018.06", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/GeoIP.dat.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/GeoIP.dat.gz" } } }, @@ -4231,7 +4231,7 @@ "other": { "name": "geronimo-specs", "version": "1.2", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/geronimo-specs-1.2.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/geronimo-specs-1.2.tar.gz" } } }, @@ -4701,7 +4701,7 @@ "other": { "name": "google-guice", "version": "5.1.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/google-guice-5.1.0.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/google-guice-5.1.0.tar.gz" } } }, @@ -4781,7 +4781,7 @@ "other": { "name": "gpm", "version": "1.20.7", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/gpm-1.20.7.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/gpm-1.20.7.tar.xz" } } }, @@ -6711,7 +6711,7 @@ "other": { "name": "hyphen-hu", "version": "0.20090612", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/nagybence-huhyphn-aa3fc85.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/nagybence-huhyphn-aa3fc85.tar.gz" } } }, @@ -6791,7 +6791,7 @@ "other": { "name": "hyphen-mi", "version": "0.20080630", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/hunspell-hyphen-mi-0.1.20080630-beta.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/hunspell-hyphen-mi-0.1.20080630-beta.tar.gz" } } }, @@ -7632,7 +7632,7 @@ "other": { "name": "jakarta-taglibs-standard", "version": "1.1.1", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/jakarta-taglibs-standard-1.1.1-src.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/jakarta-taglibs-standard-1.1.1-src.tar.bz2" } } }, @@ -7692,7 +7692,7 @@ "other": { "name": "java-cup", "version": "0.11", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/develop.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/develop.tar.bz2" } } }, @@ -7702,7 +7702,7 @@ "other": { "name": "java-cup-bootstrap", "version": "0.11", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/develop.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/develop.tar.bz2" } } }, @@ -7982,7 +7982,7 @@ "other": { "name": "jsoup", "version": "1.11.3", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/jsoup-1.11.3.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/jsoup-1.11.3.tar.gz" } } }, @@ -7992,7 +7992,7 @@ "other": { "name": "jsr-305", "version": "0.1+20130910", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/jsr-305-20130910svn.tgz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/jsr-305-20130910svn.tgz" } } }, @@ -8002,7 +8002,7 @@ "other": { "name": "jtidy", "version": "8.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/jtidy-r813.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/jtidy-r813.tar.bz2" } } }, @@ -8172,7 +8172,7 @@ "other": { "name": "kernel-mshv", "version": "5.15.126.mshv9", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/kernel-mshv-5.15.126.mshv9.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/kernel-mshv-5.15.126.mshv9.tar.gz" } } }, @@ -8192,7 +8192,7 @@ "other": { "name": "kernel-uvm", "version": "6.1.0.mshv16", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/kernel-uvm-6.1.0.mshv16.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/kernel-uvm-6.1.0.mshv16.tar.gz" } } }, @@ -10802,7 +10802,7 @@ "other": { "name": "librx", "version": "1.5", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/rx-1.5.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/rx-1.5.tar.bz2" } } }, @@ -10962,7 +10962,7 @@ "other": { "name": "libsmi", "version": "0.4.8", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/libsmi-0.4.8.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/libsmi-0.4.8.tar.gz" } } }, @@ -11272,7 +11272,7 @@ "other": { "name": "libucil", "version": "0.9.10", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/libucil-0.9.10.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/libucil-0.9.10.tar.gz" } } }, @@ -11282,7 +11282,7 @@ "other": { "name": "libunicap", "version": "0.9.12", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/libunicap-0.9.12.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/libunicap-0.9.12.tar.gz" } } }, @@ -12142,7 +12142,7 @@ "other": { "name": "lockdev", "version": "1.0.4", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/lockdev-1.0.4.20111007git.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/lockdev-1.0.4.20111007git.tar.gz" } } }, @@ -12552,7 +12552,7 @@ "other": { "name": "mailx", "version": "12.5", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/mailx-12.5.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/mailx-12.5.tar.xz" } } }, @@ -12832,7 +12832,7 @@ "other": { "name": "maven-surefire", "version": "3.2.2", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/maven-surefire-3.2.2.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/maven-surefire-3.2.2.tar.gz" } } }, @@ -12922,7 +12922,7 @@ "other": { "name": "meanwhile", "version": "1.1.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/meanwhile-1.1.0.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/meanwhile-1.1.0.tar.gz" } } }, @@ -13583,7 +13583,7 @@ "other": { "name": "mythes-el", "version": "0.20070412", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/th_el.zip" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/th_el.zip" } } }, @@ -13653,7 +13653,7 @@ "other": { "name": "mythes-mi", "version": "0.20080630", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/mythes-mi-0.1.20080630-beta.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/mythes-mi-0.1.20080630-beta.tar.gz" } } }, @@ -13713,7 +13713,7 @@ "other": { "name": "mythes-ru", "version": "0.20070613", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/thes_ru_RU_v2.zip" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/thes_ru_RU_v2.zip" } } }, @@ -13913,7 +13913,7 @@ "other": { "name": "netpbm", "version": "10.90.00", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/netpbm-10.90.00.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/netpbm-10.90.00.tar.xz" } } }, @@ -14113,7 +14113,7 @@ "other": { "name": "nodejs-nodemon", "version": "2.0.3", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/nodemon-v2.0.3-bundled.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/nodemon-v2.0.3-bundled.tar.gz" } } }, @@ -14263,7 +14263,7 @@ "other": { "name": "numad", "version": "0.5+20150602.aec1497e2b", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/numad-0.5+20150602.aec1497e2b.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/numad-0.5+20150602.aec1497e2b.tar.xz" } } }, @@ -14353,7 +14353,7 @@ "other": { "name": "objectweb-anttask", "version": "1.2", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/ow_util_ant_tasks_1.2.zip" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/ow_util_ant_tasks_1.2.zip" } } }, @@ -14363,7 +14363,7 @@ "other": { "name": "objectweb-asm", "version": "9.6", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/objectweb-asm-9.6.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/objectweb-asm-9.6.tar.gz" } } }, @@ -14883,7 +14883,7 @@ "other": { "name": "ocaml-xml-light", "version": "2.3", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/xml-light-234.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/xml-light-234.tar.gz" } } }, @@ -16094,7 +16094,7 @@ "other": { "name": "perl-Class-Data-Inheritable", "version": "0.08", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/Class-Data-Inheritable-0.08-clean.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/Class-Data-Inheritable-0.08-clean.tar.gz" } } }, @@ -17974,7 +17974,7 @@ "other": { "name": "perl-libnet", "version": "3.11", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/perl-libnet_repackaged-3.11.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/perl-libnet_repackaged-3.11.tar.gz" } } }, @@ -20574,7 +20574,7 @@ "other": { "name": "perl-XML-SAX", "version": "1.02", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/XML-SAX-1.02.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/XML-SAX-1.02.tar.gz" } } }, @@ -20914,7 +20914,7 @@ "other": { "name": "plexus-component-api", "version": "1.0~alpha15", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/plexus-component-api-1.0-alpha-15.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/plexus-component-api-1.0-alpha-15.tar.xz" } } }, @@ -21344,7 +21344,7 @@ "other": { "name": "psutils", "version": "1.23", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/psutils-1.23.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/psutils-1.23.tar.xz" } } }, @@ -21374,7 +21374,7 @@ "other": { "name": "publicsuffix", "version": "20201026", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/publicsuffix-20201026.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/publicsuffix-20201026.tar.gz" } } }, @@ -22594,7 +22594,7 @@ "other": { "name": "python-hwdata", "version": "2.3.7", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/python-hwdata-2.3.7.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/python-hwdata-2.3.7.tar.gz" } } }, @@ -24814,7 +24814,7 @@ "other": { "name": "python-waitress", "version": "1.4.4", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/python-waitress-1.4.4-nodocs.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/python-waitress-1.4.4-nodocs.tar.gz" } } }, @@ -25675,7 +25675,7 @@ "other": { "name": "rt-setup", "version": "2.1", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/rt-setup-2.1.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/rt-setup-2.1.tar.bz2" } } }, @@ -25695,7 +25695,7 @@ "other": { "name": "rtctl", "version": "1.13", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/rtctl-1.13.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/rtctl-1.13.tar.bz2" } } }, @@ -27085,7 +27085,7 @@ "other": { "name": "sblim-cmpi-syslog", "version": "0.9.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/sblim-cmpi-syslog-0.9.0.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/sblim-cmpi-syslog-0.9.0.tar.bz2" } } }, @@ -27325,7 +27325,7 @@ "other": { "name": "servletapi4", "version": "4.0.4", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/jakarta-servletapi-4-src.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/jakarta-servletapi-4-src.tar.gz" } } }, @@ -27335,7 +27335,7 @@ "other": { "name": "servletapi5", "version": "5.0.18", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/jakarta-servletapi-5-src.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/jakarta-servletapi-5-src.tar.gz" } } }, @@ -27396,7 +27396,7 @@ "other": { "name": "setuptool", "version": "1.19.11", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/setuptool-1.19.11.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/setuptool-1.19.11.tar.gz" } } }, @@ -27427,7 +27427,7 @@ "other": { "name": "sgabios", "version": "0.20180715git", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/sgabios-20180715-git72f39d4.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/sgabios-20180715-git72f39d4.tar.xz" } } }, @@ -27447,7 +27447,7 @@ "other": { "name": "sgpio", "version": "1.2.0.10", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/sgpio-1.2-0.10-src.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/sgpio-1.2-0.10-src.tar.gz" } } }, @@ -27827,7 +27827,7 @@ "other": { "name": "splix", "version": "2.0.1", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/splix-2.0.1.20130902svn.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/splix-2.0.1.20130902svn.tar.bz2" } } }, @@ -28087,7 +28087,7 @@ "other": { "name": "symlinks", "version": "1.7", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/symlinks-1.7.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/symlinks-1.7.tar.gz" } } }, @@ -28197,7 +28197,7 @@ "other": { "name": "t1lib", "version": "5.1.2", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/t1lib-5.1.2.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/t1lib-5.1.2.tar.gz" } } }, @@ -28677,7 +28677,7 @@ "other": { "name": "ttmkfdir", "version": "3.0.9", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/ttmkfdir-3.0.9.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/ttmkfdir-3.0.9.tar.bz2" } } }, @@ -28977,7 +28977,7 @@ "other": { "name": "usbip", "version": "6.6.14.1", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/usbip-6.6.14.1.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/usbip-6.6.14.1.tar.xz" } } }, @@ -29227,7 +29227,7 @@ "other": { "name": "virt-who", "version": "0.24.2", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/virt-who-0.24.2.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/virt-who-0.24.2.tar.gz" } } }, @@ -29887,7 +29887,7 @@ "other": { "name": "xhtml1-dtds", "version": "1.0", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/xhtml1-dtds-20020801.tar.xz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/xhtml1-dtds-20020801.tar.xz" } } }, @@ -30038,7 +30038,7 @@ "other": { "name": "xml-commons-apis", "version": "1.4.01", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/xml-commons-apis-1.4.01.tar.gz" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/xml-commons-apis-1.4.01.tar.gz" } } }, @@ -30058,7 +30058,7 @@ "other": { "name": "xmldb-api", "version": "0.1", - "downloadUrl": "https://cblmarinerstorage.blob.core.windows.net/sources/core/xmldb-xapi-20041010-src.tar.bz2" + "downloadUrl": "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/xmldb-xapi-20041010-src.tar.bz2" } } }, diff --git a/toolkit/Makefile b/toolkit/Makefile index aa89eaef17f..8be8c8e29ca 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -122,7 +122,7 @@ SRPMS_DIR ?= $(OUT_DIR)/SRPMS IMAGES_DIR ?= $(OUT_DIR)/images # External source server -SOURCE_URL ?= https://cblmarinerstorage.blob.core.windows.net/sources/core +SOURCE_URL ?= https://azurelinuxsrcstorage.blob.core.windows.net/sources/core # Note on order of precedence: When a variable is passed from the commandline (i.e., make PACKAGE_URL_LIST="my list"), append # assignments do not take affect without using 'override'. This means that all of the following PACKAGE_URL_LIST values will diff --git a/toolkit/docs/building/add-package.md b/toolkit/docs/building/add-package.md index 5c8077f8d1a..daed08893c2 100644 --- a/toolkit/docs/building/add-package.md +++ b/toolkit/docs/building/add-package.md @@ -272,7 +272,7 @@ $ sudo make toolchain -j8 REBUILD_TOOLCHAIN=n REBUILD_TOOLS=y DAILY_BUILD_ID=3-0 Downloading toolchain RPM: ... # Let's build our package. # NOTE, if you also changed the packages used as BuildRequires, you should add them to SRPM_PACK_LIST. -$ sudo make build-packages REBUILD_TOOLS=y SOURCE_URL='https://cblmarinerstorage.blob.core.windows.net/sources/core' PACKAGE_REBUILD_LIST='ig' SRPM_PACK_LIST='ig' RUN_CHECK=y SRPM_FILE_SIGNATURE_HANDLING=update DAILY_BUILD_ID=3-0-2024028 +$ sudo make build-packages REBUILD_TOOLS=y SOURCE_URL='https://azurelinuxsrcstorage.blob.core.windows.net/sources/core' PACKAGE_REBUILD_LIST='ig' SRPM_PACK_LIST='ig' RUN_CHECK=y SRPM_FILE_SIGNATURE_HANDLING=update DAILY_BUILD_ID=3-0-2024028 ... INFO[0003][srpmpacker] Packing 1/1 SPECs ... diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index 1a40db1c8e4..9c37f07951b 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -363,12 +363,12 @@ sudo make image CONFIG_FILE="./imageconfigs/core-efi.json" CA_CERT=/path/to/root ## Building Everything From Scratch -**NOTE: Source files must be made available for all packages. They can be placed manually in the corresponding SPEC/\* folders, `SOURCE_URL=` may be provided, or DOWNLOAD_SRPMS=y may be used to use pre-packages sources. Core Azure Linux source packages are available at `SOURCE_URL=https://cblmarinerstorage.blob.core.windows.net/sources/core`** +**NOTE: Source files must be made available for all packages. They can be placed manually in the corresponding SPEC/\* folders, `SOURCE_URL=` may be provided, or DOWNLOAD_SRPMS=y may be used to use pre-packages sources. Core Azure Linux source packages are available at `SOURCE_URL=https://azurelinuxsrcstorage.blob.core.windows.net/sources/core`** The build system can operate without using pre-built components if desired. There are several variables which enable/disable build components and sources of data. They are listed here along with their default values: ```makefile -SOURCE_URL ?= https://cblmarinerstorage.blob.core.windows.net/sources/core +SOURCE_URL ?= https://azurelinuxsrcstorage.blob.core.windows.net/sources/core PACKAGE_URL_LIST ?= https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/prod/base/$(build_arch) SRPM_URL_LIST ?= https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/prod/base/srpms REPO_LIST ?= diff --git a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt index e6449beb676..ddbb5c3a4a4 100644 --- a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt @@ -204,8 +204,8 @@ libltdl-2.4.7-1.azl3.aarch64.rpm libltdl-devel-2.4.7-1.azl3.aarch64.rpm lua-5.4.6-1.azl3.aarch64.rpm lua-libs-5.4.6-1.azl3.aarch64.rpm -azurelinux-rpm-macros-3.0-3.azl3.noarch.rpm -azurelinux-check-macros-3.0-3.azl3.noarch.rpm +azurelinux-rpm-macros-3.0-4.azl3.noarch.rpm +azurelinux-check-macros-3.0-4.azl3.noarch.rpm tdnf-3.5.6-1.azl3.aarch64.rpm tdnf-cli-libs-3.5.6-1.azl3.aarch64.rpm tdnf-devel-3.5.6-1.azl3.aarch64.rpm diff --git a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt index 06c57d569c7..36480350aea 100644 --- a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt @@ -204,8 +204,8 @@ libltdl-2.4.7-1.azl3.x86_64.rpm libltdl-devel-2.4.7-1.azl3.x86_64.rpm lua-5.4.6-1.azl3.x86_64.rpm lua-libs-5.4.6-1.azl3.x86_64.rpm -azurelinux-rpm-macros-3.0-3.azl3.noarch.rpm -azurelinux-check-macros-3.0-3.azl3.noarch.rpm +azurelinux-rpm-macros-3.0-4.azl3.noarch.rpm +azurelinux-check-macros-3.0-4.azl3.noarch.rpm tdnf-3.5.6-1.azl3.x86_64.rpm tdnf-cli-libs-3.5.6-1.azl3.x86_64.rpm tdnf-devel-3.5.6-1.azl3.x86_64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index 957ae3d03b2..ec8f354b3c8 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -5,7 +5,7 @@ audit-devel-3.1.2-1.azl3.aarch64.rpm audit-libs-3.1.2-1.azl3.aarch64.rpm autoconf-2.72-1.azl3.noarch.rpm automake-1.16.5-1.azl3.noarch.rpm -azurelinux-check-macros-3.0-3.azl3.noarch.rpm +azurelinux-check-macros-3.0-4.azl3.noarch.rpm azurelinux-repos-3.0-2.azl3.noarch.rpm azurelinux-repos-debug-3.0-2.azl3.noarch.rpm azurelinux-repos-debug-preview-3.0-2.azl3.noarch.rpm @@ -19,7 +19,7 @@ azurelinux-repos-ms-oss-3.0-2.azl3.noarch.rpm azurelinux-repos-ms-oss-preview-3.0-2.azl3.noarch.rpm azurelinux-repos-preview-3.0-2.azl3.noarch.rpm azurelinux-repos-shared-3.0-2.azl3.noarch.rpm -azurelinux-rpm-macros-3.0-3.azl3.noarch.rpm +azurelinux-rpm-macros-3.0-4.azl3.noarch.rpm bash-5.2.15-1.azl3.aarch64.rpm bash-debuginfo-5.2.15-1.azl3.aarch64.rpm bash-devel-5.2.15-1.azl3.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index 97a81334f89..814b601579d 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -5,7 +5,7 @@ audit-devel-3.1.2-1.azl3.x86_64.rpm audit-libs-3.1.2-1.azl3.x86_64.rpm autoconf-2.72-1.azl3.noarch.rpm automake-1.16.5-1.azl3.noarch.rpm -azurelinux-check-macros-3.0-3.azl3.noarch.rpm +azurelinux-check-macros-3.0-4.azl3.noarch.rpm azurelinux-repos-3.0-2.azl3.noarch.rpm azurelinux-repos-debug-3.0-2.azl3.noarch.rpm azurelinux-repos-debug-preview-3.0-2.azl3.noarch.rpm @@ -19,7 +19,7 @@ azurelinux-repos-ms-oss-3.0-2.azl3.noarch.rpm azurelinux-repos-ms-oss-preview-3.0-2.azl3.noarch.rpm azurelinux-repos-preview-3.0-2.azl3.noarch.rpm azurelinux-repos-shared-3.0-2.azl3.noarch.rpm -azurelinux-rpm-macros-3.0-3.azl3.noarch.rpm +azurelinux-rpm-macros-3.0-4.azl3.noarch.rpm bash-5.2.15-1.azl3.x86_64.rpm bash-debuginfo-5.2.15-1.azl3.x86_64.rpm bash-devel-5.2.15-1.azl3.x86_64.rpm diff --git a/toolkit/scripts/build_cargo_cache.sh b/toolkit/scripts/build_cargo_cache.sh index 26943d8ac1c..bffef3bb30c 100755 --- a/toolkit/scripts/build_cargo_cache.sh +++ b/toolkit/scripts/build_cargo_cache.sh @@ -34,7 +34,7 @@ then cp "$tarball_name" "$temp_dir" else echo "Tarball '$tarball_name' doesn't exist. Will attempt to download from blobstorage." - if ! wget -q "https://cblmarinerstorage.blob.core.windows.net/sources/core/$tarball_name" -O "$temp_dir/$tarball_name" + if ! wget -q "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/$tarball_name" -O "$temp_dir/$tarball_name" then echo "ERROR: failed to download the source tarball." exit 1 diff --git a/toolkit/scripts/build_go_vendor_cache.sh b/toolkit/scripts/build_go_vendor_cache.sh index 7002d3022bf..7275859bc74 100755 --- a/toolkit/scripts/build_go_vendor_cache.sh +++ b/toolkit/scripts/build_go_vendor_cache.sh @@ -34,7 +34,7 @@ then cp "$tarball_name" "$temp_dir" else echo "Tarball '$tarball_name' doesn't exist. Will attempt to download from blobstorage." - if ! wget -q "https://cblmarinerstorage.blob.core.windows.net/sources/core/$tarball_name" -O "$temp_dir/$tarball_name" + if ! wget -q "https://azurelinuxsrcstorage.blob.core.windows.net/sources/core/$tarball_name" -O "$temp_dir/$tarball_name" then echo "ERROR: failed to download the source tarball." exit 1 From 3cc4370ed0db593680cce5da8624274865cb0540 Mon Sep 17 00:00:00 2001 From: Riken Maharjan <106988478+rikenm1@users.noreply.github.com> Date: Fri, 17 May 2024 16:21:22 -0700 Subject: [PATCH 28/51] Refractor 3.0-dev Golden Container (#9112) --- .../containerSourceData/busybox/busybox.name | 1 + .pipelines/containerSourceData/cdi/api.name | 1 + .../containerSourceData/cdi/cloner.name | 1 + .../containerSourceData/cdi/controller.name | 1 + .../containerSourceData/cdi/importer.name | 1 + .../containerSourceData/cdi/operator.name | 1 + .../containerSourceData/cdi/uploadproxy.name | 1 + .../containerSourceData/cdi/uploadserver.name | 1 + .../certmanager/acmesolver.name | 1 + .../certmanager/cainjector.name | 1 + .../certmanager/cmctl.name | 1 + .../certmanager/controller.name | 1 + .../certmanager/webhook.name | 1 + .../influxdb/influxdb.name | 1 + .../kubevirt/virt-api.name | 1 + .../kubevirt/virt-controller.name | 1 + .../kubevirt/virt-handler.name | 1 + .../kubevirt/virt-launcher.name | 1 + .../kubevirt/virt-operator.name | 1 + .../memcached/memcached.name | 1 + .../containerSourceData/multus/multus.name | 1 + .../containerSourceData/nginx/nginx.name | 1 + ...dback-nodejs18.pkg => holdback-nodejs.pkg} | 0 .../distroless/{nodejs18.pkg => nodejs.pkg} | 2 +- .../containerSourceData/nodejs/nodejs.name | 1 + .../containerSourceData/nodejs/nodejs18.pkg | 2 - .../containerSourceData/openmpi/openmpi.name | 1 + .pipelines/containerSourceData/php/php.name | 1 + .../postgres/postgres.name | 1 + .../prometheus/prometheus.name | 1 + .../prometheusadapter/prometheusadapter.name | 1 + .../containerSourceData/python/python.name | 1 + .../containerSourceData/pytorch/pytorch.name | 1 + .../rabbitmqserver/rabbitmqserver.name | 1 + .pipelines/containerSourceData/ruby/ruby.name | 1 + .pipelines/containerSourceData/rust/rust.name | 1 + .../scripts/BuildGoldenContainer.sh | 37 ++++++++++++------- .../sriovnetworkdeviceplugin.name | 1 + .../telegraf/telegraf.name | 1 + .../tensorflow/tensorflow.name | 1 + 40 files changed, 60 insertions(+), 17 deletions(-) create mode 100644 .pipelines/containerSourceData/busybox/busybox.name create mode 100644 .pipelines/containerSourceData/cdi/api.name create mode 100644 .pipelines/containerSourceData/cdi/cloner.name create mode 100644 .pipelines/containerSourceData/cdi/controller.name create mode 100644 .pipelines/containerSourceData/cdi/importer.name create mode 100644 .pipelines/containerSourceData/cdi/operator.name create mode 100644 .pipelines/containerSourceData/cdi/uploadproxy.name create mode 100644 .pipelines/containerSourceData/cdi/uploadserver.name create mode 100644 .pipelines/containerSourceData/certmanager/acmesolver.name create mode 100644 .pipelines/containerSourceData/certmanager/cainjector.name create mode 100644 .pipelines/containerSourceData/certmanager/cmctl.name create mode 100644 .pipelines/containerSourceData/certmanager/controller.name create mode 100644 .pipelines/containerSourceData/certmanager/webhook.name create mode 100644 .pipelines/containerSourceData/influxdb/influxdb.name create mode 100644 .pipelines/containerSourceData/kubevirt/virt-api.name create mode 100644 .pipelines/containerSourceData/kubevirt/virt-controller.name create mode 100644 .pipelines/containerSourceData/kubevirt/virt-handler.name create mode 100644 .pipelines/containerSourceData/kubevirt/virt-launcher.name create mode 100644 .pipelines/containerSourceData/kubevirt/virt-operator.name create mode 100644 .pipelines/containerSourceData/memcached/memcached.name create mode 100644 .pipelines/containerSourceData/multus/multus.name create mode 100644 .pipelines/containerSourceData/nginx/nginx.name rename .pipelines/containerSourceData/nodejs/distroless/{holdback-nodejs18.pkg => holdback-nodejs.pkg} (100%) rename .pipelines/containerSourceData/nodejs/distroless/{nodejs18.pkg => nodejs.pkg} (84%) create mode 100644 .pipelines/containerSourceData/nodejs/nodejs.name delete mode 100644 .pipelines/containerSourceData/nodejs/nodejs18.pkg create mode 100644 .pipelines/containerSourceData/openmpi/openmpi.name create mode 100644 .pipelines/containerSourceData/php/php.name create mode 100644 .pipelines/containerSourceData/postgres/postgres.name create mode 100644 .pipelines/containerSourceData/prometheus/prometheus.name create mode 100644 .pipelines/containerSourceData/prometheusadapter/prometheusadapter.name create mode 100644 .pipelines/containerSourceData/python/python.name create mode 100644 .pipelines/containerSourceData/pytorch/pytorch.name create mode 100644 .pipelines/containerSourceData/rabbitmqserver/rabbitmqserver.name create mode 100644 .pipelines/containerSourceData/ruby/ruby.name create mode 100644 .pipelines/containerSourceData/rust/rust.name create mode 100644 .pipelines/containerSourceData/sriovnetworkdeviceplugin/sriovnetworkdeviceplugin.name create mode 100644 .pipelines/containerSourceData/telegraf/telegraf.name create mode 100644 .pipelines/containerSourceData/tensorflow/tensorflow.name diff --git a/.pipelines/containerSourceData/busybox/busybox.name b/.pipelines/containerSourceData/busybox/busybox.name new file mode 100644 index 00000000000..a1a4c366ccf --- /dev/null +++ b/.pipelines/containerSourceData/busybox/busybox.name @@ -0,0 +1 @@ +busybox diff --git a/.pipelines/containerSourceData/cdi/api.name b/.pipelines/containerSourceData/cdi/api.name new file mode 100644 index 00000000000..0342de851b8 --- /dev/null +++ b/.pipelines/containerSourceData/cdi/api.name @@ -0,0 +1 @@ +containerized-data-importer-api diff --git a/.pipelines/containerSourceData/cdi/cloner.name b/.pipelines/containerSourceData/cdi/cloner.name new file mode 100644 index 00000000000..808816c9c80 --- /dev/null +++ b/.pipelines/containerSourceData/cdi/cloner.name @@ -0,0 +1 @@ +containerized-data-importer-cloner diff --git a/.pipelines/containerSourceData/cdi/controller.name b/.pipelines/containerSourceData/cdi/controller.name new file mode 100644 index 00000000000..fb8480f0466 --- /dev/null +++ b/.pipelines/containerSourceData/cdi/controller.name @@ -0,0 +1 @@ +containerized-data-importer-controller diff --git a/.pipelines/containerSourceData/cdi/importer.name b/.pipelines/containerSourceData/cdi/importer.name new file mode 100644 index 00000000000..6144493f051 --- /dev/null +++ b/.pipelines/containerSourceData/cdi/importer.name @@ -0,0 +1 @@ +containerized-data-importer-importer diff --git a/.pipelines/containerSourceData/cdi/operator.name b/.pipelines/containerSourceData/cdi/operator.name new file mode 100644 index 00000000000..a2707ad445d --- /dev/null +++ b/.pipelines/containerSourceData/cdi/operator.name @@ -0,0 +1 @@ +containerized-data-importer-operator diff --git a/.pipelines/containerSourceData/cdi/uploadproxy.name b/.pipelines/containerSourceData/cdi/uploadproxy.name new file mode 100644 index 00000000000..4a6907a2d70 --- /dev/null +++ b/.pipelines/containerSourceData/cdi/uploadproxy.name @@ -0,0 +1 @@ +containerized-data-importer-uploadproxy diff --git a/.pipelines/containerSourceData/cdi/uploadserver.name b/.pipelines/containerSourceData/cdi/uploadserver.name new file mode 100644 index 00000000000..77d787d7f65 --- /dev/null +++ b/.pipelines/containerSourceData/cdi/uploadserver.name @@ -0,0 +1 @@ +containerized-data-importer-uploadserver diff --git a/.pipelines/containerSourceData/certmanager/acmesolver.name b/.pipelines/containerSourceData/certmanager/acmesolver.name new file mode 100644 index 00000000000..73be14a5d67 --- /dev/null +++ b/.pipelines/containerSourceData/certmanager/acmesolver.name @@ -0,0 +1 @@ +cert-manager-acmesolver diff --git a/.pipelines/containerSourceData/certmanager/cainjector.name b/.pipelines/containerSourceData/certmanager/cainjector.name new file mode 100644 index 00000000000..585afeb7ba2 --- /dev/null +++ b/.pipelines/containerSourceData/certmanager/cainjector.name @@ -0,0 +1 @@ +cert-manager-cainjector diff --git a/.pipelines/containerSourceData/certmanager/cmctl.name b/.pipelines/containerSourceData/certmanager/cmctl.name new file mode 100644 index 00000000000..7a9f7d43f36 --- /dev/null +++ b/.pipelines/containerSourceData/certmanager/cmctl.name @@ -0,0 +1 @@ +cert-manager-cmctl diff --git a/.pipelines/containerSourceData/certmanager/controller.name b/.pipelines/containerSourceData/certmanager/controller.name new file mode 100644 index 00000000000..afef2317b8b --- /dev/null +++ b/.pipelines/containerSourceData/certmanager/controller.name @@ -0,0 +1 @@ +cert-manager-controller diff --git a/.pipelines/containerSourceData/certmanager/webhook.name b/.pipelines/containerSourceData/certmanager/webhook.name new file mode 100644 index 00000000000..789b6a4b02b --- /dev/null +++ b/.pipelines/containerSourceData/certmanager/webhook.name @@ -0,0 +1 @@ +cert-manager-webhook diff --git a/.pipelines/containerSourceData/influxdb/influxdb.name b/.pipelines/containerSourceData/influxdb/influxdb.name new file mode 100644 index 00000000000..30775bb658f --- /dev/null +++ b/.pipelines/containerSourceData/influxdb/influxdb.name @@ -0,0 +1 @@ +influxdb diff --git a/.pipelines/containerSourceData/kubevirt/virt-api.name b/.pipelines/containerSourceData/kubevirt/virt-api.name new file mode 100644 index 00000000000..2a28b6ab661 --- /dev/null +++ b/.pipelines/containerSourceData/kubevirt/virt-api.name @@ -0,0 +1 @@ +kubevirt-virt-api diff --git a/.pipelines/containerSourceData/kubevirt/virt-controller.name b/.pipelines/containerSourceData/kubevirt/virt-controller.name new file mode 100644 index 00000000000..fcaf0328106 --- /dev/null +++ b/.pipelines/containerSourceData/kubevirt/virt-controller.name @@ -0,0 +1 @@ +kubevirt-virt-controller diff --git a/.pipelines/containerSourceData/kubevirt/virt-handler.name b/.pipelines/containerSourceData/kubevirt/virt-handler.name new file mode 100644 index 00000000000..994b89a434a --- /dev/null +++ b/.pipelines/containerSourceData/kubevirt/virt-handler.name @@ -0,0 +1 @@ +kubevirt-virt-handler diff --git a/.pipelines/containerSourceData/kubevirt/virt-launcher.name b/.pipelines/containerSourceData/kubevirt/virt-launcher.name new file mode 100644 index 00000000000..27ca5989d3c --- /dev/null +++ b/.pipelines/containerSourceData/kubevirt/virt-launcher.name @@ -0,0 +1 @@ +kubevirt-virt-launcher diff --git a/.pipelines/containerSourceData/kubevirt/virt-operator.name b/.pipelines/containerSourceData/kubevirt/virt-operator.name new file mode 100644 index 00000000000..396a2a845c6 --- /dev/null +++ b/.pipelines/containerSourceData/kubevirt/virt-operator.name @@ -0,0 +1 @@ +kubevirt-virt-operator diff --git a/.pipelines/containerSourceData/memcached/memcached.name b/.pipelines/containerSourceData/memcached/memcached.name new file mode 100644 index 00000000000..51b7e42eb61 --- /dev/null +++ b/.pipelines/containerSourceData/memcached/memcached.name @@ -0,0 +1 @@ +memcached diff --git a/.pipelines/containerSourceData/multus/multus.name b/.pipelines/containerSourceData/multus/multus.name new file mode 100644 index 00000000000..946d4d3b2c6 --- /dev/null +++ b/.pipelines/containerSourceData/multus/multus.name @@ -0,0 +1 @@ +multus diff --git a/.pipelines/containerSourceData/nginx/nginx.name b/.pipelines/containerSourceData/nginx/nginx.name new file mode 100644 index 00000000000..68b7d12d44c --- /dev/null +++ b/.pipelines/containerSourceData/nginx/nginx.name @@ -0,0 +1 @@ +nginx diff --git a/.pipelines/containerSourceData/nodejs/distroless/holdback-nodejs18.pkg b/.pipelines/containerSourceData/nodejs/distroless/holdback-nodejs.pkg similarity index 100% rename from .pipelines/containerSourceData/nodejs/distroless/holdback-nodejs18.pkg rename to .pipelines/containerSourceData/nodejs/distroless/holdback-nodejs.pkg diff --git a/.pipelines/containerSourceData/nodejs/distroless/nodejs18.pkg b/.pipelines/containerSourceData/nodejs/distroless/nodejs.pkg similarity index 84% rename from .pipelines/containerSourceData/nodejs/distroless/nodejs18.pkg rename to .pipelines/containerSourceData/nodejs/distroless/nodejs.pkg index ef8b0826702..a3dd92d9c6e 100644 --- a/.pipelines/containerSourceData/nodejs/distroless/nodejs18.pkg +++ b/.pipelines/containerSourceData/nodejs/distroless/nodejs.pkg @@ -1,3 +1,3 @@ distroless-packages-base -nodejs18 +nodejs prebuilt-ca-certificates diff --git a/.pipelines/containerSourceData/nodejs/nodejs.name b/.pipelines/containerSourceData/nodejs/nodejs.name new file mode 100644 index 00000000000..e36de65c4cc --- /dev/null +++ b/.pipelines/containerSourceData/nodejs/nodejs.name @@ -0,0 +1 @@ +nodejs diff --git a/.pipelines/containerSourceData/nodejs/nodejs18.pkg b/.pipelines/containerSourceData/nodejs/nodejs18.pkg deleted file mode 100644 index e4c2ea2a619..00000000000 --- a/.pipelines/containerSourceData/nodejs/nodejs18.pkg +++ /dev/null @@ -1,2 +0,0 @@ -ca-certificates -nodejs18 diff --git a/.pipelines/containerSourceData/openmpi/openmpi.name b/.pipelines/containerSourceData/openmpi/openmpi.name new file mode 100644 index 00000000000..6bd6ad4fb3a --- /dev/null +++ b/.pipelines/containerSourceData/openmpi/openmpi.name @@ -0,0 +1 @@ +openmpi diff --git a/.pipelines/containerSourceData/php/php.name b/.pipelines/containerSourceData/php/php.name new file mode 100644 index 00000000000..2c0ea7b5a35 --- /dev/null +++ b/.pipelines/containerSourceData/php/php.name @@ -0,0 +1 @@ +php diff --git a/.pipelines/containerSourceData/postgres/postgres.name b/.pipelines/containerSourceData/postgres/postgres.name new file mode 100644 index 00000000000..a02a1cc751d --- /dev/null +++ b/.pipelines/containerSourceData/postgres/postgres.name @@ -0,0 +1 @@ +postgresql diff --git a/.pipelines/containerSourceData/prometheus/prometheus.name b/.pipelines/containerSourceData/prometheus/prometheus.name new file mode 100644 index 00000000000..26b17a679f3 --- /dev/null +++ b/.pipelines/containerSourceData/prometheus/prometheus.name @@ -0,0 +1 @@ +prometheus diff --git a/.pipelines/containerSourceData/prometheusadapter/prometheusadapter.name b/.pipelines/containerSourceData/prometheusadapter/prometheusadapter.name new file mode 100644 index 00000000000..aef6dc50c48 --- /dev/null +++ b/.pipelines/containerSourceData/prometheusadapter/prometheusadapter.name @@ -0,0 +1 @@ +prometheus-adapter diff --git a/.pipelines/containerSourceData/python/python.name b/.pipelines/containerSourceData/python/python.name new file mode 100644 index 00000000000..fdc793e786a --- /dev/null +++ b/.pipelines/containerSourceData/python/python.name @@ -0,0 +1 @@ +python diff --git a/.pipelines/containerSourceData/pytorch/pytorch.name b/.pipelines/containerSourceData/pytorch/pytorch.name new file mode 100644 index 00000000000..c2414f47812 --- /dev/null +++ b/.pipelines/containerSourceData/pytorch/pytorch.name @@ -0,0 +1 @@ +python3-pytorch diff --git a/.pipelines/containerSourceData/rabbitmqserver/rabbitmqserver.name b/.pipelines/containerSourceData/rabbitmqserver/rabbitmqserver.name new file mode 100644 index 00000000000..f8d6ed52ad8 --- /dev/null +++ b/.pipelines/containerSourceData/rabbitmqserver/rabbitmqserver.name @@ -0,0 +1 @@ +rabbitmq-server diff --git a/.pipelines/containerSourceData/ruby/ruby.name b/.pipelines/containerSourceData/ruby/ruby.name new file mode 100644 index 00000000000..6cec9344a1b --- /dev/null +++ b/.pipelines/containerSourceData/ruby/ruby.name @@ -0,0 +1 @@ +ruby diff --git a/.pipelines/containerSourceData/rust/rust.name b/.pipelines/containerSourceData/rust/rust.name new file mode 100644 index 00000000000..871732e64f9 --- /dev/null +++ b/.pipelines/containerSourceData/rust/rust.name @@ -0,0 +1 @@ +rust diff --git a/.pipelines/containerSourceData/scripts/BuildGoldenContainer.sh b/.pipelines/containerSourceData/scripts/BuildGoldenContainer.sh index 68ace7d4017..7170ef9ea2e 100755 --- a/.pipelines/containerSourceData/scripts/BuildGoldenContainer.sh +++ b/.pipelines/containerSourceData/scripts/BuildGoldenContainer.sh @@ -10,7 +10,7 @@ set -e # - b) ACR name (e.g. azurelinepreview, acrafoimages, etc.) # - c) Container repository name (e.g. base/nodejs, base/postgres, base/kubevirt/cdi-apiserver, etc.) # - d) Image name (e.g. nodejs, postgres, cdi, etc.) -# - e) Component name (e.g. nodejs18, postgresql, containerized-data-importer-api, etc.) +# - e) Component file name (e.g. nodejs.name, postgres.name, api.name, etc.) # - f) Package file name (e.g. nodejs18.pkg, postgres.pkg, api.pkg, etc.) # - g) Dockerfile name (e.g. Dockerfile-nodejs, Dockerfile-Postgres, Dockerfile-cdi-apiserver, etc.) # - h) Docker build arguments (e.g. '--build-arg BINARY_NAME="cdi-apiserver" --build-arg USER=1001') @@ -38,10 +38,11 @@ set -e # ~/azurelinux/.pipelines/containerSourceData # ├── nodejs # │ ├── distroless -# │ │ ├── holdback-nodejs18.pkg -# │ │ ├── nodejs18.pkg +# │ │ ├── holdback-nodejs.pkg +# │ │ ├── nodejs.pkg # │ ├── Dockerfile-Nodejs -# │ ├── nodejs18.pkg +# │ ├── nodejs.pkg +# | |── nodejs.name # ├── configuration # │ ├── acrRepoV2.json # ├── scripts @@ -62,7 +63,7 @@ while getopts ":a:b:c:d:e:f:g:h:i:j:k:l:m:n:o:p:q:r:s:t:u:v:" OPTIONS; do b ) ACR=$OPTARG;; c ) REPOSITORY=$OPTARG;; d ) IMAGE=$OPTARG;; - e ) COMPONENT=$OPTARG;; + e ) COMPONENT_FILE=$OPTARG;; f ) PACKAGE_FILE=$OPTARG;; g ) DOCKERFILE=$OPTARG;; h ) DOCKER_BUILD_ARGS=$OPTARG;; @@ -105,7 +106,7 @@ function print_inputs { echo "ACR -> $ACR" echo "REPOSITORY -> $REPOSITORY" echo "IMAGE -> $IMAGE" - echo "COMPONENT -> $COMPONENT" + echo "COMPONENT_FILE -> $COMPONENT_FILE" echo "PACKAGE_FILE -> $PACKAGE_FILE" echo "DOCKERFILE -> $DOCKERFILE" echo "DOCKER_BUILD_ARGS -> $DOCKER_BUILD_ARGS" @@ -210,6 +211,20 @@ function initialization { echo "End of Life -> $END_OF_LIFE_1_YEAR" } +function get_packages_to_install { + echo "+++ Get packages to install" + packagesFilePath="$CONTAINER_SRC_DIR/$IMAGE/$PACKAGE_FILE" + PACKAGES_TO_INSTALL=$(paste -s -d' ' < "$packagesFilePath") + echo "Packages to install -> $PACKAGES_TO_INSTALL" +} + +function get_component_name { + echo "+++ Get Component name" + componentFilePath="$CONTAINER_SRC_DIR/$IMAGE/$COMPONENT_FILE" + COMPONENT=$(cat "$componentFilePath") + echo "Component name -> $COMPONENT" +} + function prepare_dockerfile { echo "+++ Prepare dockerfile" # Copy original dockerfile from Azure Linux repo. @@ -230,13 +245,6 @@ function prepare_dockerfile { echo "" } -function get_packages_to_install { - echo "+++ Get packages to install" - packagesFilePath="$CONTAINER_SRC_DIR/$IMAGE/$PACKAGE_FILE" - PACKAGES_TO_INSTALL=$(paste -s -d' ' < "$packagesFilePath") - echo "Packages to install -> $PACKAGES_TO_INSTALL" -} - function prepare_docker_directory { echo "+++ Prepare docker directory" # Get additional required files for the container build from Azure Linux repo. @@ -389,8 +397,9 @@ function distroless_container { print_inputs validate_inputs initialization -prepare_dockerfile get_packages_to_install +get_component_name +prepare_dockerfile prepare_docker_directory docker_build set_image_tag diff --git a/.pipelines/containerSourceData/sriovnetworkdeviceplugin/sriovnetworkdeviceplugin.name b/.pipelines/containerSourceData/sriovnetworkdeviceplugin/sriovnetworkdeviceplugin.name new file mode 100644 index 00000000000..aa9668b5b90 --- /dev/null +++ b/.pipelines/containerSourceData/sriovnetworkdeviceplugin/sriovnetworkdeviceplugin.name @@ -0,0 +1 @@ +sriov-network-device-plugin diff --git a/.pipelines/containerSourceData/telegraf/telegraf.name b/.pipelines/containerSourceData/telegraf/telegraf.name new file mode 100644 index 00000000000..e6b8775bef7 --- /dev/null +++ b/.pipelines/containerSourceData/telegraf/telegraf.name @@ -0,0 +1 @@ +telegraf diff --git a/.pipelines/containerSourceData/tensorflow/tensorflow.name b/.pipelines/containerSourceData/tensorflow/tensorflow.name new file mode 100644 index 00000000000..da9768469a8 --- /dev/null +++ b/.pipelines/containerSourceData/tensorflow/tensorflow.name @@ -0,0 +1 @@ +python3-tensorflow From ff219df00bc49d3b7cb4b8a3e36d3d9b41c369c8 Mon Sep 17 00:00:00 2001 From: Pawel Winogrodzki Date: Mon, 20 May 2024 14:56:39 -0700 Subject: [PATCH 29/51] Adding admin approval for certificate changes. (CP: #8859) (#8875) --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ac4af83d444..acf9f2aaa47 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -87,6 +87,9 @@ /toolkit/scripts/toolchain/cgmanifest.json @microsoft/cbl-mariner-admins /toolkit/scripts/toolchain/create_toolchain_in_container.sh @microsoft/cbl-mariner-admins +# Modifications to the trusted CA certificates require admin approval. +/SPECS/*ca-certificates*/* @microsoft/cbl-mariner-admins + # Image Customizer /toolkit/tools/imagecustomizer/ @microsoft/cbl-mariner-imagecustomizer /toolkit/tools/imagecustomizerapi/ @microsoft/cbl-mariner-imagecustomizer From a54fff37662e89e50e47f4ad4b3291c612ae0b42 Mon Sep 17 00:00:00 2001 From: Elaheh Dehghani <108492863+ellie-di@users.noreply.github.com> Date: Mon, 20 May 2024 15:39:12 -0700 Subject: [PATCH 30/51] Allow a custom path to be specified for custom snapshot for the pre-cacher (#9137) --- toolkit/Makefile | 2 ++ toolkit/docs/building/building.md | 1 + toolkit/scripts/precache.mk | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/toolkit/Makefile b/toolkit/Makefile index 8be8c8e29ca..be9193a72f5 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -121,6 +121,8 @@ RPMS_DIR ?= $(OUT_DIR)/RPMS SRPMS_DIR ?= $(OUT_DIR)/SRPMS IMAGES_DIR ?= $(OUT_DIR)/images +PRECACHER_SNAPSHOT ?= $(rpms_snapshot) + # External source server SOURCE_URL ?= https://azurelinuxsrcstorage.blob.core.windows.net/sources/core diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index 9c37f07951b..35788c25946 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -873,6 +873,7 @@ To reproduce an ISO build, run the same make invocation as before, but set: | RPMS_DIR | `$(OUT_DIR)`/RPMS | Directory to place RPMs in | SRPMS_DIR | `$(OUT_DIR)`/SRPMS | Directory to place SRPMs in | IMAGES_DIR | `$(OUT_DIR)`/images | Directory to place images in +| PRECACHER_SNAPSHOT | `$(OUT_DIR)`/rpms_snapshot.json | Location of snapshot file for the pre-cacher --- diff --git a/toolkit/scripts/precache.mk b/toolkit/scripts/precache.mk index 70633ac2562..6e4441bf01b 100644 --- a/toolkit/scripts/precache.mk +++ b/toolkit/scripts/precache.mk @@ -26,13 +26,13 @@ clean-precache: # it does, so add the phony target as a dependency to the flag file .PHONY: precache_always_run_phony pre-cache: $(STATUS_FLAGS_DIR)/precache.flag -$(STATUS_FLAGS_DIR)/precache.flag: $(go-precacher) $(chroot_worker) $(rpms_snapshot) precache_always_run_phony +$(STATUS_FLAGS_DIR)/precache.flag: $(go-precacher) $(chroot_worker) $(PRECACHER_SNAPSHOT) precache_always_run_phony @if [ "$(DISABLE_UPSTREAM_REPOS)" = "y" ]; then \ echo "ERROR: Upstream repos are disabled (DISABLE_UPSTREAM_REPOS=y), cannot precache RPMs"; \ exit 1; \ fi $(go-precacher) \ - --snapshot "$(rpms_snapshot)" \ + --snapshot "$(PRECACHER_SNAPSHOT)" \ --output-dir "$(remote_rpms_cache_dir)" \ --output-summary-file "$(precache_downloaded_files)" \ --repo-urls-file "$(repo_urls_file)" \ From dc8752a04cfecd0162189a2d45034782c94614e3 Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Mon, 20 May 2024 16:18:49 -0700 Subject: [PATCH 31/51] Refactor shell execute functions. (#9088) --- toolkit/tools/imagegen/diskutils/diskutils.go | 11 +- .../imagegen/installutils/installutils.go | 32 +- toolkit/tools/internal/logger/log.go | 32 -- .../repocloner/rpmrepocloner/rpmrepocloner.go | 14 +- .../packagerepo/repoutils/repoquery.go | 21 +- toolkit/tools/internal/rpm/rpm.go | 21 -- .../tools/internal/safechroot/safechroot.go | 7 +- toolkit/tools/internal/shell/execbuilder.go | 338 ++++++++++++++++++ toolkit/tools/internal/shell/shell.go | 172 +-------- toolkit/tools/internal/userutils/userutils.go | 6 +- toolkit/tools/liveinstaller/liveinstaller.go | 14 +- .../pkg/imagecustomizerlib/customizeboot.go | 2 +- .../imagecustomizerlib/customizepackages.go | 20 +- .../customizepartitionsfilecopy.go | 7 +- .../imagecustomizerlib/liveosisobuilder.go | 2 +- toolkit/tools/roast/formats/ova.go | 29 +- .../scheduler/buildagents/chrootagent.go | 17 +- 17 files changed, 451 insertions(+), 294 deletions(-) create mode 100644 toolkit/tools/internal/shell/execbuilder.go diff --git a/toolkit/tools/imagegen/diskutils/diskutils.go b/toolkit/tools/imagegen/diskutils/diskutils.go index 4a77f15d4dc..62672747d92 100644 --- a/toolkit/tools/imagegen/diskutils/diskutils.go +++ b/toolkit/tools/imagegen/diskutils/diskutils.go @@ -20,6 +20,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/retry" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/sirupsen/logrus" ) var ( @@ -329,14 +330,12 @@ func BlockOnDiskIOByIds(debugName string, maj string, min string) (err error) { ) // Find the entry with Major#, Minor#, ..., IOs which matches our disk - onStdout := func(args ...interface{}) { - + onStdout := func(line string) { // Bail early if we already found the entry if foundEntry { return } - line := args[0].(string) deviceStatsFields := strings.Fields(line) if maj == deviceStatsFields[majIdx] && min == deviceStatsFields[minIdx] { outstandingOps = deviceStatsFields[outstandingOpsIdx] @@ -344,7 +343,11 @@ func BlockOnDiskIOByIds(debugName string, maj string, min string) (err error) { } } - err = shell.ExecuteLiveWithCallback(onStdout, logger.Log.Error, true, "cat", "/proc/diskstats") + err = shell.NewExecBuilder("cat", "/proc/diskstats"). + StdoutCallback(onStdout). + WarnLogLines(shell.DefaultWarnLogLines). + LogLevel(logrus.TraceLevel, logrus.ErrorLevel). + Execute() if err != nil { return } diff --git a/toolkit/tools/imagegen/installutils/installutils.go b/toolkit/tools/imagegen/installutils/installutils.go index 0416c4dd9b3..54112c6e632 100644 --- a/toolkit/tools/imagegen/installutils/installutils.go +++ b/toolkit/tools/imagegen/installutils/installutils.go @@ -32,6 +32,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/tdnf" "github.com/microsoft/azurelinux/toolkit/tools/internal/timestamp" "github.com/microsoft/azurelinux/toolkit/tools/internal/userutils" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -636,15 +637,10 @@ func TdnfInstallWithProgress(packageName, installRoot string, currentPackagesIns packagesInstalled = currentPackagesInstalled - onStdout := func(args ...interface{}) { + onStdout := func(line string) { const tdnfInstallPrefix = "Installing/Updating: " // Only process lines that match tdnfInstallPrefix - if len(args) == 0 { - return - } - - line := args[0].(string) if !strings.HasPrefix(line, tdnfInstallPrefix) { return } @@ -673,9 +669,12 @@ func TdnfInstallWithProgress(packageName, installRoot string, currentPackagesIns } // TDNF 3.x uses repositories from installchroot instead of host. Passing setopt for repo files directory to use local repo for installroot installation - err = shell.ExecuteLiveWithCallback(onStdout, logger.Log.Warn, true, "tdnf", "-v", "install", packageName, - "--installroot", installRoot, "--nogpgcheck", "--assumeyes", "--setopt", "reposdir=/etc/yum.repos.d/", - releaseverCliArg) + err = shell.NewExecBuilder("tdnf", "-v", "install", packageName, "--installroot", installRoot, "--nogpgcheck", + "--assumeyes", "--setopt", "reposdir=/etc/yum.repos.d/", releaseverCliArg). + StdoutCallback(onStdout). + LogLevel(logrus.TraceLevel, logrus.WarnLevel). + WarnLogLines(shell.DefaultWarnLogLines). + Execute() if err != nil { logger.Log.Warnf("Failed to tdnf install: %v. Package name: %v", err, packageName) } @@ -1824,7 +1823,6 @@ func SELinuxUpdateConfig(selinuxMode configuration.SELinux, installChroot *safec func SELinuxRelabelFiles(installChroot *safechroot.Chroot, mountPointToFsTypeMap map[string]string, isRootFS bool, ) (err error) { const ( - squashErrors = false fileContextBasePath = "etc/selinux/%s/contexts/files/file_contexts" ) var listOfMountsToLabel []string @@ -1884,17 +1882,17 @@ func SELinuxRelabelFiles(installChroot *safechroot.Chroot, mountPointToFsTypeMap // We only want to print basic info, filter out the real output unless at trace level (Execute call handles that) files := 0 lastFile := "" - onStdout := func(args ...interface{}) { - if len(args) > 0 { - files++ - lastFile = fmt.Sprintf("%v", args) - } + onStdout := func(line string) { + files++ + lastFile = line if (files % 1000) == 0 { ReportActionf("SELinux: labelled %d files", files) } } - err := shell.ExecuteLiveWithCallback(onStdout, logger.Log.Warn, squashErrors, "setfiles", "-m", "-v", "-r", - targetRootPath, fileContextPath, targetPath) + err := shell.NewExecBuilder("setfiles", "-m", "-v", "-r", targetRootPath, fileContextPath, targetPath). + StdoutCallback(onStdout). + LogLevel(logrus.TraceLevel, logrus.WarnLevel). + Execute() if err != nil { return fmt.Errorf("failed while labeling files (last file: %s) %w", lastFile, err) } diff --git a/toolkit/tools/internal/logger/log.go b/toolkit/tools/internal/logger/log.go index b79b7ccebeb..4dcd535629e 100644 --- a/toolkit/tools/internal/logger/log.go +++ b/toolkit/tools/internal/logger/log.go @@ -6,14 +6,12 @@ package logger import ( - "bufio" "io" "log" "os" "path/filepath" "runtime" "strings" - "sync" "github.com/sirupsen/logrus" ) @@ -172,36 +170,6 @@ func WarningOnError(err interface{}, args ...interface{}) { } } -// StreamOutput calls the provided logFunction on every line from the provided pipe -// outputChan will contain the N most recent lines of output, based on the length of the channel -func StreamOutput(pipe io.Reader, logFunction func(...interface{}), wg *sync.WaitGroup, outputChan chan string) { - for scanner := bufio.NewScanner(pipe); scanner.Scan(); { - line := scanner.Text() - logFunction(line) - - Log.Tracef("StreamOutput:\t'%s'", line) - - // Optionally buffer the output to print in the event of an error - if outputChan != nil { - // We are most interested in the most recent messages, if the channel is full drop the oldest entries - if len(outputChan) == cap(outputChan) { - select { - case <-outputChan: - // The buffer is full, discard the oldest value - default: - } - } - select { - case outputChan <- line: - default: - // In the event the buffer is full, drop the line. The block above should avoid this occuring however - } - } - } - - wg.Done() -} - // ReplaceStderrWriter replaces the stderr writer and returns the old one func ReplaceStderrWriter(newOut io.Writer) (oldOut io.Writer) { return stderrHook.ReplaceWriter(newOut) diff --git a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go index bea7301d383..351e5894599 100644 --- a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go +++ b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go @@ -20,6 +20,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" "github.com/microsoft/azurelinux/toolkit/tools/internal/tdnf" "github.com/microsoft/azurelinux/toolkit/tools/internal/timestamp" + "github.com/sirupsen/logrus" ) // RepoFlag* flags are used to denote which repos the cloner is allowed to use for its queries. @@ -535,12 +536,7 @@ func (r *RpmRepoCloner) ClonedRepoContents() (repoContents *repocloner.RepoConte // and we don't want to list them twice. foundPackages := map[string]bool{} repoContents = &repocloner.RepoContents{} - onStdout := func(args ...interface{}) { - if len(args) == 0 { - return - } - - line := args[0].(string) + onStdout := func(line string) { matches := tdnf.ListedPackageRegex.FindStringSubmatch(line) if len(matches) != tdnf.ListMaxMatchLen { return @@ -575,7 +571,11 @@ func (r *RpmRepoCloner) ClonedRepoContents() (repoContents *repocloner.RepoConte releaseverCliArg, } - return shell.ExecuteLiveWithCallback(onStdout, logger.Log.Warn, true, "tdnf", tdnfArgs...) + return shell.NewExecBuilder("tdnf", tdnfArgs...). + StdoutCallback(onStdout). + LogLevel(logrus.TraceLevel, logrus.WarnLevel). + WarnLogLines(shell.DefaultWarnLogLines). + Execute() }) return diff --git a/toolkit/tools/internal/packagerepo/repoutils/repoquery.go b/toolkit/tools/internal/packagerepo/repoutils/repoquery.go index 6bccb95fa73..f4a8b46ba0b 100644 --- a/toolkit/tools/internal/packagerepo/repoutils/repoquery.go +++ b/toolkit/tools/internal/packagerepo/repoutils/repoquery.go @@ -17,6 +17,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" "github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils" "github.com/microsoft/azurelinux/toolkit/tools/internal/timestamp" + "github.com/sirupsen/logrus" ) const ( @@ -199,7 +200,6 @@ func getPackageRepoPathsFromUrl(repoUrl string) (packageURLs []string, err error const ( reqoqueryTool = "repoquery" randomNameLength = 10 - printErrorOutput = true ) var queryCommonArgList = []string{"-y", "-q", "--disablerepo=*", "-a", "--location"} @@ -214,13 +214,16 @@ func getPackageRepoPathsFromUrl(repoUrl string) (packageURLs []string, err error repoPathArg := fmt.Sprintf("--repofrompath=mariner-precache-%s,%s", randomName, repoUrl) finalArgList := append(queryCommonArgList, repoPathArg) - onStdout := func(args ...interface{}) { - line := args[0].(string) + onStdout := func(line string) { packageURLs = append(packageURLs, line) } // Run the repoquery command - err = shell.ExecuteLiveWithCallback(onStdout, logger.Log.Warn, printErrorOutput, reqoqueryTool, finalArgList...) + err = shell.NewExecBuilder(reqoqueryTool, finalArgList...). + WarnLogLines(shell.DefaultWarnLogLines). + LogLevel(logrus.TraceLevel, logrus.WarnLevel). + StdoutCallback(onStdout). + Execute() if err != nil { err = fmt.Errorf("failed to run repoquery command:\n%w", err) return @@ -234,20 +237,22 @@ func getPackageRepoUrlsFromRepoFiles() (packageURLs []string, err error) { const ( reqoqueryTool = "repoquery" randomNameLength = 10 - printErrorOutput = true ) // We have removed all other repo files from the chroot, so we can blindly enable all repos to get the full list of packages var queryCommonArgList = []string{"-y", "-q", "--enablerepo=*", "-a", "--location"} logger.Log.Info("Getting package data from repo files") - onStdout := func(args ...interface{}) { - line := args[0].(string) + onStdout := func(line string) { packageURLs = append(packageURLs, line) } // Run the repoquery command - err = shell.ExecuteLiveWithCallback(onStdout, logger.Log.Warn, printErrorOutput, reqoqueryTool, queryCommonArgList...) + err = shell.NewExecBuilder(reqoqueryTool, queryCommonArgList...). + WarnLogLines(shell.DefaultWarnLogLines). + LogLevel(logrus.TraceLevel, logrus.WarnLevel). + StdoutCallback(onStdout). + Execute() if err != nil { err = fmt.Errorf("failed to run repoquery command:\n%w", err) return diff --git a/toolkit/tools/internal/rpm/rpm.go b/toolkit/tools/internal/rpm/rpm.go index e5a3532530b..81ecb31c72e 100644 --- a/toolkit/tools/internal/rpm/rpm.go +++ b/toolkit/tools/internal/rpm/rpm.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/microsoft/azurelinux/toolkit/tools/internal/exe" - "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" "github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils" @@ -148,26 +147,6 @@ func GetBasePackageNameFromSpecFile(specPath string) (basePackageName string, er return } -// SetMacroDir adds RPM_CONFIGDIR=$(newMacroDir) into the shell's environment for the duration of a program. -// To restore the environment the caller can use shell.SetEnvironment() with the returned origenv. -// On an empty string argument return success immediately and do not modify the environment. -func SetMacroDir(newMacroDir string) (origenv []string, err error) { - origenv = shell.CurrentEnvironment() - if newMacroDir == "" { - return - } - exists, err := file.DirExists(newMacroDir) - if err != nil || exists == false { - err = fmt.Errorf("directory (%s) does not exist", newMacroDir) - return - } - - env := append(shell.CurrentEnvironment(), fmt.Sprintf("RPM_CONFIGDIR=%s", newMacroDir)) - shell.SetEnvironment(env) - - return -} - // ExtractNameFromRPMPath strips the version from an RPM file name. i.e. pkg-name-1.2.3-4.cm2.x86_64.rpm -> pkg-name func ExtractNameFromRPMPath(rpmFilePath string) (packageName string, err error) { baseName := filepath.Base(rpmFilePath) diff --git a/toolkit/tools/internal/safechroot/safechroot.go b/toolkit/tools/internal/safechroot/safechroot.go index aac2868f6c2..3ad4f5709b1 100644 --- a/toolkit/tools/internal/safechroot/safechroot.go +++ b/toolkit/tools/internal/safechroot/safechroot.go @@ -763,15 +763,14 @@ func killGPGComponents(componentsToKill []string, availableComponents map[string // listGPGComponents will return a set of all GPG component. func listGPGComponents() (components map[string]bool, err error) { - stdout, stderr, err := shell.Execute("gpgconf", "--list-components") - + stdout, stderr, err := shell.NewExecBuilder("gpgconf", "--list-components"). + LogLevel(logrus.DebugLevel, logrus.DebugLevel). + ExecuteCaptureOuput() if err != nil { err = fmt.Errorf("failed to list GPG components.\nerr:%w\nstderr: %s", err, stderr) return } - logger.Log.Debugf("gpgconf --list-components output:\n%s", stdout) - components = make(map[string]bool) // Split --list-components stdout into a list of name tags, one for each component diff --git a/toolkit/tools/internal/shell/execbuilder.go b/toolkit/tools/internal/shell/execbuilder.go new file mode 100644 index 00000000000..57b140bb1dd --- /dev/null +++ b/toolkit/tools/internal/shell/execbuilder.go @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package shell + +import ( + "bufio" + "fmt" + "io" + "math" + "os/exec" + "strings" + "sync" + + "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" + "github.com/sirupsen/logrus" +) + +const ( + // LogDisabledLevel is a fake logrus log level, that is used by ExecBuilder to represent that logging should be + // disabled. + LogDisabledLevel logrus.Level = math.MaxUint32 + + // DefaultWarnLogLines is a default value that can be used with the WarnLogLines function. + DefaultWarnLogLines int = 1500 +) + +type LogCallback func(line string) + +type ExecBuilder struct { + command string + args []string + workingDirectory string + environmentVariables []string + stdinString string + stdoutLogLevel logrus.Level + stderrLogLevel logrus.Level + stdoutCallback LogCallback + stderrCallback LogCallback + errorStderrLines int + warnLogLines int +} + +// NewExecBuilder initializes a new execution builder object. +func NewExecBuilder(command string, args ...string) ExecBuilder { + b := ExecBuilder{ + command: command, + args: args, + stdoutLogLevel: logrus.DebugLevel, + stderrLogLevel: logrus.DebugLevel, + } + return b +} + +// WorkingDirectory sets the working directory for the command to be executed. +func (b ExecBuilder) WorkingDirectory(path string) ExecBuilder { + b.workingDirectory = path + return b +} + +// EnvironmentVariables sets the complete list of environment variables for the command to be executed. +func (b ExecBuilder) EnvironmentVariables(environmentVariables []string) ExecBuilder { + b.environmentVariables = environmentVariables + return b +} + +// Stdin sets a string value to be passed to the process via stdin. +func (b ExecBuilder) Stdin(value string) ExecBuilder { + b.stdinString = value + return b +} + +// Sets the log level for stdout lines. +func (b ExecBuilder) StdoutLogLevel(stdoutLogLevel logrus.Level) ExecBuilder { + b.stdoutLogLevel = stdoutLogLevel + return b +} + +// Sets the log level for stderr lines. +func (b ExecBuilder) StderrLogLevel(stderrLogLevel logrus.Level) ExecBuilder { + b.stderrLogLevel = stderrLogLevel + return b +} + +// Sets the log level for stdout and stderr lines. +func (b ExecBuilder) LogLevel(stdoutLogLevel logrus.Level, stderrLogLevel logrus.Level) ExecBuilder { + b.stdoutLogLevel = stdoutLogLevel + b.stderrLogLevel = stderrLogLevel + return b +} + +// ErrorStderrLines sets the number of stderr lines to add to the error object, if the execution fails. +func (b ExecBuilder) ErrorStderrLines(lines int) ExecBuilder { + b.errorStderrLines = lines + return b +} + +// WarnLogLines sets the number of stdout/stderr lines that will be printed as warning logs if the process returns an +// error. +// +// Note: This function exists for the sake of compatability with existing code. It is generally preferable to set the +// stdout and stderr log levels to an appropriate value. +func (b ExecBuilder) WarnLogLines(lines int) ExecBuilder { + b.warnLogLines = lines + return b +} + +// StdoutCallback sets a callback function that it called for each line of stdout. +func (b ExecBuilder) StdoutCallback(stdoutCallback LogCallback) ExecBuilder { + b.stdoutCallback = stdoutCallback + return b +} + +// StderrCallback sets a callback function that it called for each line of stderr. +func (b ExecBuilder) StderrCallback(stderrCallback LogCallback) ExecBuilder { + b.stderrCallback = stderrCallback + return b +} + +// Callbacks sets the callback functions for both stdout and stderr. +func (b ExecBuilder) Callbacks(stdoutCallback LogCallback, stderrCallback LogCallback) ExecBuilder { + b.stdoutCallback = stdoutCallback + b.stderrCallback = stderrCallback + return b +} + +func (b ExecBuilder) Execute() error { + _, _, err := b.executeHelper(false /*captureOutput*/) + return err +} + +func (b ExecBuilder) ExecuteCaptureOuput() (string, string, error) { + return b.executeHelper(true /*captureOutput*/) +} + +func (b ExecBuilder) executeHelper(captureOutput bool) (string, string, error) { + stdoutLinesChans := []chan string(nil) + stdErrLinesChans := []chan string(nil) + + var warnLogChan chan string + if b.warnLogLines > 0 { + // Setup WarnLogLines. + warnLogChan = make(chan string, b.warnLogLines) + stdoutLinesChans = append(stdoutLinesChans, warnLogChan) + stdErrLinesChans = append(stdErrLinesChans, warnLogChan) + } + + var errorChan chan string + if b.errorStderrLines > 0 { + // Setup ErrorStderrLines. + errorChan = make(chan string, b.errorStderrLines) + stdErrLinesChans = append(stdErrLinesChans, errorChan) + } + + stdoutResultChan := chan string(nil) + stderrResultChan := chan string(nil) + if captureOutput { + // Setup output capture. + stdoutResultChan = make(chan string, 1) + stderrResultChan = make(chan string, 1) + } + + // Setup process. + cmd := exec.Command(b.command, b.args...) + cmd.Dir = b.workingDirectory + cmd.Env = b.environmentVariables + + if b.stdinString != "" { + cmd.Stdin = strings.NewReader(b.stdinString) + } + + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + err = fmt.Errorf("failed to open stdout pipe:\n%w", err) + return "", "", err + } + defer stdoutPipe.Close() + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + err = fmt.Errorf("failed to open stderr pipe:\n%w", err) + return "", "", err + } + defer stderrPipe.Close() + + // Start process. + err = trackAndStartProcess(cmd) + if err != nil { + err = fmt.Errorf("failed to start process:\n%w", err) + return "", "", err + } + + defer untrackProcess(cmd) + + // Read stdout and stderr. + wg := new(sync.WaitGroup) + wg.Add(2) + go execBuilderReadPipe(stdoutPipe, wg, b.stdoutCallback, b.stdoutLogLevel, stdoutLinesChans, stdoutResultChan) + go execBuilderReadPipe(stderrPipe, wg, b.stderrCallback, b.stderrLogLevel, stdErrLinesChans, stderrResultChan) + + // Wait for process to exit. + wg.Wait() + err = cmd.Wait() + + // Cleanup the WarnLogLines and ErrorStderrLines channels. + // Note: While technically senders are suppose to close channels, it is ok to do it here because of the use of the + // waitgroup (wg). + if warnLogChan != nil { + close(warnLogChan) + } + + if errorChan != nil { + close(errorChan) + } + + stdout := "" + stderr := "" + if captureOutput { + // Get the string values of stdout and stderr. + stdout = <-stdoutResultChan + stderr = <-stderrResultChan + } + + if err != nil { + if warnLogChan != nil { + // Report last x lines of process's output (stdout and stderr) as warning logs. + logger.Log.Errorf("Call to %s returned error, last %d lines of output:", b.command, b.warnLogLines) + for line := range warnLogChan { + logger.Log.Warn(line) + } + } + + if errorChan != nil { + // Add last x line from stderr to the error message. + builder := strings.Builder{} + for errLine := range errorChan { + if builder.Len() > 0 { + builder.WriteString("\n") + } + builder.WriteString(errLine) + } + + errLines := builder.String() + if errLines != "" { + err = fmt.Errorf("%s\n%w", errLines, err) + } + } + } + + return stdout, stderr, err +} + +func execBuilderReadPipe(pipe io.Reader, wg *sync.WaitGroup, logCallback LogCallback, logLevel logrus.Level, + linesOutputChans []chan string, outputResultChan chan string, +) { + defer wg.Done() + + outputBuilder := strings.Builder{} + + reader := bufio.NewReader(pipe) + for { + // Read up to the next line. + bytes, err := reader.ReadBytes('\n') + + // Drop \n or \r\n from line. + omitBytes := 0 + if len(bytes) >= 1 && bytes[len(bytes)-1] == '\n' { + omitBytes = 1 + if len(bytes) >= 2 && bytes[len(bytes)-2] == '\r' { + omitBytes = 2 + } + } + + line := string(bytes[:len(bytes)-omitBytes]) + + if logCallback != nil { + // Call user callback. + logCallback(line) + } + + lineIsBlank := strings.TrimSpace(line) == "" + lastLine := err != nil + + // Most command-line tools will add a blank line at the of the stdout/stderr. + // We don't need such lines in our own logs. + if !lastLine || !lineIsBlank { + if logLevel <= logrus.TraceLevel { + // Log the line. + logger.Log.Log(logLevel, line) + } + + for _, linesOutputChan := range linesOutputChans { + channelDropAndPush(line, linesOutputChan) + } + } + + if outputResultChan != nil { + // Collect the entire stream into a single string. + outputBuilder.Write(bytes) + } + + if err != nil { + break + } + } + + if outputResultChan != nil { + // Return the full stream as a string. + output := outputBuilder.String() + outputResultChan <- output + close(outputResultChan) + } +} + +// channelDropAndPush treats a channel as a circular buffer. +func channelDropAndPush(line string, outputChan chan string) { + const maxRetries = 8 + + for i := 0; i < maxRetries; i++ { + if len(outputChan) == cap(outputChan) { + // The buffer is full, discard the oldest value. + select { + case <-outputChan: + default: + } + } + + select { + case outputChan <- line: + // Line was pushed. + return + + default: + // The event buffer is full, presumably from another goroutine pushing an entry. + // So, loop back around and try again. + } + } +} diff --git a/toolkit/tools/internal/shell/shell.go b/toolkit/tools/internal/shell/shell.go index 70a2655dc74..708c98e5334 100644 --- a/toolkit/tools/internal/shell/shell.go +++ b/toolkit/tools/internal/shell/shell.go @@ -12,6 +12,7 @@ import ( "sync" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -72,174 +73,38 @@ func PermanentlyStopAllChildProcesses(signal unix.Signal) { // Execute runs the provided command. func Execute(program string, args ...string) (stdout, stderr string, err error) { - return ExecuteInDirectory("", program, args...) -} - -// Execute runs the provided command in a specific working directory. -func ExecuteInDirectory(workingDirectory, program string, args ...string) (stdout, stderr string, err error) { - var ( - outBuf bytes.Buffer - errBuf bytes.Buffer - ) - - cmd := exec.Command(program, args...) - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - - if workingDirectory != "" { - cmd.Dir = workingDirectory - } - - err = trackAndStartProcess(cmd) - if err != nil { - return - } - - defer untrackProcess(cmd) - - err = cmd.Wait() - return outBuf.String(), errBuf.String(), err + return NewExecBuilder(program, args...). + LogLevel(logrus.TraceLevel, logrus.DebugLevel). + ExecuteCaptureOuput() } // ExecuteWithStdin - Run the command and use Stdin to pass input during execution func ExecuteWithStdin(input, program string, args ...string) (stdout, stderr string, err error) { - var ( - outBuf bytes.Buffer - errBuf bytes.Buffer - ) - - cmd := exec.Command(program, args...) - cmd.Stdout = &outBuf - cmd.Stderr = &errBuf - cmd.Stdin = strings.NewReader(input) - - err = trackAndStartProcess(cmd) - if err != nil { - return - } - - defer untrackProcess(cmd) - - err = cmd.Wait() - return outBuf.String(), errBuf.String(), err + return NewExecBuilder(program, args...). + LogLevel(logrus.TraceLevel, logrus.DebugLevel). + Stdin(input). + ExecuteCaptureOuput() } // ExecuteLive runs a command in the shell and logs it in real-time func ExecuteLive(squashErrors bool, program string, args ...string) (err error) { - var ( - onStdout func(...interface{}) - onStderr func(...interface{}) - ) + b := NewExecBuilder(program, args...). + LogLevel(logrus.DebugLevel, logrus.DebugLevel) - onStdout = logger.Log.Debug - if squashErrors { - onStderr = logger.Log.Debug - } else { - onStderr = logger.Log.Warn + if !squashErrors { + b = b.StderrLogLevel(logrus.WarnLevel) } - return ExecuteLiveWithCallback(onStdout, onStderr, false, program, args...) + return b.Execute() } // ExecuteLiveWithErr runs a command in the shell and logs it in real-time. // In addition, if there is an error, the last x lines of stderr will be attached to the err object. func ExecuteLiveWithErr(stderrLines int, program string, args ...string) (err error) { - return ExecuteLiveWithErrAndCallbacks(stderrLines, logger.Log.Debug, logger.Log.Debug, program, args...) -} - -// ExecuteLiveWithErr runs a command in the shell and logs it in real-time. -// In addition, if there is an error, the last x lines of stderr will be attached to the err object. -func ExecuteLiveWithErrAndCallbacks(stderrLines int, onStdout, onStderr func(...interface{}), program string, - args ...string, -) (err error) { - stderrChan := make(chan string, stderrLines) - - err = ExecuteLiveWithCallbackAndChannels(onStdout, onStderr, nil, stderrChan, program, args...) - close(stderrChan) - if err != nil { - errLines := "" - for errLine := range stderrChan { - if errLines != "" { - errLines += "\n" - } - errLines += errLine - } - - if errLines != "" { - err = fmt.Errorf("%s\n%w", errLines, err) - } - return - } - return nil -} - -// ExecuteLiveWithCallback runs a command in the shell and invokes the provided callbacks in real-time on each line of stdout and stderr. -// If printOutputOnError is true, the full output of the command will be printed after completion if the command returns an error. In the event -// the buffer becomes full the oldest buffered output is discarded. -func ExecuteLiveWithCallback(onStdout, onStderr func(...interface{}), printOutputOnError bool, program string, args ...string) (err error) { - var outputChan chan string - const outputChanBufferSize = 1500 - - if printOutputOnError { - outputChan = make(chan string, outputChanBufferSize) - } - - err = ExecuteLiveWithCallbackAndChannels(onStdout, onStderr, outputChan, outputChan, program, args...) - if err != nil { - return - } - - // Optionally dump the output in the event of an error - if outputChan != nil { - close(outputChan) - } - if err != nil && printOutputOnError { - logger.Log.Errorf("Call to %s returned error, last %d lines of output:", program, outputChanBufferSize) - for line := range outputChan { - logger.Log.Warn(line) - } - } - - return -} - -func ExecuteLiveWithCallbackAndChannels(onStdout, onStderr func(...interface{}), - stdoutChannel, stderrChannel chan string, - program string, args ...string, -) (err error) { - cmd := exec.Command(program, args...) - - stdoutPipe, err := cmd.StdoutPipe() - if err != nil { - logger.Log.Error("ExecuteLive failed to start StdoutPipe ", err) - return - } - defer stdoutPipe.Close() - - stderrPipe, err := cmd.StderrPipe() - if err != nil { - logger.Log.Error("ExecuteLive failed to start StderrPipe ", err) - return - } - defer stderrPipe.Close() - - err = trackAndStartProcess(cmd) - if err != nil { - return - } - - defer untrackProcess(cmd) - - wg := new(sync.WaitGroup) - wg.Add(2) - - go logger.StreamOutput(stdoutPipe, onStdout, wg, stdoutChannel) - go logger.StreamOutput(stderrPipe, onStderr, wg, stderrChannel) - - wg.Wait() - err = cmd.Wait() - - return + return NewExecBuilder(program, args...). + LogLevel(logrus.DebugLevel, logrus.DebugLevel). + ErrorStderrLines(stderrLines). + Execute() } // ExecuteAndLogToFile runs a command in the shell and redirects stdout to the given file @@ -266,7 +131,6 @@ func ExecuteAndLogToFile(filepath string, command string, args ...string) { logger.Log.Errorf("Command '%s' failed with: '%s'. Error: '%s'", command, errBuf.String(), err) return } - return } // MustExecuteLive executes the shell command. @@ -283,7 +147,7 @@ func MustExecuteLive(command string, args ...string) { func trackAndStartProcess(cmd *exec.Cmd) (err error) { logger.Log.Debugf("Executing: %v", cmd.Args) - if len(currentEnv) > 0 { + if cmd.Env == nil && len(currentEnv) > 0 { cmd.Env = currentEnv } diff --git a/toolkit/tools/internal/userutils/userutils.go b/toolkit/tools/internal/userutils/userutils.go index a597756b200..6c213c69bca 100644 --- a/toolkit/tools/internal/userutils/userutils.go +++ b/toolkit/tools/internal/userutils/userutils.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/randomization" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/sirupsen/logrus" ) const ( @@ -40,7 +41,10 @@ func HashPassword(password string) (string, error) { // Generate hashed password based on salt value provided. // -6 option indicates to use the SHA256/SHA512 algorithm - stdout, _, err := shell.ExecuteWithStdin(password, "openssl", "passwd", "-6", "-salt", salt, "-stdin") + stdout, _, err := shell.NewExecBuilder("openssl", "passwd", "-6", "-salt", salt, "-stdin"). + Stdin(password). + LogLevel(shell.LogDisabledLevel, logrus.DebugLevel). + ExecuteCaptureOuput() if err != nil { return "", fmt.Errorf("failed to generate hashed password:\n%w", err) } diff --git a/toolkit/tools/liveinstaller/liveinstaller.go b/toolkit/tools/liveinstaller/liveinstaller.go index 042a5106158..0811c4984fb 100644 --- a/toolkit/tools/liveinstaller/liveinstaller.go +++ b/toolkit/tools/liveinstaller/liveinstaller.go @@ -20,6 +20,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/jsonutils" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" "gopkg.in/alecthomas/kingpin.v2" @@ -396,18 +397,12 @@ func terminalAttendedInstall(cfg configuration.Config, progress chan int, status return } - onStdout := func(args ...interface{}) { + onStdout := func(line string) { const ( progressPrefix = "progress:" actionPrefix = "action:" ) - if len(args) == 0 { - return - } - - line := args[0].(string) - if strings.HasPrefix(line, progressPrefix) { reportedProgress, err := strconv.Atoi(strings.TrimPrefix(line, progressPrefix)) if err != nil { @@ -423,7 +418,10 @@ func terminalAttendedInstall(cfg configuration.Config, progress chan int, status args.emitProgress = true program, commandArgs := formatImagerCommand(args) - err = shell.ExecuteLiveWithCallback(onStdout, logger.Log.Warn, false, program, commandArgs...) + err = shell.NewExecBuilder(program, commandArgs...). + LogLevel(logrus.TraceLevel, logrus.WarnLevel). + StdoutCallback(onStdout). + Execute() return } diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go b/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go index d4741e7737c..492731e21fc 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go @@ -647,7 +647,7 @@ func getGrub2ConfigFilePath(imageChroot *safechroot.Chroot) string { func regenerateInitrd(imageChroot *safechroot.Chroot) error { logger.Log.Infof("Regenerate initramfs file") - err := imageChroot.Run(func() error { + err := imageChroot.UnsafeRun(func() error { // The 'mkinitrd' command was removed in Azure Linux 3.0 in favor of using 'dracut' directly. mkinitrdExists, err := file.CommandExists("mkinitrd") if err != nil { diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizepackages.go b/toolkit/tools/pkg/imagecustomizerlib/customizepackages.go index 72b5731d708..60695be8371 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizepackages.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizepackages.go @@ -13,6 +13,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/sirupsen/logrus" ) const ( @@ -165,13 +166,7 @@ func installOrUpdatePackages(action string, allPackagesToAdd []string, imageChro func callTdnf(tnfArgs []string, tdnfMessagePrefix string, imageChroot *safechroot.Chroot) error { seenTransactionErrorMessage := false - stdoutCallback := func(args ...interface{}) { - if len(args) == 0 { - return - } - - line := args[0].(string) - + stdoutCallback := func(line string) { if !seenTransactionErrorMessage { // Check if this line marks the start of a transaction error message. seenTransactionErrorMessage = tdnfTransactionError.MatchString(line) @@ -182,11 +177,16 @@ func callTdnf(tnfArgs []string, tdnfMessagePrefix string, imageChroot *safechroo logger.Log.Warn(line) } else if strings.HasPrefix(line, tdnfMessagePrefix) { logger.Log.Debug(line) + } else { + logger.Log.Trace(line) } } - return imageChroot.Run(func() error { - return shell.ExecuteLiveWithErrAndCallbacks(1, stdoutCallback, logger.Log.Debug, "tdnf", - tnfArgs...) + return imageChroot.UnsafeRun(func() error { + return shell.NewExecBuilder("tdnf", tnfArgs...). + StdoutCallback(stdoutCallback). + LogLevel(shell.LogDisabledLevel, logrus.DebugLevel). + ErrorStderrLines(1). + Execute() }) } diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizepartitionsfilecopy.go b/toolkit/tools/pkg/imagecustomizerlib/customizepartitionsfilecopy.go index 79af44b2174..ed5c4e18e66 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizepartitionsfilecopy.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizepartitionsfilecopy.go @@ -7,9 +7,9 @@ import ( "fmt" "github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi" - "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/sirupsen/logrus" ) func customizePartitionsUsingFileCopy(buildDir string, baseConfigPath string, config *imagecustomizerapi.Config, @@ -56,7 +56,10 @@ func copyPartitionFiles(sourceRoot, targetRoot string) error { copyArgs := []string{"--verbose", "--no-clobber", "-a", "--no-dereference", "--sparse", "always", sourceRoot, targetRoot} - err := shell.ExecuteLiveWithErrAndCallbacks(1, func(...interface{}) {}, logger.Log.Debug, "cp", copyArgs...) + err := shell.NewExecBuilder("cp", copyArgs...). + LogLevel(logrus.TraceLevel, logrus.DebugLevel). + ErrorStderrLines(1). + Execute() if err != nil { return fmt.Errorf("failed to copy files:\n%w", err) } diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go index 98372d918a8..db3a06412c9 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go @@ -629,7 +629,7 @@ func (b *LiveOSIsoBuilder) generateInitrdImage(rootfsSourceDir, artifactsSourceD } initrdPathInChroot := "/initrd.img" - err = chroot.Run(func() error { + err = chroot.UnsafeRun(func() error { dracutParams := []string{ initrdPathInChroot, "--kver", b.artifacts.kernelVersion, diff --git a/toolkit/tools/roast/formats/ova.go b/toolkit/tools/roast/formats/ova.go index 1cf29b7b640..104af888b97 100644 --- a/toolkit/tools/roast/formats/ova.go +++ b/toolkit/tools/roast/formats/ova.go @@ -19,6 +19,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -110,14 +111,18 @@ func (o *Ova) Convert(input, output string, isInputFile bool) (err error) { logger.Log.Infof(`Converting "%s" to "%s"`, input, vmdkFilePath) - err = shell.ExecuteLiveWithCallback(logger.Log.Info, logger.Log.Warn, false, "qemu-img", "convert", "-f", "raw", input, "-O", "vmdk", vmdkFilePath) + err = shell.NewExecBuilder("qemu-img", "convert", "-f", "raw", input, "-O", "vmdk", vmdkFilePath). + LogLevel(logrus.InfoLevel, logrus.WarnLevel). + Execute() if err != nil { return err } // This step produces the disk1 vmdk and .mf files generated by ovftool from vmx template logger.Log.Debugf(`Expanding template "%s" to "%s"`, vmxFilePath, ovfFilePath) - err = shell.ExecuteLiveWithCallback(logger.Log.Info, logger.Log.Warn, false, "ovftool", vmxFilePath, ovfFilePath) + err = shell.NewExecBuilder("ovftool", vmxFilePath, ovfFilePath). + LogLevel(logrus.InfoLevel, logrus.WarnLevel). + Execute() if err != nil { return } @@ -157,7 +162,9 @@ func (o *Ova) Convert(input, output string, isInputFile bool) (err error) { vmdkDisk1FileName := strings.TrimSuffix(vmdkBase, filepath.Ext(vmdkBase)) + "-disk1.vmdk" vmdkDisk1FilePath := filepath.Join(filepath.Dir(vmdkFilePath), vmdkDisk1FileName) - err = shell.ExecuteLiveWithCallback(logger.Log.Info, logger.Log.Warn, false, "openssl", "sha1", "-out", mfFilePath, vmdkDisk1FilePath, ovfFilePath) + err = shell.NewExecBuilder("openssl", "sha1", "-out", mfFilePath, vmdkDisk1FilePath, ovfFilePath). + LogLevel(logrus.InfoLevel, logrus.WarnLevel). + Execute() if err != nil { return } @@ -169,19 +176,11 @@ func (o *Ova) Convert(input, output string, isInputFile bool) (err error) { vmdkDisk1FileBase := filepath.Base(vmdkDisk1FilePath) artifactsFolder, _ := filepath.Split(vmdkDisk1FilePath) - // cd into the common directory to tar files at the toplevel directory - logger.Log.Debugf("Changing directory to %s to run tar for OVA generation.", artifactsFolder) - currentPwd, err := os.Getwd() - if err != nil { - return - } - os.Chdir(artifactsFolder) - // OVA is just a tar archive with .ovf, .mf and other artifacts (disk) - err = shell.ExecuteLiveWithCallback(logger.Log.Info, logger.Log.Warn, false, "tar", "-cf", output, "--format=ustar", ovfFileBase, mfFileBase, vmdkDisk1FileBase) - - logger.Log.Debugf("Changing directory back to %s after running tar for OVA generation.", currentPwd) - os.Chdir(currentPwd) + err = shell.NewExecBuilder("tar", "-cf", output, "--format=ustar", ovfFileBase, mfFileBase, vmdkDisk1FileBase). + LogLevel(logrus.InfoLevel, logrus.WarnLevel). + WorkingDirectory(artifactsFolder). + Execute() // Check error from creating OVA after going back to the old WD if err != nil { diff --git a/toolkit/tools/scheduler/buildagents/chrootagent.go b/toolkit/tools/scheduler/buildagents/chrootagent.go index 12a22d1fd3a..fca6c47c0a1 100644 --- a/toolkit/tools/scheduler/buildagents/chrootagent.go +++ b/toolkit/tools/scheduler/buildagents/chrootagent.go @@ -8,8 +8,8 @@ import ( "path/filepath" "strings" - "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/sirupsen/logrus" ) // ChrootAgentFlag is the build-agent option for ChrootAgent. @@ -46,17 +46,16 @@ func (c *ChrootAgent) BuildPackage(basePackageName, inputFile, logName, outArch logFile = filepath.Join(c.config.LogDir, logName) var lastStdoutLine string - onStdout := func(args ...interface{}) { - if len(args) == 0 { - return - } - - lastStdoutLine = strings.TrimSpace(args[0].(string)) - logger.Log.Trace(lastStdoutLine) + onStdout := func(line string) { + lastStdoutLine = strings.TrimSpace(line) } args := serializeChrootBuildAgentConfig(c.config, basePackageName, inputFile, logFile, outArch, runCheck, dependencies) - err = shell.ExecuteLiveWithCallback(onStdout, logger.Log.Trace, true, c.config.Program, args...) + err = shell.NewExecBuilder(c.config.Program, args...). + StdoutCallback(onStdout). + LogLevel(logrus.TraceLevel, logrus.TraceLevel). + WarnLogLines(shell.DefaultWarnLogLines). + Execute() if err == nil && lastStdoutLine != "" { builtFiles = strings.Split(lastStdoutLine, delimiter) From 9b4c138d117f9abe44799b321b746b0c9fa3e9ff Mon Sep 17 00:00:00 2001 From: Elaheh Dehghani <108492863+ellie-di@users.noreply.github.com> Date: Mon, 20 May 2024 16:20:42 -0700 Subject: [PATCH 32/51] [CHERRY-PICK] Upgrade and move libdwarf from extended to core (#6827) (#9141) Co-authored-by: sindhu-karri <33163197+sindhu-karri@users.noreply.github.com> --- .../libdwarf/libdwarf.signatures.json | 5 -- SPECS/libdwarf/libdwarf.signatures.json | 5 ++ .../libdwarf/libdwarf.spec | 51 ++++++++++--------- SPECS/libdwarf/libdwarf_skip_test.patch | 24 +++++++++ cgmanifest.json | 4 +- 5 files changed, 59 insertions(+), 30 deletions(-) delete mode 100644 SPECS-EXTENDED/libdwarf/libdwarf.signatures.json create mode 100644 SPECS/libdwarf/libdwarf.signatures.json rename {SPECS-EXTENDED => SPECS}/libdwarf/libdwarf.spec (88%) create mode 100644 SPECS/libdwarf/libdwarf_skip_test.patch diff --git a/SPECS-EXTENDED/libdwarf/libdwarf.signatures.json b/SPECS-EXTENDED/libdwarf/libdwarf.signatures.json deleted file mode 100644 index e248eba2c56..00000000000 --- a/SPECS-EXTENDED/libdwarf/libdwarf.signatures.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "Signatures": { - "libdwarf-20200114.tar.gz": "cffd8d600ca3181a5194324c38d50f94deb197249b2dea92d18969a7eadd2c34" - } -} diff --git a/SPECS/libdwarf/libdwarf.signatures.json b/SPECS/libdwarf/libdwarf.signatures.json new file mode 100644 index 00000000000..9f0ef88da6b --- /dev/null +++ b/SPECS/libdwarf/libdwarf.signatures.json @@ -0,0 +1,5 @@ +{ + "Signatures": { + "libdwarf-0.9.0.tar.xz": "d3cad80a337276a7581bb90ebcddbd743484a99a959157c066dd30f7535db59b" + } +} diff --git a/SPECS-EXTENDED/libdwarf/libdwarf.spec b/SPECS/libdwarf/libdwarf.spec similarity index 88% rename from SPECS-EXTENDED/libdwarf/libdwarf.spec rename to SPECS/libdwarf/libdwarf.spec index 484265f25e6..a1fb5bde2ee 100644 --- a/SPECS-EXTENDED/libdwarf/libdwarf.spec +++ b/SPECS/libdwarf/libdwarf.spec @@ -1,15 +1,16 @@ Name: libdwarf -Version: 20200114 -Release: 3%{?dist} -Summary: Library to access the DWARF Debugging file format +Version: 0.9.0 +Release: 1%{?dist} +Summary: Library to access the DWARF Debugging file format -License: LGPLv2 +License: LGPL-2.1-only AND BSD-2-Clause-FreeBSD Vendor: Microsoft Corporation Distribution: Azure Linux -URL: http://www.prevanders.net/dwarf.html -Source0: http://www.prevanders.net/%{name}-%{version}.tar.gz +URL: https://www.prevanders.net/dwarf.html +Source0: https://www.prevanders.net/%{name}-%{version}.tar.xz +Patch0: libdwarf_skip_test.patch -BuildRequires: gcc binutils-devel elfutils-libelf-devel dos2unix +BuildRequires: gcc make python3 %description Library to access the DWARF debugging file format which supports @@ -18,7 +19,7 @@ and Fortran. Please see http://www.dwarfstd.org for DWARF specification. %package devel Summary: Library and header files of libdwarf -License: LGPLv2 +License: LGPL-2.1-only AND BSD-2-Clause-FreeBSD Requires: %{name} = %{version}-%{release} %description devel @@ -26,7 +27,7 @@ Development package containing library and header files of libdwarf. %package static Summary: Static libdwarf library -License: LGPLv2 +License: LGPL-2.1-only AND BSD-2-Clause-FreeBSD Requires: %{name}-devel = %{version}-%{release} %description static @@ -34,11 +35,11 @@ Static libdwarf library. %package tools Summary: Tools for accessing DWARF debugging information -License: GPLv2 +License: GPL-2.0-only AND BSD-2-Clause-FreeBSD Requires: %{name} = %{version}-%{release} %description tools -C++ version of dwarfdump (dwarfdump2) command-line utilities +C++ version of dwarfdump (dwarfdump2) command-line utilities to access DWARF debug information. @@ -55,19 +56,17 @@ sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool %install %make_install -mkdir %{buildroot}%{_includedir}/libdwarf -mv %{buildroot}%{_includedir}/*.h %{buildroot}%{_includedir}/libdwarf %check -LD_LIBRARY_PATH=$PWD/libdwarf/.libs %__make check +TZ=:America/Los_Angeles %__make check %files -%doc libdwarf/ChangeLog libdwarf/README -%license libdwarf/COPYING libdwarf/LIBDWARFCOPYRIGHT libdwarf/LGPL.txt -%{_libdir}/libdwarf.so.* -%exclude %{_datadir}/libdwarf +%doc src/lib/libdwarf/ChangeLog src/lib/libdwarf/README +%license src/lib/libdwarf/COPYING src/lib/libdwarf/LIBDWARFCOPYRIGHT src/lib/libdwarf/LGPL.txt +%{_libdir}/libdwarf.so.0 +%{_libdir}/libdwarf.so.0.* %files static @@ -75,21 +74,27 @@ LD_LIBRARY_PATH=$PWD/libdwarf/.libs %__make check %files devel -%doc libdwarf/*.pdf -%{_includedir}/libdwarf +%doc doc/*.pdf +%{_includedir}/libdwarf-0 %{_libdir}/libdwarf.so %exclude %{_libdir}/*.la +%{_libdir}/pkgconfig/libdwarf.pc %files tools -%doc dwarfdump/README dwarfdump/ChangeLog -%license dwarfdump/COPYING dwarfdump/DWARFDUMPCOPYRIGHT dwarfdump/GPL.txt +%license src/bin/dwarfdump/COPYING src/bin/dwarfdump/DWARFDUMPCOPYRIGHT src/bin/dwarfdump/GPL.txt %{_bindir}/dwarfdump %{_datadir}/dwarfdump/dwarfdump.conf %{_mandir}/man1/dwarfdump.1.gz %changelog +* Tue Jan 02 2024 Sindhu Karri - 0.9.0-1 +- Upgraded to 0.9.0 +- License verified +- Promoted package to CBL-Mariner Core repository +- Added patch libdwarf_skip_test.patch to skip two tests test_dwarfdumpPE.sh and test_dwarfdumpMacos.sh that require the packages to be installed in system repositories. Patch is added to avoid ptest failure due to issue https://github.com/davea42/libdwarf-code/issues/212 + * Fri Oct 15 2021 Pawel Winogrodzki - 20200114-3 - Initial CBL-Mariner import from Fedora 32 (license: MIT). @@ -313,7 +318,7 @@ LD_LIBRARY_PATH=$PWD/libdwarf/.libs %__make check - 0.20090324-4 - Adding _smp_mflags for libdwarf build - Move CFLAGS override from configure to make - + * Mon Mar 30 2009 - Suravee Suthikulpanit - 0.20090324-3 - Remove AutoreqProv no diff --git a/SPECS/libdwarf/libdwarf_skip_test.patch b/SPECS/libdwarf/libdwarf_skip_test.patch new file mode 100644 index 00000000000..c5151713302 --- /dev/null +++ b/SPECS/libdwarf/libdwarf_skip_test.patch @@ -0,0 +1,24 @@ +diff --git a/test/Makefile.am b/test/Makefile.am +index 2a2827e..b7c3ed7 100644 +--- a/test/Makefile.am ++++ b/test/Makefile.am +@@ -272,7 +272,7 @@ if HAVE_DWARFEXAMPLE + TESTS += test_debuglink-a.sh test_debuglink-b.sh + endif + endif +-TESTS += test_dwarfdumpLinux.sh test_dwarfdumpPE.sh test_dwarfdumpMacos.sh ++TESTS += test_dwarfdumpLinux.sh + if HAVE_DWARFEXAMPLE + TESTS += test_jitreaderdiff.sh + endif +@@ -297,8 +297,8 @@ test_debuglink-b.sh \ + dummyexecutable \ + dummyexecutable.debug \ + dummysourceignore \ +-test_dwarfdumpLinux.sh test_dwarfdumpMacos.sh \ +-test_dwarfdumpPE.sh test_dwarfdumpsetup.sh \ ++test_dwarfdumpLinux.sh \ ++test_dwarfdumpsetup.sh \ + test_dwarfdump.py \ + test_dwarf_leb.c \ + test_dwarf_tied.c \ diff --git a/cgmanifest.json b/cgmanifest.json index d72e4117387..c774d67ed11 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -9111,8 +9111,8 @@ "type": "other", "other": { "name": "libdwarf", - "version": "20200114", - "downloadUrl": "http://www.prevanders.net/libdwarf-20200114.tar.gz" + "version": "0.9.0", + "downloadUrl": "https://www.prevanders.net/libdwarf-0.9.0.tar.xz" } } }, From 62b6183dcfdcc3017522d1a1b78a97f127feea01 Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Mon, 20 May 2024 16:27:26 -0700 Subject: [PATCH 33/51] [AUTOPATCHER-CORE] Upgrade ansible to 2.17.0 to address CVEs (#9163) --- SPECS/ansible/ansible.signatures.json | 2 +- SPECS/ansible/ansible.spec | 5 ++++- cgmanifest.json | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/SPECS/ansible/ansible.signatures.json b/SPECS/ansible/ansible.signatures.json index 09590db9aee..2580eee18b9 100644 --- a/SPECS/ansible/ansible.signatures.json +++ b/SPECS/ansible/ansible.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "ansible-2.15.3.tar.gz": "1ce23850aefcee176931d3f4f190af8c87461e63b90d0a84a3188d30a83343c2" + "ansible-2.17.0.tar.gz": "8ade6a00bdc256c65dfb3c05e05cfffbbcb7f0a37a0cb978c2ab388e37416d14" } } diff --git a/SPECS/ansible/ansible.spec b/SPECS/ansible/ansible.spec index 764f830f2d7..cdb1ced1ece 100644 --- a/SPECS/ansible/ansible.spec +++ b/SPECS/ansible/ansible.spec @@ -1,6 +1,6 @@ Summary: Configuration-management, application deployment, cloud provisioning system Name: ansible -Version: 2.15.3 +Version: 2.17.0 Release: 1%{?dist} License: GPLv3+ Vendor: Microsoft Corporation @@ -47,6 +47,9 @@ python3 setup.py test %{python3_sitelib}/* %changelog +* Mon May 20 2024 CBL-Mariner Servicing Account - 2.17.0-1 +- Auto-upgrade to 2.17.0 - none + * Fri Oct 27 2023 CBL-Mariner Servicing Account - 2.15.3-1 - Auto-upgrade to 2.15.3 - Azure Linux 3.0 - package upgrades diff --git a/cgmanifest.json b/cgmanifest.json index c774d67ed11..52e0438a1c5 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -205,8 +205,8 @@ "type": "other", "other": { "name": "ansible", - "version": "2.15.3", - "downloadUrl": "https://github.com/ansible/ansible/archive/refs/tags/v2.15.3.tar.gz" + "version": "2.17.0", + "downloadUrl": "https://github.com/ansible/ansible/archive/refs/tags/v2.17.0.tar.gz" } } }, From 69f7ce85a6a8dbc780fb06603e5eeeb955ef9a42 Mon Sep 17 00:00:00 2001 From: AZaugg Date: Mon, 20 May 2024 18:52:40 -0700 Subject: [PATCH 34/51] Namespace httpd include files into its own directory (#6221) --- SPECS/httpd/httpd.spec | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SPECS/httpd/httpd.spec b/SPECS/httpd/httpd.spec index 6e5204f3bfc..2409076dace 100644 --- a/SPECS/httpd/httpd.spec +++ b/SPECS/httpd/httpd.spec @@ -3,7 +3,7 @@ Summary: The Apache HTTP Server Name: httpd Version: 2.4.58 -Release: 2%{?dist} +Release: 3%{?dist} License: Apache-2.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -153,6 +153,7 @@ Security (TLS) protocols. --enable-mpms-shared=all \ --enable-ssl \ --exec-prefix=%{_prefix} \ + --includedir=%{_includedir}/httpd \ --libexecdir=%{_libdir}/httpd/modules \ --prefix=%{_sysconfdir}/httpd \ --sysconfdir=%{_confdir}/httpd/conf \ @@ -274,6 +275,7 @@ fi %{_bindir}/apxs %{_bindir}/dbmmanage %{_mandir}/man1/apxs.1* +%{_includedir}/httpd/* %{_includedir}/* %{_rpmconfigdir}/macros.d/macros.httpd @@ -345,6 +347,9 @@ fi %{_libexecdir}/httpd-ssl-pass-dialog %changelog +* Thu May 09 2024 Andy Zaugg - 2.4.58-3 +- Namespace httpd-devel include files into a httpd directory + * Fri Apr 05 2024 Betty Lakes - 2.4.58-2 - Move from pcre to pcre2 From 4c9971e61822439199451274449dd6a5fbed377a Mon Sep 17 00:00:00 2001 From: Christopher Co <35273088+christopherco@users.noreply.github.com> Date: Tue, 21 May 2024 08:07:58 -0700 Subject: [PATCH 35/51] Update facter version to support Mariner (#9167) Co-authored-by: AZaugg --- SPECS-EXTENDED/facter/facter.signatures.json | 2 +- SPECS-EXTENDED/facter/facter.spec | 7 +++++-- cgmanifest.json | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/SPECS-EXTENDED/facter/facter.signatures.json b/SPECS-EXTENDED/facter/facter.signatures.json index bf724194bfa..9d27e165bfc 100644 --- a/SPECS-EXTENDED/facter/facter.signatures.json +++ b/SPECS-EXTENDED/facter/facter.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "facter-4.2.5.gem": "e88e3fa874c1c735779704d1a4dd69b255ad5e34c8912857864469a852cb3f8d" + "facter-4.2.13.gem": "a4f293b585176b080c8f10e9adb7a4d1cfd484268dfef518b162a0422450264c" } } diff --git a/SPECS-EXTENDED/facter/facter.spec b/SPECS-EXTENDED/facter/facter.spec index 50c4e540d2b..02c708f8d3e 100644 --- a/SPECS-EXTENDED/facter/facter.spec +++ b/SPECS-EXTENDED/facter/facter.spec @@ -11,8 +11,8 @@ %global debug_package %{nil} Name: facter -Version: 4.2.5 -Release: 2%{?dist} +Version: 4.2.13 +Release: 1%{?dist} Summary: Command and ruby library for gathering system information Vendor: Microsoft Corporation Distribution: Azure Linux @@ -100,6 +100,9 @@ GEM_HOME="%{buildroot}%{gem_dir}" %{buildroot}%{_bindir}/facter %doc %{gem_docdir} %changelog +* Tue May 07 2024 Andy Zaugg 4.2.13-1 +- Bumped version to facter version which has Mariner Linux Support + * Thu Dec 30 2021 Suresh Babu Chalamalasetty 4.2.5-2 - Initial CBL-Mariner import from Fedora 35 (license: MIT) - License verified diff --git a/cgmanifest.json b/cgmanifest.json index 52e0438a1c5..dcb02002e4b 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -3448,8 +3448,8 @@ "type": "other", "other": { "name": "facter", - "version": "4.2.5", - "downloadUrl": "https://downloads.puppetlabs.com/facter/facter-4.2.5.gem" + "version": "4.2.13", + "downloadUrl": "https://downloads.puppetlabs.com/facter/facter-4.2.13.gem" } } }, From 3d42d8a38cc9dfe64c8b695c287d06c7eedfca4d Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 21 May 2024 19:18:41 +0200 Subject: [PATCH 36/51] Bump golang version to 1.22.3-1, new bootstrap (#9022) --- SPECS/golang/golang.signatures.json | 5 +- SPECS/golang/golang.spec | 91 +++++++++++++++++------------ cgmanifest.json | 24 +------- 3 files changed, 58 insertions(+), 62 deletions(-) diff --git a/SPECS/golang/golang.signatures.json b/SPECS/golang/golang.signatures.json index 64362d8bbdc..40e4553bbd4 100644 --- a/SPECS/golang/golang.signatures.json +++ b/SPECS/golang/golang.signatures.json @@ -1,7 +1,8 @@ { "Signatures": { - "go.20230802.5.src.tar.gz": "56b9e0e0c3c13ca95d5efa6de4e7d49a9d190eca77919beff99d33cd3fa74e95", + "go1.22.3-20240507.3.src.tar.gz": "43d600d563ac00c2e9ca485691c26114b29496ec6f811431469c85f495df23c8", "go1.4-bootstrap-20171003.tar.gz": "f4ff5b5eb3a3cae1c993723f3eab519c5bae18866b5e5f96fe1102f0cb5c3e52", - "go.20240403.5.src.tar.gz": "0769b8d4684836eb0760d3434958b64a41f532ae369962ef9778c81906448c40" + "go.20230802.5.src.tar.gz": "56b9e0e0c3c13ca95d5efa6de4e7d49a9d190eca77919beff99d33cd3fa74e95", + "go.20240206.2.src.tar.gz": "7982e0011aa9ab95fd0530404060410af4ba57326d26818690f334fdcb6451cd" } } diff --git a/SPECS/golang/golang.spec b/SPECS/golang/golang.spec index 322969b3c94..0d957e9b9b2 100644 --- a/SPECS/golang/golang.spec +++ b/SPECS/golang/golang.spec @@ -1,8 +1,7 @@ -%global bootstrap_compiler_version 1.19.12-1 %global goroot %{_libdir}/golang %global gopath %{_datadir}/gocode +%global ms_go_filename go1.22.3-20240507.3.src.tar.gz %global ms_go_revision 1 -%global ms_go_buildid 20240403.5 %ifarch aarch64 %global gohostarch arm64 %else @@ -15,18 +14,23 @@ %define __find_requires %{nil} Summary: Go Name: golang -Version: 1.21.9 -Release: 2%{?dist} +Version: 1.22.3 +Release: 1%{?dist} License: BSD-3-Clause Vendor: Microsoft Corporation Distribution: Azure Linux Group: System Environment/Security URL: https://github.com/microsoft/go -Source0: https://github.com/microsoft/go/releases/download/v%{version}-%{ms_go_revision}/go.%{ms_go_buildid}.src.tar.gz -# Same content as https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz +Source0: https://github.com/microsoft/go/releases/download/v%{version}-%{ms_go_revision}/%{ms_go_filename} + +# bootstrap 00, same content as https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz Source1: https://github.com/microsoft/go/releases/download/v1.4.0-1/go1.4-bootstrap-20171003.tar.gz -Source2: https://github.com/microsoft/go/releases/download/%{bootstrap_compiler_version}/go.20230802.5.src.tar.gz Patch0: go14_bootstrap_aarch64.patch +# bootstrap 01 +Source2: https://github.com/microsoft/go/releases/download/v1.19.12-1/go.20230802.5.src.tar.gz +# bootstrap 02 +Source3: https://github.com/microsoft/go/releases/download/v1.20.14-1/go.20240206.2.src.tar.gz + Provides: %{name} = %{version} Provides: go = %{version}-%{release} Provides: msft-golang = %{version}-%{release} @@ -35,43 +39,50 @@ Provides: msft-golang = %{version}-%{release} Go is an open source programming language that makes it easy to build simple, reliable, and efficient software. %prep -# Setup go 1.4 bootstrap source +# Setup bootstrap source tar xf %{SOURCE1} --no-same-owner patch -Np1 --ignore-whitespace < %{PATCH0} +mv -v go go-bootstrap-00 -mv -v go go-bootstrap +tar xf %{SOURCE2} --no-same-owner +mv -v go go-bootstrap-01 + +tar xf %{SOURCE3} --no-same-owner +mv -v go go-bootstrap-02 %setup -q -n go %build -# (go >= 1.20 bootstraps with go >= 1.17) -# This condition makes go compiler >= 1.20 build a 3 step process: -# - Build the bootstrap compiler 1.4 (bootstrap bits in c) -# - Use the 1.4 compiler to build %{bootstrap_compiler_version} -# - Use the %{bootstrap_compiler_version} compiler to build go >= 1.20 compiler -# PS: Since go compiles fairly quickly, the extra overhead is arounnd 2-3 minutes +# go 1.4 bootstraps with C. +# go 1.20 bootstraps with go >= 1.17.13 +# go >= 1.22 bootstraps with go >= 1.20.14 +# +# These conditions make building the current go compiler from C a multistep +# process. Approximately once a year, the bootstrap requirement is moved +# forward, adding another step. +# +# PS: Since go compiles fairly quickly, the extra overhead is around 2-3 minutes # on a reasonable machine. -# Build go 1.4 bootstrap -pushd %{_topdir}/BUILD/go-bootstrap/src -CGO_ENABLED=0 ./make.bash -popd -mv -v %{_topdir}/BUILD/go-bootstrap %{_libdir}/golang -export GOROOT=%{_libdir}/golang - -# Use go1.4 bootstrap to compile go%{bootstrap_compiler_version} (bootstrap) -export GOROOT_BOOTSTRAP=%{_libdir}/golang -mkdir -p %{_topdir}/BUILD/go%{bootstrap_compiler_version} -tar xf %{SOURCE2} -C %{_topdir}/BUILD/go%{bootstrap_compiler_version} --strip-components=1 -pushd %{_topdir}/BUILD/go%{bootstrap_compiler_version}/src -CGO_ENABLED=0 ./make.bash -popd - -# Nuke the older go1.4 bootstrap -rm -rf %{_libdir}/golang - -# Make go%{bootstrap_compiler_version} as the new bootstrapper -mv -v %{_topdir}/BUILD/go%{bootstrap_compiler_version} %{_libdir}/golang +# Use prev bootstrap to compile next bootstrap. +function go_bootstrap() { + local bootstrap=$1 + local new_root=%{_topdir}/BUILD/go-bootstrap-${bootstrap} + ( + cd ${new_root}/src + CGO_ENABLED=0 ./make.bash + ) + # Nuke the older bootstrapper + rm -rf %{_libdir}/golang + # Install the new bootstrapper + mv -v $new_root %{_libdir}/golang + export GOROOT=%{_libdir}/golang + export GOROOT_BOOTSTRAP=%{_libdir}/golang +} + +go_bootstrap 00 +go_bootstrap 01 +go_bootstrap 02 # Build current go version export GOHOSTOS=linux @@ -82,9 +93,10 @@ export GOROOT="`pwd`" export GOPATH=%{gopath} export GOROOT_FINAL=%{_bindir}/go rm -f %{gopath}/src/runtime/*.c -pushd src -./make.bash --no-clean -popd +( + cd src + ./make.bash --no-clean +) %install @@ -141,6 +153,9 @@ fi %{_bindir}/* %changelog +* Tue May 07 2024 Davis Goodin - 1.22.3-1 +- Bump version to 1.22.3-1 + * Wed May 08 2024 Davis Goodin - 1.21.9-2 - Remove explicit Go env variable defaults diff --git a/cgmanifest.json b/cgmanifest.json index dcb02002e4b..e1f52594a60 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -4600,28 +4600,8 @@ "type": "other", "other": { "name": "golang", - "version": "1.18.8", - "downloadUrl": "https://golang.org/dl/go1.18.8.src.tar.gz" - } - } - }, - { - "component": { - "type": "other", - "other": { - "name": "golang", - "version": "1.20.10", - "downloadUrl": "https://golang.org/dl/go1.20.10.src.tar.gz" - } - } - }, - { - "component": { - "type": "other", - "other": { - "name": "golang", - "version": "1.21.9", - "downloadUrl": "https://github.com/microsoft/go/releases/download/v1.21.9-1/go.20240403.5.src.tar.gz" + "version": "1.22.3", + "downloadUrl": "https://github.com/microsoft/go/releases/download/v1.22.3-1/go1.22.3-20240507.3.src.tar.gz" } } }, From b1c1da134f4ea8e70b3373329be591914bf47cc2 Mon Sep 17 00:00:00 2001 From: sharath-srikanth-chellappa <115591284+sharath-srikanth-chellappa@users.noreply.github.com> Date: Tue, 21 May 2024 10:25:13 -0700 Subject: [PATCH 37/51] Add patch to add network interface renaming support for CAPM3 Met (#9066) Co-authored-by: liulanze Co-authored-by: Sharath Srikanth Chellappa --- ...rface-Renaming-Support-for-CAPM3-Met.patch | 470 ++++++++++++++++++ SPECS/cloud-init/cloud-init.spec | 8 +- 2 files changed, 476 insertions(+), 2 deletions(-) create mode 100644 SPECS/cloud-init/Add-Network-Interface-Renaming-Support-for-CAPM3-Met.patch diff --git a/SPECS/cloud-init/Add-Network-Interface-Renaming-Support-for-CAPM3-Met.patch b/SPECS/cloud-init/Add-Network-Interface-Renaming-Support-for-CAPM3-Met.patch new file mode 100644 index 00000000000..66134e7793b --- /dev/null +++ b/SPECS/cloud-init/Add-Network-Interface-Renaming-Support-for-CAPM3-Met.patch @@ -0,0 +1,470 @@ +From b2e6b0381f6cc23191053854ea7db5ac78c1ec82 Mon Sep 17 00:00:00 2001 +From: Vince Perri +Date: Thu, 22 Dec 2022 15:17:32 +0000 +Subject: [PATCH] Add Network Interface Renaming Support for CAPM3 + Metal3DataTemplates + +The CAPM3 Metal3DataTemplate specification doesn't allow the "name" attribute in +networkData.links.ethernets, but the OpenStack cloud-init provider +implementation uses this attribute to (re)name network interfaces. This means +that when using CAPM3 Metal3DataTemplates, renaming network interfaces isn't +possible. + +This patch fixes this by providing a means through which to rename network +interfaces by using the "id" attribute found in the CAPM3 Metal3DataTemplate +specification. This is a temporary fix until the "name" attribute is added to +the specification. +--- + cloudinit/sources/helpers/openstack.py | 11 +-- + .../sources/helpers/test_openstack.py | 8 +- + tests/unittests/sources/test_configdrive.py | 84 ++++++++++--------- + tests/unittests/test_net.py | 80 ++++++------------ + 4 files changed, 79 insertions(+), 104 deletions(-) + +diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py +index d2260baa0..f995ce4b1 100644 +--- a/cloudinit/sources/helpers/openstack.py ++++ b/cloudinit/sources/helpers/openstack.py +@@ -596,13 +596,14 @@ def convert_net_json(network_json=None, known_macs=None): + # present. The 'id' in the spec is currently implemented as the host + # nic's name, meaning something like 'tap-adfasdffd'. We do not want + # to name guest devices with such ugly names. ++ link_mac_addr = None + if "name" in link: + cfg["name"] = link["name"] +- +- link_mac_addr = None +- if link.get("ethernet_mac_address"): +- link_mac_addr = link.get("ethernet_mac_address").lower() +- link_id_info[link["id"]] = link_mac_addr ++ if link.get("ethernet_mac_address"): ++ link_mac_addr = link.get("ethernet_mac_address").lower() ++ link_id_info[link["id"]] = link_mac_addr ++ elif "name" not in link: ++ cfg["name"] = link["id"] + + curinfo = { + "name": cfg.get("name"), +diff --git a/tests/unittests/sources/helpers/test_openstack.py b/tests/unittests/sources/helpers/test_openstack.py +index ac8e2a354..143c12796 100644 +--- a/tests/unittests/sources/helpers/test_openstack.py ++++ b/tests/unittests/sources/helpers/test_openstack.py +@@ -42,9 +42,9 @@ class TestConvertNetJson(test_helpers.CiTestCase): + "version": 1, + "config": [ + { +- "mac_address": "fa:16:3e:9c:bf:3d", ++ "mac_address": None, + "mtu": None, +- "name": "eth0", ++ "name": "tapcd9f6d46-4a", + "subnets": [{"type": "dhcp4"}], + "type": "physical", + }, +@@ -94,9 +94,9 @@ class TestConvertNetJson(test_helpers.CiTestCase): + "version": 1, + "config": [ + { +- "mac_address": "fa:16:3e:9c:bf:3d", ++ "mac_address": None, + "mtu": None, +- "name": "eth0", ++ "name": "tapcd9f6d46-4a", + "subnets": [ + { + "type": "static", +diff --git a/tests/unittests/sources/test_configdrive.py b/tests/unittests/sources/test_configdrive.py +index 70da4812a..e0afa2936 100644 +--- a/tests/unittests/sources/test_configdrive.py ++++ b/tests/unittests/sources/test_configdrive.py +@@ -731,16 +731,16 @@ class TestNetJson(CiTestCase): + "version": 1, + "config": [ + { +- "mac_address": "fa:16:3e:69:b0:58", ++ "mac_address": None, + "mtu": None, +- "name": "enp0s1", ++ "name": "tap2ecc7709-b3", + "subnets": [{"type": "ipv6_dhcpv6-stateless"}], + "type": "physical", + }, + { +- "mac_address": "fa:16:3e:d4:57:ad", ++ "mac_address": None, + "mtu": None, +- "name": "enp0s2", ++ "name": "tap2f88d109-5b", + "subnets": [{"type": "ipv6_dhcpv6-stateful"}], + "type": "physical", + "accept-ra": True, +@@ -792,15 +792,15 @@ class TestNetJson(CiTestCase): + { + "subnets": [{"type": "dhcp4"}], + "type": "physical", +- "mac_address": "fa:16:3e:69:b0:58", +- "name": "enp0s1", ++ "mac_address": None, ++ "name": "tap2ecc7709-b3", + "mtu": None, + }, + { + "subnets": [{"type": "dhcp4"}], + "type": "physical", +- "mac_address": "fa:16:3e:d4:57:ad", +- "name": "enp0s2", ++ "mac_address": None, ++ "name": "tap2f88d109-5b", + "mtu": None, + }, + { +@@ -824,8 +824,8 @@ class TestNetJson(CiTestCase): + "version": 1, + "config": [ + { +- "name": "foo3", +- "mac_address": "fa:16:3e:ed:9a:59", ++ "name": "tap1a81968a-79", ++ "mac_address": None, + "mtu": None, + "type": "physical", + "subnets": [ +@@ -877,7 +877,7 @@ class TestConvertNetworkData(CiTestCase): + + def test_conversion_fills_names(self): + ncfg = openstack.convert_net_json(NETWORK_DATA, known_macs=KNOWN_MACS) +- expected = set(["nic0", "enp0s1", "enp0s2"]) ++ expected = set(["nic0", "tap2ecc7709-b3", "tap2f88d109-5b"]) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + +@@ -890,18 +890,20 @@ class TestConvertNetworkData(CiTestCase): + get_interfaces_by_mac.return_value = macs + + ncfg = openstack.convert_net_json(NETWORK_DATA) +- expected = set(["nic0", "ens1", "enp0s2"]) ++ expected = set(["nic0", "tap2ecc7709-b3", "tap2f88d109-5b"]) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + +- def test_convert_raises_value_error_on_missing_name(self): +- macs = {"aa:aa:aa:aa:aa:00": "ens1"} +- self.assertRaises( +- ValueError, +- openstack.convert_net_json, +- NETWORK_DATA, +- known_macs=macs, +- ) ++ # Commenting this function out since we have modified the code to always add ++ # a name irrespective of it is present in the link info or not. ++ # def test_convert_raises_value_error_on_missing_name(self): ++ # macs = {"aa:aa:aa:aa:aa:00": "ens1"} ++ # self.assertRaises( ++ # ValueError, ++ # openstack.convert_net_json, ++ # NETWORK_DATA, ++ # known_macs=macs, ++ # ) + + def test_conversion_with_route(self): + ncfg = openstack.convert_net_json( +@@ -935,7 +937,7 @@ class TestConvertNetworkData(CiTestCase): + for i in ncfg["config"]: + if i.get("type") == "physical": + physicals.add(i["name"]) +- self.assertEqual(physicals, set(("foo1", "foo2"))) ++ self.assertEqual(physicals, set(("tap77a0dc5b-72", "tap7d6b7bec-93"))) + + def test_bond_conversion(self): + # light testing of bond conversion and eni rendering of bond +@@ -961,24 +963,26 @@ class TestConvertNetworkData(CiTestCase): + ] + ) + self.assertEqual( +- sorted(["oeth0", "oeth1", "bond0", "bond0.602", "bond0.612"]), ++ sorted(["eth0", "eth1", "bond0", "bond0.602", "bond0.612"]), + interfaces, + ) + +- words = eni_rendering.split() +- # 'eth0' and 'eth1' are the ids. because their mac adresses +- # map to other names, we should not see them in the ENI +- self.assertNotIn("eth0", words) +- self.assertNotIn("eth1", words) ++ # Because we set the name to link["id"] if it is not encountered, ++ # we should see eth0 or eth1 in the eni rendering. Hence this check does not hold good. ++ # words = eni_rendering.split() ++ # self.assertNotIn("eth0", words) ++ # self.assertNotIn("eth1", words) + +- # oeth0 and oeth1 are the interface names for eni. +- # bond0 will be generated for the bond. Each should be auto. +- self.assertIn("auto oeth0", eni_rendering) +- self.assertIn("auto oeth1", eni_rendering) ++ # We should be seeing eth0 and eth1 as the names for the physical interfaces ++ # as we have named them based on the link id and not on the known_macs. ++ self.assertIn("auto eth0", eni_rendering) ++ self.assertIn("auto eth1", eni_rendering) + self.assertIn("auto bond0", eni_rendering) +- # The bond should have the given mac address +- pos = eni_rendering.find("auto bond0") +- self.assertIn(BOND_MAC, eni_rendering[pos:]) ++ ++ # Since we are setting the mac address for all interfaces to none, we ++ # are commenting the check down below. ++ # pos = eni_rendering.find("auto bond0") ++ # self.assertIn(BOND_MAC, eni_rendering[pos:]) + + def test_vlan(self): + # light testing of vlan config conversion and eni rendering +@@ -994,9 +998,9 @@ class TestConvertNetworkData(CiTestCase): + ) as f: + eni_rendering = f.read() + +- self.assertIn("iface enp0s1", eni_rendering) ++ self.assertIn("iface eth0", eni_rendering) + self.assertIn("address 10.0.1.5", eni_rendering) +- self.assertIn("auto enp0s1.602", eni_rendering) ++ self.assertIn("auto eth0.602", eni_rendering) + + def test_mac_addrs_can_be_upper_case(self): + # input mac addresses on rackspace may be upper case +@@ -1012,8 +1016,8 @@ class TestConvertNetworkData(CiTestCase): + + expected = { + "nic0": "fa:16:3e:05:30:fe", +- "enp0s1": "fa:16:3e:69:b0:58", +- "enp0s2": "fa:16:3e:d4:57:ad", ++ "tap2ecc7709-b3": None, ++ "tap2f88d109-5b": None, + } + self.assertEqual(expected, config_name2mac) + +@@ -1031,8 +1035,8 @@ class TestConvertNetworkData(CiTestCase): + + expected = { + "nic0": "fa:16:3e:05:30:fe", +- "enp0s1": "fa:16:3e:69:b0:58", +- "enp0s2": "fa:16:3e:d4:57:ad", ++ "tap2ecc7709-b3": None, ++ "tap2f88d109-5b": None, + } + self.assertEqual(expected, config_name2mac) + +diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py +index c5509536a..eb14c5db3 100644 +--- a/tests/unittests/test_net.py ++++ b/tests/unittests/test_net.py +@@ -534,13 +534,12 @@ OS_SAMPLES = [ + }, + "out_sysconfig_opensuse": [ + ( +- "etc/sysconfig/network/ifcfg-eth0", ++ "etc/sysconfig/network/ifcfg-tap1a81968a-79", + """ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=static + IPADDR=172.19.1.34 +-LLADDR=fa:16:3e:ed:9a:59 + NETMASK=255.255.252.0 + STARTMODE=auto + """.lstrip(), +@@ -564,25 +563,20 @@ dns = none + ), + ( + "etc/udev/rules.d/85-persistent-net-cloud-init.rules", +- "".join( +- [ +- 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', +- 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n', +- ] +- ), ++ # Since we do not set mac address, we are expecting the content to be nil ++ "", + ), + ], + "out_sysconfig_rhel": [ + ( +- "etc/sysconfig/network-scripts/ifcfg-eth0", ++ "etc/sysconfig/network-scripts/ifcfg-tap1a81968a-79", + """ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=none + DEFROUTE=yes +-DEVICE=eth0 ++DEVICE=tap1a81968a-79 + GATEWAY=172.19.3.254 +-HWADDR=fa:16:3e:ed:9a:59 + IPADDR=172.19.1.34 + NETMASK=255.255.252.0 + NM_CONTROLLED=no +@@ -610,12 +604,8 @@ dns = none + ), + ( + "etc/udev/rules.d/70-persistent-net.rules", +- "".join( +- [ +- 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', +- 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n', +- ] +- ), ++ # Since we do not set mac address, we are expecting the content to be nil ++ "", + ), + ], + "expected_network_manager": [ +@@ -623,23 +613,23 @@ dns = none + "".join( + [ + "etc/NetworkManager/system-connections", +- "/cloud-init-eth0.nmconnection", ++ "/cloud-init-tap1a81968a-79.nmconnection", + ] + ), + """ + # Generated by cloud-init. Changes will be lost. + + [connection] +-id=cloud-init eth0 +-uuid=1dd9a779-d327-56e1-8454-c65e2556c12c ++id=cloud-init tap1a81968a-79 ++uuid=2e85b264-dffb-5635-9b6c-616838eb1130 + autoconnect-priority=120 + type=ethernet ++interface-name=tap1a81968a-79 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] +-mac-address=FA:16:3E:ED:9A:59 + + [ipv4] + method=manual +@@ -695,14 +685,13 @@ route1=0.0.0.0/0,172.19.3.254 + }, + "out_sysconfig_opensuse": [ + ( +- "etc/sysconfig/network/ifcfg-eth0", ++ "etc/sysconfig/network/ifcfg-tap1a81968a-79", + """ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=static + IPADDR=172.19.1.34 + IPADDR1=10.0.0.10 +-LLADDR=fa:16:3e:ed:9a:59 + NETMASK=255.255.252.0 + NETMASK1=255.255.255.0 + STARTMODE=auto +@@ -727,25 +716,20 @@ dns = none + ), + ( + "etc/udev/rules.d/85-persistent-net-cloud-init.rules", +- "".join( +- [ +- 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', +- 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n', +- ] +- ), ++ # Since we do not set mac address, we are expecting the content to be nil ++ "", + ), + ], + "out_sysconfig_rhel": [ + ( +- "etc/sysconfig/network-scripts/ifcfg-eth0", ++ "etc/sysconfig/network-scripts/ifcfg-tap1a81968a-79", + """ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=none + DEFROUTE=yes +-DEVICE=eth0 ++DEVICE=tap1a81968a-79 + GATEWAY=172.19.3.254 +-HWADDR=fa:16:3e:ed:9a:59 + IPADDR=172.19.1.34 + IPADDR1=10.0.0.10 + NETMASK=255.255.252.0 +@@ -775,12 +759,8 @@ dns = none + ), + ( + "etc/udev/rules.d/70-persistent-net.rules", +- "".join( +- [ +- 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', +- 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n', +- ] +- ), ++ # Since we do not set mac address, we are expecting the content to be nil ++ "", + ), + ], + }, +@@ -852,7 +832,7 @@ dns = none + }, + "out_sysconfig_opensuse": [ + ( +- "etc/sysconfig/network/ifcfg-eth0", ++ "etc/sysconfig/network/ifcfg-tap1a81968a-79", + """ + # Created by cloud-init automatically, do not edit. + # +@@ -861,7 +841,6 @@ IPADDR=172.19.1.34 + IPADDR6=2001:DB8::10/64 + IPADDR6_1=2001:DB9::10/64 + IPADDR6_2=2001:DB10::10/64 +-LLADDR=fa:16:3e:ed:9a:59 + NETMASK=255.255.252.0 + STARTMODE=auto + """.lstrip(), +@@ -885,25 +864,20 @@ dns = none + ), + ( + "etc/udev/rules.d/85-persistent-net-cloud-init.rules", +- "".join( +- [ +- 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', +- 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n', +- ] +- ), ++ # Since we do not set mac address, we are expecting the content to be nil ++ "", + ), + ], + "out_sysconfig_rhel": [ + ( +- "etc/sysconfig/network-scripts/ifcfg-eth0", ++ "etc/sysconfig/network-scripts/ifcfg-tap1a81968a-79", + """ + # Created by cloud-init automatically, do not edit. + # + BOOTPROTO=none + DEFROUTE=yes +-DEVICE=eth0 ++DEVICE=tap1a81968a-79 + GATEWAY=172.19.3.254 +-HWADDR=fa:16:3e:ed:9a:59 + IPADDR=172.19.1.34 + IPV6ADDR=2001:DB8::10/64 + IPV6ADDR_SECONDARIES="2001:DB9::10/64 2001:DB10::10/64" +@@ -937,12 +911,8 @@ dns = none + ), + ( + "etc/udev/rules.d/70-persistent-net.rules", +- "".join( +- [ +- 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', +- 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n', +- ] +- ), ++ # Since we do not set mac address, we are expecting the content to be nil ++ "", + ), + ], + }, +-- +2.34.1 diff --git a/SPECS/cloud-init/cloud-init.spec b/SPECS/cloud-init/cloud-init.spec index ed7c6c45bcc..f06201068b4 100644 --- a/SPECS/cloud-init/cloud-init.spec +++ b/SPECS/cloud-init/cloud-init.spec @@ -1,7 +1,7 @@ Summary: Cloud instance init scripts Name: cloud-init Version: 23.4.3 -Release: 1%{?dist} +Release: 2%{?dist} License: GPLv3 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -9,7 +9,8 @@ Group: System Environment/Base URL: https://launchpad.net/cloud-init Source0: https://github.com/canonical/%{name}/archive/refs/tags/%{version}.tar.gz#/%{name}-%{version}.tar.gz Source1: 10-azure-kvp.cfg -Patch: 0001-Add-new-distro-azurelinux-for-Microsoft-Azure-Linux.patch +Patch0: 0001-Add-new-distro-azurelinux-for-Microsoft-Azure-Linux.patch +Patch1: Add-Network-Interface-Renaming-Support-for-CAPM3-Met.patch %define cl_services cloud-config.service cloud-config.target cloud-final.service cloud-init.service cloud-init.target cloud-init-local.service BuildRequires: automake BuildRequires: dbus @@ -143,6 +144,9 @@ make check %{?_smp_mflags} %config(noreplace) %{_sysconfdir}/cloud/cloud.cfg.d/10-azure-kvp.cfg %changelog +* Thu May 9 2024 Sharath Srikanth Chellappa - 23.4.3-2 +- Add patch to add network interface renaming support for CAPM3 Met. + * Mon Feb 26 2024 Dan Streetman - 23.4.3-1 - update to 23.4.3 - Use new 'azurelinux' cloud-init distro From 0aa73acf3eeca4bea9640660f78720d694ea307c Mon Sep 17 00:00:00 2001 From: Neha Agarwal <58672330+neha170@users.noreply.github.com> Date: Tue, 21 May 2024 11:38:48 -0700 Subject: [PATCH 38/51] postgresql: update to v16.3 to fix CVE-2024-4317 (#9156) --- SPECS/postgresql/postgresql.signatures.json | 2 +- SPECS/postgresql/postgresql.spec | 5 ++++- cgmanifest.json | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/SPECS/postgresql/postgresql.signatures.json b/SPECS/postgresql/postgresql.signatures.json index 1fe5ffede5b..ea00ed1d560 100644 --- a/SPECS/postgresql/postgresql.signatures.json +++ b/SPECS/postgresql/postgresql.signatures.json @@ -1,5 +1,5 @@ { "Signatures": { - "postgresql-16.1.tar.bz2": "ce3c4d85d19b0121fe0d3f8ef1fa601f71989e86f8a66f7dc3ad546dd5564fec" + "postgresql-16.3.tar.bz2": "331963d5d3dc4caf4216a049fa40b66d6bcb8c730615859411b9518764e60585" } } diff --git a/SPECS/postgresql/postgresql.spec b/SPECS/postgresql/postgresql.spec index 261d3a278e7..a52c7d0dbb0 100644 --- a/SPECS/postgresql/postgresql.spec +++ b/SPECS/postgresql/postgresql.spec @@ -1,6 +1,6 @@ Summary: PostgreSQL database engine Name: postgresql -Version: 16.1 +Version: 16.3 Release: 1%{?dist} License: PostgreSQL Vendor: Microsoft Corporation @@ -173,6 +173,9 @@ sudo -u nobody -s /bin/bash -c "PATH=$PATH make -k check" %{_libdir}/libpgtypes.a %changelog +* Mon May 20 2024 Neha Agarwal - 16.3-1 +- Upgrade to version 16.3 to fix CVE-2024-4317 + * Wed Dec 20 2023 Sharath Srikanth Chellappa - 16.1-1 - Upgrade to 16.1 - Removing postmaster since it is deprecated in v15 (https://www.postgresql.org/docs/15/app-postmaster.html) diff --git a/cgmanifest.json b/cgmanifest.json index e1f52594a60..bbdcfb784de 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -21123,8 +21123,8 @@ "type": "other", "other": { "name": "postgresql", - "version": "16.1", - "downloadUrl": "https://ftp.postgresql.org/pub/source/v16.1/postgresql-16.1.tar.bz2" + "version": "16.3", + "downloadUrl": "https://ftp.postgresql.org/pub/source/v16.3/postgresql-16.3.tar.bz2" } } }, From 741866791162586bbc33660d6af600e1ceacca32 Mon Sep 17 00:00:00 2001 From: Neha Agarwal <58672330+neha170@users.noreply.github.com> Date: Tue, 21 May 2024 14:08:55 -0700 Subject: [PATCH 39/51] libuv: update to v1.48.0 to fix CVE-2024-24806 (#9159) --- SPECS/libuv/libuv.signatures.json | 6 +++--- SPECS/libuv/libuv.spec | 13 ++++++++----- cgmanifest.json | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/SPECS/libuv/libuv.signatures.json b/SPECS/libuv/libuv.signatures.json index 034a1304e32..a0c0437c2db 100644 --- a/SPECS/libuv/libuv.signatures.json +++ b/SPECS/libuv/libuv.signatures.json @@ -1,5 +1,5 @@ { - "Signatures": { - "libuv-v1.46.0.tar.gz": "111f83958b9fdc65f1489195d25f342b9f7a3e683140c60e62c00fbaccddddce" - } + "Signatures": { + "libuv-v1.48.0.tar.gz": "7f1db8ac368d89d1baf163bac1ea5fe5120697a73910c8ae6b2fffb3551d59fb" + } } diff --git a/SPECS/libuv/libuv.spec b/SPECS/libuv/libuv.spec index d936dbb1e98..0293edd44b6 100644 --- a/SPECS/libuv/libuv.spec +++ b/SPECS/libuv/libuv.spec @@ -1,6 +1,6 @@ Summary: Cross-platform asynchronous I/O Name: libuv -Version: 1.46.0 +Version: 1.48.0 Release: 1%{?dist} License: MIT AND CC-BY Vendor: Microsoft Corporation @@ -75,6 +75,9 @@ sudo -u test make -k check %{_libdir}/%{name}.a %changelog +* Mon May 20 2024 Neha Agarwal - 1.48.0-1 +- Upgrade to version 1.48.0 to fix CVE-2024-24806 + * Fri Oct 27 2023 CBL-Mariner Servicing Account - 1.46.0-1 - Auto-upgrade to 1.46.0 - Azure Linux 3.0 - package upgrades @@ -82,8 +85,8 @@ sudo -u test make -k check - Upgrade to version 1.43.0 - License Verified -* Fri Dec 04 2020 Andrew Phelps - 1.38.0-2 -- Fix check tests. +* Fri Dec 04 2020 Andrew Phelps - 1.38.0-2 +- Fix check tests. -* Wed May 27 2020 Daniel McIlvaney - 1.38.0-1 -- Original version for CBL-Mariner +* Wed May 27 2020 Daniel McIlvaney - 1.38.0-1 +- Original version for CBL-Mariner diff --git a/cgmanifest.json b/cgmanifest.json index bbdcfb784de..4f6d7640add 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -11351,8 +11351,8 @@ "type": "other", "other": { "name": "libuv", - "version": "1.46.0", - "downloadUrl": "https://dist.libuv.org/dist/v1.46.0/libuv-v1.46.0.tar.gz" + "version": "1.48.0", + "downloadUrl": "https://dist.libuv.org/dist/v1.48.0/libuv-v1.48.0.tar.gz" } } }, From ba18a05bae20bd58848ce8a8168dc4a87a78e6dd Mon Sep 17 00:00:00 2001 From: Neha Agarwal <58672330+neha170@users.noreply.github.com> Date: Tue, 21 May 2024 14:09:07 -0700 Subject: [PATCH 40/51] less: patch CVE-2024-32487 (#9174) --- SPECS/less/CVE-2024-32487.patch | 67 +++++++++++++++++++++++++++++++++ SPECS/less/less.spec | 6 ++- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 SPECS/less/CVE-2024-32487.patch diff --git a/SPECS/less/CVE-2024-32487.patch b/SPECS/less/CVE-2024-32487.patch new file mode 100644 index 00000000000..f506b89e7c4 --- /dev/null +++ b/SPECS/less/CVE-2024-32487.patch @@ -0,0 +1,67 @@ +From 007521ac3c95bc76e3d59c6dbfe75d06c8075c33 Mon Sep 17 00:00:00 2001 +From: Mark Nudelman +Date: Thu, 11 Apr 2024 17:49:48 -0700 +Subject: [PATCH] Fix bug when viewing a file whose name contains a newline. + +--- + filename.c | 31 +++++++++++++++++++++++++------ + 1 file changed, 25 insertions(+), 6 deletions(-) + +diff --git a/filename.c b/filename.c +index f90e0e82..a52c6354 100644 +--- a/filename.c ++++ b/filename.c +@@ -133,6 +133,15 @@ static constant char * metachars(void) + return (strchr(metachars(), c) != NULL); + } + ++/* ++ * Must use quotes rather than escape char for this metachar? ++ */ ++static int must_quote(char c) ++{ ++ /* {{ Maybe the set of must_quote chars should be configurable? }} */ ++ return (c == '\n'); ++} ++ + /* + * Insert a backslash before each metacharacter in a string. + */ +@@ -164,6 +173,9 @@ public char * shell_quoten(constant char *s, size_t slen) + * doesn't support escape chars. Use quotes. + */ + use_quotes = 1; ++ } else if (must_quote(*p)) ++ { ++ len += 3; /* open quote + char + close quote */ + } else + { + /* +@@ -193,15 +205,22 @@ public char * shell_quoten(constant char *s, size_t slen) + { + while (*s != '\0') + { +- if (metachar(*s)) ++ if (!metachar(*s)) + { +- /* +- * Add the escape char. +- */ ++ *p++ = *s++; ++ } else if (must_quote(*s)) ++ { ++ /* Surround the char with quotes. */ ++ *p++ = openquote; ++ *p++ = *s++; ++ *p++ = closequote; ++ } else ++ { ++ /* Insert an escape char before the char. */ + strcpy(p, esc); + p += esclen; ++ *p++ = *s++; + } +- *p++ = *s++; + } + *p = '\0'; + } diff --git a/SPECS/less/less.spec b/SPECS/less/less.spec index e8c5350656b..f19b9d71b61 100644 --- a/SPECS/less/less.spec +++ b/SPECS/less/less.spec @@ -1,7 +1,7 @@ Summary: Text file viewer Name: less Version: 643 -Release: 1%{?dist} +Release: 2%{?dist} License: GPLv3+ OR BSD Vendor: Microsoft Corporation Distribution: Azure Linux @@ -10,6 +10,7 @@ URL: https://www.greenwoodsoftware.com/less Source0: https://www.greenwoodsoftware.com/less/%{name}-%{version}.tar.gz BuildRequires: ncurses-devel Requires: ncurses +Patch0: CVE-2024-32487.patch %description The Less package contains a text file viewer @@ -31,6 +32,9 @@ The Less package contains a text file viewer %{_mandir}/*/* %changelog +* Tue May 21 2024 Neha Agarwal - 643-2 +- Patch CVE-2024-32487 + * Fri Oct 27 2023 CBL-Mariner Servicing Account - 643-1 - Auto-upgrade to 643 - Azure Linux 3.0 - package upgrades From 77258ff44b34017c5d566e4b57d881c4c508cefc Mon Sep 17 00:00:00 2001 From: Daniel McIlvaney Date: Tue, 21 May 2024 17:17:28 -0700 Subject: [PATCH 41/51] Better document and prompt user with --no-cleanup (#9153) Co-authored-by: Pawel Winogrodzki --- toolkit/docs/building/building.md | 2 +- toolkit/tools/internal/logger/log.go | 50 ++++++++ toolkit/tools/internal/logger/log_test.go | 133 ++++++++++++++++++++ toolkit/tools/internal/pkggraph/pkggraph.go | 11 +- toolkit/tools/scheduler/scheduler.go | 15 +++ 5 files changed, 205 insertions(+), 6 deletions(-) create mode 100644 toolkit/tools/internal/logger/log_test.go diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index 35788c25946..db508731313 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -822,7 +822,7 @@ To reproduce an ISO build, run the same make invocation as before, but set: | EXTRA_BUILD_LAYERS | 0 | How many additional layers of the build graph to build beyond the requested packages (useful for testing changes in dependent packages) | IMAGE_TAG | (empty) | Text appended to a resulting image name - empty by default. Does not apply to the initrd. The text will be prepended with a hyphen. | CONCURRENT_PACKAGE_BUILDS | 0 | The maximum number of concurrent package builds that are allowed at once. If set to 0 this defaults to the number of logical CPUs. -| CLEANUP_PACKAGE_BUILDS | y | Cleanup a package build's working directory when it finishes. Note that `build` directory will still be removed on a successful package build even when this is turned off. +| CLEANUP_PACKAGE_BUILDS | y | Cleanup a package build's working directory (`./build/worker/chroot//*`) when it finishes. Note that `rpmbuild`'s `BUILD` directory will still be removed on a successful package build even when this is turned off. Consider `make containerized-rpmbuild SRPM_PACKLIST=` for debugging build issues instead. The user must call `sudo make clean-build-packages-workers` to tidy any uncleaned build environments after a build. | USE_PACKAGE_BUILD_CACHE | y | Skip building a package if it and its dependencies are already built. | NUM_OF_ANALYTICS_RESULTS | 10 | The number of entries to print when using the `graphanalytics` tool. If set to 0 this will print all available results. | TARGET_ARCH | | The architecture of the machine that will run the package binaries. diff --git a/toolkit/tools/internal/logger/log.go b/toolkit/tools/internal/logger/log.go index 4dcd535629e..f7e82508a1f 100644 --- a/toolkit/tools/internal/logger/log.go +++ b/toolkit/tools/internal/logger/log.go @@ -6,12 +6,14 @@ package logger import ( + "fmt" "io" "log" "os" "path/filepath" "runtime" "strings" + "unicode/utf8" "github.com/sirupsen/logrus" ) @@ -213,3 +215,51 @@ func setHookLogLevel(hook *writerHook, level string) (err error) { hook.SetLevel(logLevel) return } + +// PrintMessageBox prints a message box to the log with the specified log level. +func PrintMessageBox(level logrus.Level, message []string) { + for _, line := range FormatMessageBox(message) { + Log.Log(level, line) + } +} + +// FormatMessageBox formats a message into a box with a border. The box is automatically sized to fit the longest line. +// Each line will be centered in the box. +func FormatMessageBox(message []string) []string { + maxLineLength := 0 + for _, line := range message { + len := utf8.RuneCountInString(line) + if len > maxLineLength { + maxLineLength = len + } + } + lines := []string{messageBoxTopString(maxLineLength)} + for _, line := range message { + lines = append(lines, messageBoxMiddleString(line, maxLineLength)) + } + lines = append(lines, messageBoxBottomString(maxLineLength)) + return lines +} + +func messageBoxTopString(width int) string { + return fmt.Sprintf("╔═%s═╗", strings.Repeat("═", width)) +} + +func messageBoxMiddleString(s string, width int) string { + return fmt.Sprintf("║ %s ║", messageBoxPadString(s, width)) +} + +func messageBoxBottomString(width int) string { + return fmt.Sprintf("╚═%s═╝", strings.Repeat("═", width)) +} + +func messageBoxPadString(s string, width int) string { + lineLen := utf8.RuneCountInString(s) + if lineLen >= width { + return s + } + padding := width - lineLen + paddingL := padding / 2 + paddingR := padding - paddingL + return fmt.Sprintf("%s%s%s", strings.Repeat(" ", paddingL), s, strings.Repeat(" ", paddingR)) +} diff --git a/toolkit/tools/internal/logger/log_test.go b/toolkit/tools/internal/logger/log_test.go new file mode 100644 index 00000000000..5af7a7bdaaa --- /dev/null +++ b/toolkit/tools/internal/logger/log_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Shared logger + +package logger + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPadString(t *testing.T) { + tests := []struct { + name string + s string + width int + want string + }{ + { + name: "empty string", + s: "", + width: 10, + want: " ", + }, + { + name: "short string", + s: "hello", + width: 10, + want: " hello ", + }, + { + name: "equal string", + s: "hello", + width: 5, + want: "hello", + }, + { + name: "long string", + s: "this is a long string", + width: 10, + want: "this is a long string", + }, + { + name: "utf8 multirune", + s: "世", + width: 10, + want: " 世 ", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := messageBoxPadString(tt.s, tt.width) + assert.Equal(t, tt.want, got) + }) + } + +} + +func TestFormatWarningBox(t *testing.T) { + tests := []struct { + name string + message []string + want []string + }{ + { + name: "empty message", + message: []string{}, + want: []string{ + "╔══╗", + "╚══╝", + }, + }, + { + name: "single line message", + message: []string{"hello"}, + want: []string{ + "╔═══════╗", + "║ hello ║", + "╚═══════╝", + }, + }, + { + name: "multi line message", + message: []string{"hello", "world"}, + want: []string{ + "╔═══════╗", + "║ hello ║", + "║ world ║", + "╚═══════╝", + }, + }, + { + name: "Different lenghts", + message: []string{ + "this is a long message", + "odd", + "even", + }, + want: []string{ + "╔════════════════════════╗", + "║ this is a long message ║", + "║ odd ║", + "║ even ║", + "╚════════════════════════╝", + }, + }, + { + name: "utf8 multirune", + message: []string{ + "Hello", + "世", // ~"world" + "世界", // "world" + }, + // unicode characters can take up more than one character space, this is too complex to handle + // so we just accept the inconsistency + want: []string{ + "╔═══════╗", + "║ Hello ║", + "║ 世 ║", + "║ 世界 ║", + "╚═══════╝", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + box := FormatMessageBox(tt.message) + assert.Equal(t, tt.want, box) + }) + } +} diff --git a/toolkit/tools/internal/pkggraph/pkggraph.go b/toolkit/tools/internal/pkggraph/pkggraph.go index 0c0d4bcc97d..816aabe6ebc 100644 --- a/toolkit/tools/internal/pkggraph/pkggraph.go +++ b/toolkit/tools/internal/pkggraph/pkggraph.go @@ -1747,11 +1747,12 @@ func formatCycleErrorMessage(cycle []*PkgNode, err error) error { // Hydrating the toolchain RPMs was required to resolve the cycles at one point. This is no longer the case, but // we should leave a message here to avoid confusion. - logger.Log.Warn("╔════════════════════════════════════════════════════════════════════════════════════════════════╗") - logger.Log.Warn("║ 'copy-toolchain-rpms' should no longer be required to resolve cycles even when using online ║") - logger.Log.Warn("║ toolchain rpms. If you see this message, there is likely a legitimate cycle in the dependency ║") - logger.Log.Warn("║ graph. ║") - logger.Log.Warn("╚════════════════════════════════════════════════════════════════════════════════════════════════╝") + message := []string{ + "'copy-toolchain-rpms' should no longer be required to resolve cycles even when using online", + "toolchain rpms. If you see this message, there is likely a legitimate cycle in the dependency", + "graph.", + } + logger.PrintMessageBox(logrus.WarnLevel, message) return fmt.Errorf("unfixable circular dependency in dependency graph (%s):\n%w", cycleStringBuilder.String(), err) } diff --git a/toolkit/tools/scheduler/scheduler.go b/toolkit/tools/scheduler/scheduler.go index 8a7d1a89845..f9e7c3a0035 100644 --- a/toolkit/tools/scheduler/scheduler.go +++ b/toolkit/tools/scheduler/scheduler.go @@ -21,6 +21,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/pkg/profile" "github.com/microsoft/azurelinux/toolkit/tools/scheduler/buildagents" "github.com/microsoft/azurelinux/toolkit/tools/scheduler/schedulerutils" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" "gopkg.in/alecthomas/kingpin.v2" @@ -210,6 +211,20 @@ func main() { logger.Log.Warnf("Failed to initialize the ccache manager:\n%v", err) } } + + if *noCleanup { + message := []string{ + "ATTENTION!", + "", + "'--no-cleanup' requested. Build agent directories will not be removed automatically.", + "(" + *workDir + "/*)", + "Manual cleanup is required.", + "Use 'make clean-build-packages-workers' to remove build agent directories.", + "", + "Also consider using 'make containerized-rpmbuild' to debug package build issues", + } + logger.PrintMessageBox(logrus.InfoLevel, message) + } } // cancelOutstandingBuilds stops any builds that are currently running. From bed860730be5da664df4d32043fc9b689f704e1f Mon Sep 17 00:00:00 2001 From: Daniel McIlvaney Date: Tue, 21 May 2024 17:33:45 -0700 Subject: [PATCH 42/51] Sanitize licenses in core image packages (#9147) Co-authored-by: Pawel Winogrodzki --- SPECS/cmake/cmake.spec | 15 +++++++++++++-- .../cyrus-sasl-bootstrap.spec | 7 ++++--- SPECS/cyrus-sasl/cyrus-sasl.spec | 7 ++++--- SPECS/ethtool/ethtool.spec | 9 ++++++--- SPECS/libdb/libdb.spec | 10 +++++----- SPECS/librdkafka/librdkafka.spec | 12 +++++++++--- SPECS/lmdb/lmdb.spec | 12 ++++++------ SPECS/qemu/qemu.spec | 10 +++++++--- SPECS/userspace-rcu/userspace-rcu.spec | 9 +++++++-- .../manifests/package/toolchain_aarch64.txt | 4 ++-- .../manifests/package/toolchain_x86_64.txt | 4 ++-- 11 files changed, 65 insertions(+), 34 deletions(-) diff --git a/SPECS/cmake/cmake.spec b/SPECS/cmake/cmake.spec index 36e90172708..155bcf86981 100644 --- a/SPECS/cmake/cmake.spec +++ b/SPECS/cmake/cmake.spec @@ -2,7 +2,7 @@ Summary: Cmake Name: cmake Version: 3.28.2 -Release: 2%{?dist} +Release: 3%{?dist} License: BSD AND LGPLv2+ Vendor: Microsoft Corporation Distribution: Azure Linux @@ -60,6 +60,15 @@ find %{buildroot} -type f -name "*.la" -delete -print install -Dpm0644 %{SOURCE1} %{buildroot}%{_libdir}/rpm/macros.d/macros.cmake sed -i -e "s|@@CMAKE_VERSION@@|%{version}|" -e "s|@@CMAKE_MAJOR_VERSION@@|%{major_version}|" %{buildroot}%{_libdir}/rpm/macros.d/macros.cmake +# Collect all license files into one spot +for f in Copyright.txt cmcppdap/NOTICE cmcurl/COPYING cmlibrhash/COPYING cmlibuv/LICENSE cmnghttp2/COPYING cmsys/Copyright.txt; do + filename_part=$(basename $f) + dir_part=$(dirname $f) + mkdir -p ./Licenses/$dir_part + mv %{buildroot}%{_prefix}/doc/%{name}-*/$f ./Licenses/$dir_part/$filename_part +done +find "%{buildroot}%{_prefix}/doc" -type d -empty -delete + %check # Removing static libraries to fix issues with the "ParseImplicitLinkInfo" test runs for the "craype-C-Cray-8.7.input" and "craype-CXX-Cray-8.7.input" inputs. # Should be removed once the issue is fixed upstream and we apply the fix: https://gitlab.kitware.com/cmake/cmake/-/issues/22470. @@ -78,9 +87,11 @@ bin/ctest --force-new-ctest-process --rerun-failed --output-on-failure %{_datadir}/emacs/site-lisp/cmake-mode.el %{_datadir}/vim/vimfiles/* %{_libdir}/rpm/macros.d/macros.cmake -%{_prefix}/doc/%{name}-*/* %changelog +* Thu May 16 2024 Daniel McIlvaney - 3.28.2-3 +- Sanitize license files + * Fri Mar 29 2024 Andrew Phelps - 3.28.2-2 - Fix JDK test issue diff --git a/SPECS/cyrus-sasl-bootstrap/cyrus-sasl-bootstrap.spec b/SPECS/cyrus-sasl-bootstrap/cyrus-sasl-bootstrap.spec index 20f301245af..50bb3790204 100644 --- a/SPECS/cyrus-sasl-bootstrap/cyrus-sasl-bootstrap.spec +++ b/SPECS/cyrus-sasl-bootstrap/cyrus-sasl-bootstrap.spec @@ -5,7 +5,7 @@ Summary: Cyrus Simple Authentication Service Layer (SASL) library Name: %{_base_name}-bootstrap Version: 2.1.28 -Release: 6%{?dist} +Release: 7%{?dist} License: BSD with advertising Vendor: Microsoft Corporation Distribution: Azure Linux @@ -89,7 +89,6 @@ make [ %{buildroot} != "/"] && rm -rf %{buildroot}/* make DESTDIR=%{buildroot} install find %{buildroot} -type f -name "*.la" -delete -print -install -D -m644 COPYING %{buildroot}%{_datadir}/licenses/%{name}/LICENSE %{_fixperms} %{buildroot}/* mkdir -p %{buildroot}%{_sysconfdir}/sysconfig @@ -147,7 +146,6 @@ make %{?_smp_mflags} check /lib/systemd/system/saslauthd.service %{_libdir}/systemd/system-preset/50-saslauthd.preset %{_sbindir}/* -%{_datadir}/licenses/%{name}/LICENSE %{_mandir}/man8/* %files devel @@ -194,6 +192,9 @@ make %{?_smp_mflags} check %exclude %{_plugindir2}/libsql.so.%{_soversion}* %changelog +* Thu May 16 2024 Daniel McIlvaney - 2.1.28-7 +- Sanitize license files + * Mon Feb 05 2024 Dan Streetman - 2.1.28-6 - workaround "circular dependencies" from build tooling diff --git a/SPECS/cyrus-sasl/cyrus-sasl.spec b/SPECS/cyrus-sasl/cyrus-sasl.spec index 90b450e4dbc..a3db4bafa46 100644 --- a/SPECS/cyrus-sasl/cyrus-sasl.spec +++ b/SPECS/cyrus-sasl/cyrus-sasl.spec @@ -4,7 +4,7 @@ Summary: Cyrus Simple Authentication Service Layer (SASL) library Name: cyrus-sasl Version: 2.1.28 -Release: 6%{?dist} +Release: 7%{?dist} License: BSD with advertising Vendor: Microsoft Corporation Distribution: Azure Linux @@ -195,7 +195,6 @@ make [ %{buildroot} != "/"] && rm -rf %{buildroot}/* make DESTDIR=%{buildroot} install find %{buildroot} -type f -name "*.la" -delete -print -install -D -m644 COPYING %{buildroot}%{_datadir}/licenses/%{name}/LICENSE %{_fixperms} %{buildroot}/* mkdir -p %{buildroot}%{_sysconfdir}/sysconfig @@ -253,7 +252,6 @@ make %{?_smp_mflags} check /lib/systemd/system/saslauthd.service %{_libdir}/systemd/system-preset/50-saslauthd.preset %{_sbindir}/* -%{_datadir}/licenses/%{name}/LICENSE %{_mandir}/man8/* %files devel @@ -311,6 +309,9 @@ make %{?_smp_mflags} check %{_plugindir2}/libsql.so.%{_soversion}* %changelog +* Thu May 16 2024 Daniel McIlvaney - 2.1.28-7 +- Sanitize license files + * Mon Feb 05 2024 Dan Streetman - 2.1.28-6 - match bootstrap version diff --git a/SPECS/ethtool/ethtool.spec b/SPECS/ethtool/ethtool.spec index 51ecf6da264..53bff41002f 100644 --- a/SPECS/ethtool/ethtool.spec +++ b/SPECS/ethtool/ethtool.spec @@ -1,7 +1,7 @@ Summary: Standard Linux utility for controlling network drivers and hardware Name: ethtool Version: 6.4 -Release: 1%{?dist} +Release: 2%{?dist} License: GPLv2 URL: https://www.kernel.org/pub/software/network/ethtool/ Group: Productivity/Networking/Diagnostic @@ -30,14 +30,17 @@ make install DESTDIR=%{buildroot} make %{?_smp_mflags} check %files -%doc AUTHORS COPYING NEWS README ChangeLog +%license COPYING LICENSE +%doc AUTHORS NEWS README ChangeLog %defattr(-,root,root) -%license LICENSE /sbin/* %{_mandir} %{_datadir}/bash-completion/completions/ethtool %changelog +* Thu May 16 2024 Daniel McIlvaney - 6.4-2 +- Sanitize license files + * Fri Oct 27 2023 CBL-Mariner Servicing Account - 6.4-1 - Auto-upgrade to 6.4 - Azure Linux 3.0 - package upgrades diff --git a/SPECS/libdb/libdb.spec b/SPECS/libdb/libdb.spec index a954c935fb8..df7fd0d8af6 100644 --- a/SPECS/libdb/libdb.spec +++ b/SPECS/libdb/libdb.spec @@ -1,7 +1,7 @@ Summary: The Berkley DB database library for C Name: libdb Version: 5.3.28 -Release: 7%{?dist} +Release: 8%{?dist} License: BSD Vendor: Microsoft Corporation Distribution: Azure Linux @@ -60,15 +60,12 @@ pushd build_unix make DESTDIR=%{buildroot} docdir=%{_docdir}/%{name}-%{version} install popd find %{buildroot} -type f -name "*.la" -delete -print -install -v -d -m755 %{buildroot}/%{_datadir}/licenses/ -install -D -m755 LICENSE %{buildroot}/%{_datadir}/licenses/LICENSE -install -D -m755 README %{buildroot}/%{_datadir}/licenses/README %files %defattr(-,root,root) %license LICENSE +%doc README %{_libdir}/*.so -%{_datadir}/licenses/* %files docs %defattr(-,root,root) @@ -95,6 +92,9 @@ install -D -m755 README %{buildroot}/%{_datadir}/licenses/README %{_bindir}/db*_tuner %changelog +* Thu May 16 2024 Daniel McIlvaney - 5.3.28-8 +- Sanitize license files + * Tue Apr 12 2022 Nicolas Guibourge - 5.3.28-7 - Align with CBL-Mariner 1.0 to address following CVEs: - Patch CVE-2019-2708 diff --git a/SPECS/librdkafka/librdkafka.spec b/SPECS/librdkafka/librdkafka.spec index b257e1f338c..41d2dca762b 100644 --- a/SPECS/librdkafka/librdkafka.spec +++ b/SPECS/librdkafka/librdkafka.spec @@ -5,7 +5,7 @@ Summary: The Apache Kafka C library Name: librdkafka Version: 2.3.0 -Release: 1%{?dist} +Release: 2%{?dist} # files like src/crc32c.c are under zlib license # files like win32/wingetopt.c are under ISC # files like src/rdfnv1a.c are under Public Domain @@ -66,6 +66,9 @@ make check %install DESTDIR=%{buildroot} make install +# Remove extraneous license files from docs +rm %{buildroot}%{_docdir}/librdkafka/LICENSE %{buildroot}%{_docdir}/librdkafka/LICENSES.txt + %post -n %{name}%{soname} -p /sbin/ldconfig %postun -n %{name}%{soname} -p /sbin/ldconfig @@ -79,8 +82,8 @@ DESTDIR=%{buildroot} make install %doc %{_docdir}/librdkafka/INTRODUCTION.md %doc %{_docdir}/librdkafka/STATISTICS.md %doc %{_docdir}/librdkafka/CHANGELOG.md -%license %{_docdir}/librdkafka/LICENSE -%doc %{_docdir}/librdkafka/LICENSES.txt +%license LICENSE +%license LICENSES.txt %files -n %{name}-devel %defattr(-,root,root) @@ -97,6 +100,9 @@ DESTDIR=%{buildroot} make install %{_libdir}/pkgconfig/rdkafka++-static.pc %changelog +* Thu May 16 2024 Daniel McIlvaney - 2.3.0-2 +- Sanitize license files + * Thu Dec 21 2023 Neha Agarwal - 2.3.0-1 - Update to v2.3.0 diff --git a/SPECS/lmdb/lmdb.spec b/SPECS/lmdb/lmdb.spec index b8060d23f3e..0d5dd8c1bd6 100644 --- a/SPECS/lmdb/lmdb.spec +++ b/SPECS/lmdb/lmdb.spec @@ -1,7 +1,7 @@ Summary: Lightning memory-mapped database Name: lmdb Version: 0.9.31 -Release: 1%{?dist} +Release: 2%{?dist} License: OpenLDAP Vendor: Microsoft Corporation Distribution: Azure Linux @@ -43,8 +43,6 @@ make prefix=%{_prefix} DESTDIR=%{buildroot} install mkdir -p %{buildroot}%{_docdir}/%{name} mkdir -p %{buildroot}%{_defaultlicensedir}/%{name} mkdir -p %{buildroot}%{_libdir}/pkgconfig -install -m0644 COPYRIGHT %{buildroot}%{_docdir}/%{name} -install -m0644 LICENSE %{buildroot}%{_defaultlicensedir}/%{name} install -m0755 %{SOURCE1} %{buildroot}%{_libdir}/pkgconfig %post @@ -55,7 +53,6 @@ install -m0755 %{SOURCE1} %{buildroot}%{_libdir}/pkgconfig # First argument is 2 => Upgrade %files -%license libraries/liblmdb/LICENSE %{_mandir}/* %{_bindir}/* @@ -65,11 +62,14 @@ install -m0755 %{SOURCE1} %{buildroot}%{_libdir}/pkgconfig %{_libdir}/pkgconfig/%{name}.pc %files libs -%{_docdir}/%{name}/COPYRIGHT -%{_defaultlicensedir}/%{name}/LICENSE +%license libraries/liblmdb/COPYRIGHT +%license libraries/liblmdb/LICENSE %{_libdir}/*.so %changelog +* Thu May 16 2024 Daniel McIlvaney - 0.9.31-2 +- Sanitize license files + * Fri Oct 27 2023 CBL-Mariner Servicing Account - 0.9.31-1 - Auto-upgrade to 0.9.31 - Azure Linux 3.0 - package upgrades diff --git a/SPECS/qemu/qemu.spec b/SPECS/qemu/qemu.spec index e8fd06e5b8e..78743efa8e8 100644 --- a/SPECS/qemu/qemu.spec +++ b/SPECS/qemu/qemu.spec @@ -438,7 +438,7 @@ Obsoletes: sgabios-bin <= 1:0.20180715git-10.fc38 Summary: QEMU is a FAST! processor emulator Name: qemu Version: 8.2.0 -Release: 6%{?dist} +Release: 7%{?dist} License: Apache-2.0 AND BSD-2-Clause AND BSD-3-Clause AND FSFAP AND GPL-1.0-or-later AND GPL-2.0-only AND GPL-2.0-or-later AND GPL-2.0-or-later WITH GCC-exception-2.0 AND LGPL-2.0-only AND LGPL-2.0-or-later AND LGPL-2.1-only AND LGPL-2.1-or-later AND MIT AND LicenseRef-Fedora-Public-Domain AND CC-BY-3.0 URL: http://www.qemu.org/ @@ -2127,7 +2127,7 @@ install -D -p -m 0644 %{modprobe_kvm_conf} %{buildroot}%{_sysconfdir}/modprobe.d %endif # Copy some static data into place -install -D -p -m 0644 -t %{buildroot}%{qemudocdir} README.rst COPYING COPYING.LIB LICENSE docs/interop/qmp-spec.rst +install -D -p -m 0644 -t %{buildroot}%{qemudocdir} README.rst docs/interop/qmp-spec.rst install -D -p -m 0644 qemu.sasl %{buildroot}%{_sysconfdir}/sasl2/%{name}.conf install -m 0644 scripts/dump-guest-memory.py %{buildroot}%{_datadir}/%{name} @@ -2523,7 +2523,8 @@ useradd -r -u 107 -g qemu -G kvm -d / -s /sbin/nologin \ %endif %files -n qemu-guest-agent -%doc COPYING README.rst +%license COPYING COPYING.LIB LICENSE +%doc README.rst %{_bindir}/qemu-ga %if ! %{azl} %{_mandir}/man8/qemu-ga.8* @@ -3467,6 +3468,9 @@ useradd -r -u 107 -g qemu -G kvm -d / -s /sbin/nologin \ %changelog +* Thu May 16 2024 Daniel McIlvaney - 8.2.0-7 +- Sanitize license files + * Mon May 13 2024 Chris Co - 8.2.0-6 - Update to build dep latest glibc-static version diff --git a/SPECS/userspace-rcu/userspace-rcu.spec b/SPECS/userspace-rcu/userspace-rcu.spec index 5a1f8f2910b..612b3d47530 100644 --- a/SPECS/userspace-rcu/userspace-rcu.spec +++ b/SPECS/userspace-rcu/userspace-rcu.spec @@ -1,7 +1,7 @@ Summary: user space RCU (read-copy-update) Name: userspace-rcu Version: 0.14.0 -Release: 1%{?dist} +Release: 2%{?dist} License: LGPLv2+ Vendor: Microsoft Corporation Distribution: Azure Linux @@ -39,23 +39,28 @@ make %{?_smp_mflags} %install make DESTDIR=%{buildroot} install find %{buildroot} -type f -name "*.la" -delete -print +# Don't duplicate LICENSE file in the doc directory +rm %{buildroot}%{_datadir}/doc/userspace-rcu/LICENSE %check make %{?_smp_mflags} check %files +%license LICENSE %{_libdir}/*.so.* %{_includedir}/* %{_datadir}/* %files devel %defattr(-,root,root) -%license LICENSE %{_libdir}/pkgconfig/* %{_libdir}/*.so %{_includedir}/* %changelog +* Thu May 16 2024 Daniel McIlvaney - 0.14.0-2 +- Sanitize license files + * Fri Oct 27 2023 CBL-Mariner Servicing Account - 0.14.0-1 - Auto-upgrade to 0.14.0 - Azure Linux 3.0 - package upgrades diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index ec8f354b3c8..d376be57f29 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -45,8 +45,8 @@ check-debuginfo-0.15.2-1.azl3.aarch64.rpm chkconfig-1.25-1.azl3.aarch64.rpm chkconfig-debuginfo-1.25-1.azl3.aarch64.rpm chkconfig-lang-1.25-1.azl3.aarch64.rpm -cmake-3.28.2-2.azl3.aarch64.rpm -cmake-debuginfo-3.28.2-2.azl3.aarch64.rpm +cmake-3.28.2-3.azl3.aarch64.rpm +cmake-debuginfo-3.28.2-3.azl3.aarch64.rpm coreutils-9.4-2.azl3.aarch64.rpm coreutils-debuginfo-9.4-2.azl3.aarch64.rpm coreutils-lang-9.4-2.azl3.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index 814b601579d..f60a15d06f3 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -46,8 +46,8 @@ check-debuginfo-0.15.2-1.azl3.x86_64.rpm chkconfig-1.25-1.azl3.x86_64.rpm chkconfig-debuginfo-1.25-1.azl3.x86_64.rpm chkconfig-lang-1.25-1.azl3.x86_64.rpm -cmake-3.28.2-2.azl3.x86_64.rpm -cmake-debuginfo-3.28.2-2.azl3.x86_64.rpm +cmake-3.28.2-3.azl3.x86_64.rpm +cmake-debuginfo-3.28.2-3.azl3.x86_64.rpm coreutils-9.4-2.azl3.x86_64.rpm coreutils-debuginfo-9.4-2.azl3.x86_64.rpm coreutils-lang-9.4-2.azl3.x86_64.rpm From c7f96bb8aa5aea18c43398cd360cb5e2d66a93de Mon Sep 17 00:00:00 2001 From: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> Date: Tue, 21 May 2024 22:07:35 -0700 Subject: [PATCH 43/51] [AUTOPATCHER-CORE] Upgrade fluent-bit to 3.0.3 (#9122) --- SPECS/fluent-bit/fluent-bit.signatures.json | 6 +++--- SPECS/fluent-bit/fluent-bit.spec | 5 ++++- cgmanifest.json | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/SPECS/fluent-bit/fluent-bit.signatures.json b/SPECS/fluent-bit/fluent-bit.signatures.json index ac0e0f117de..21993c7fe6f 100644 --- a/SPECS/fluent-bit/fluent-bit.signatures.json +++ b/SPECS/fluent-bit/fluent-bit.signatures.json @@ -1,5 +1,5 @@ { - "Signatures": { - "fluent-bit-2.2.2.tar.gz": "8e7e951b2907e9d29508699c71c8949a4a22d750d54ffa5ee5b96537e59371dd" - } + "Signatures": { + "fluent-bit-3.0.3.tar.gz": "2f3b82e8b7802fe92625009531592f12d500ff61a02ecf4cd27e0f138a6dc566" + } } diff --git a/SPECS/fluent-bit/fluent-bit.spec b/SPECS/fluent-bit/fluent-bit.spec index 5d8293ef05c..a61c3e69e10 100644 --- a/SPECS/fluent-bit/fluent-bit.spec +++ b/SPECS/fluent-bit/fluent-bit.spec @@ -1,6 +1,6 @@ Summary: Fast and Lightweight Log processor and forwarder for Linux, BSD and OSX Name: fluent-bit -Version: 2.2.2 +Version: 3.0.3 Release: 1%{?dist} License: Apache-2.0 Vendor: Microsoft Corporation @@ -75,6 +75,9 @@ Development files for %{name} %{_libdir}/fluent-bit/*.so %changelog +* Thu May 16 2024 CBL-Mariner Servicing Account - 3.0.3-1 +- Auto-upgrade to 3.0.3 - https://microsoft.visualstudio.com/OS/_workitems/edit/50531424 + * Tue Feb 20 2024 Sumedh Sharma - 2.2.2-1 - Upgrade to version 2.2.2 diff --git a/cgmanifest.json b/cgmanifest.json index 4f6d7640add..2e51df0abeb 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -3678,8 +3678,8 @@ "type": "other", "other": { "name": "fluent-bit", - "version": "2.2.2", - "downloadUrl": "https://github.com/fluent/fluent-bit/archive/refs/tags/v2.2.2.tar.gz" + "version": "3.0.3", + "downloadUrl": "https://github.com/fluent/fluent-bit/archive/refs/tags/v3.0.3.tar.gz" } } }, From c897f7c47c0804bd9a8daae0ae412d006d426195 Mon Sep 17 00:00:00 2001 From: Neha Agarwal <58672330+neha170@users.noreply.github.com> Date: Wed, 22 May 2024 10:00:21 -0700 Subject: [PATCH 44/51] jasper: patch CVE-2024-31744 (#9176) --- SPECS/jasper/CVE-2024-31744.patch | 28 ++++++++++++++++++++++++++++ SPECS/jasper/jasper.spec | 17 +++++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 SPECS/jasper/CVE-2024-31744.patch diff --git a/SPECS/jasper/CVE-2024-31744.patch b/SPECS/jasper/CVE-2024-31744.patch new file mode 100644 index 00000000000..47256a78257 --- /dev/null +++ b/SPECS/jasper/CVE-2024-31744.patch @@ -0,0 +1,28 @@ +From 6d084c53a77762f41bb5310713a5f1872fef55f5 Mon Sep 17 00:00:00 2001 +From: Michael Adams +Date: Fri, 29 Mar 2024 07:57:29 -0700 +Subject: [PATCH] Fixes #381. + +Added a missing check to the jpc_dec_process_sod function of the JPC codec. +Added another image to the test set. +--- + data/test/bad/318.jpc | Bin 0 -> 320 bytes + src/libjasper/jpc/jpc_dec.c | 4 +++- + 2 files changed, 3 insertions(+), 1 deletion(-) + create mode 100644 data/test/bad/318.jpc + +diff --git a/src/libjasper/jpc/jpc_dec.c b/src/libjasper/jpc/jpc_dec.c +index e76aa400..125a29ba 100644 +--- a/src/libjasper/jpc/jpc_dec.c ++++ b/src/libjasper/jpc/jpc_dec.c +@@ -611,7 +611,9 @@ static int jpc_dec_process_sod(jpc_dec_t *dec, jpc_ms_t *ms) + if (dec->pkthdrstreams) { + /* Get the stream containing the packet header data for this + tile-part. */ +- if (!(tile->pkthdrstream = jpc_streamlist_remove(dec->pkthdrstreams, 0))) { ++ if (jpc_streamlist_numstreams(dec->pkthdrstreams) != 0 && ++ !(tile->pkthdrstream = jpc_streamlist_remove(dec->pkthdrstreams, ++ 0))) { + return -1; + } + } diff --git a/SPECS/jasper/jasper.spec b/SPECS/jasper/jasper.spec index 6ce879ecf1a..c5201ceb231 100644 --- a/SPECS/jasper/jasper.spec +++ b/SPECS/jasper/jasper.spec @@ -6,7 +6,7 @@ Summary: Implementation of the JPEG-2000 standard, Part 1 Name: jasper Version: 4.2.1 -Release: 1%{?dist} +Release: 2%{?dist} License: JasPer-2.0 Vendor: Microsoft Corporation @@ -15,13 +15,14 @@ URL: http://www.ece.uvic.ca/~frodo/jasper/ Source0: https://github.com/jasper-software/%{name}/archive/refs/tags/version-%{version}.tar.gz#/%{name}-%{version}.tar.gz # architecture related patches +Patch0: CVE-2024-31744.patch Patch100: jasper-2.0.2-test-ppc64-disable.patch Patch101: jasper-2.0.2-test-ppc64le-disable.patch Patch102: jasper-4.1.0-test-i686-disable.patch # autoreconf BuildRequires: cmake -BuildRequires: freeglut-devel +BuildRequires: freeglut-devel BuildRequires: libGLU-devel BuildRequires: libjpeg-devel BuildRequires: libXmu-devel libXi-devel @@ -62,6 +63,7 @@ Requires: %{name}-libs%{?_isa} = %{version}-%{release} %prep %setup -q -n %{name}-version-%{version} +%patch 0 -p1 # Need to disable one test to be able to build it on ppc64 arch # At ppc64 this test just stuck (nothing happend - no exception or error) @@ -129,6 +131,9 @@ make test -C builder %changelog +* Tue May 21 2024 Neha Agarwal - 4.2.1-2 +- Patch CVE-2024-31744. + * Tue Feb 13 2024 Vince Perri - 4.2.1-1 - Upgrade to 4.2.1 based on Fedora 40. - License verified. @@ -277,7 +282,7 @@ make test -C builder - CVE-2015-5221 - Use-after-free and double-free flaws (#1255714) - CVE-2016-1867 - out-of-bounds read in the jpc_pi_nextcprl() function (#1298138) - CVE-2016-1577 - double free vulnerability in jas_iccattrval_destroy (#1314468) -- CVE-2016-2116 - memory leak in jas_iccprof_createfrombuf causing +- CVE-2016-2116 - memory leak in jas_iccprof_createfrombuf causing memory consumption (#1314473) * Thu Feb 04 2016 Fedora Release Engineering - 1.900.1-32 @@ -345,7 +350,7 @@ make test -C builder * Tue Oct 13 2009 Rex Dieter - 1.900.1-13 - CVE-2008-3520 jasper: multiple integer overflows in jas_alloc calls (#461476) -- CVE-2008-3522 jasper: possible buffer overflow in +- CVE-2008-3522 jasper: possible buffer overflow in jas_stream_printf() (#461478) * Fri Jul 24 2009 Fedora Release Engineering - 1.900.1-12 @@ -420,11 +425,11 @@ make test -C builder * Tue Oct 18 2005 Rex Dieter 1.701.0-6 - token %%check section -- --enable-shared +- --enable-shared * Mon Oct 17 2005 Rex Dieter 1.701.0-5 - use %%{?dist} -- BR: libGL-devel +- BR: libGL-devel * Thu Apr 7 2005 Michael Schwendt - rebuilt From f8455e82bef6bb1a1631ea2aad788aa94ba17824 Mon Sep 17 00:00:00 2001 From: Neha Agarwal <58672330+neha170@users.noreply.github.com> Date: Wed, 22 May 2024 10:12:17 -0700 Subject: [PATCH 45/51] bump release to build with fixed libuv CVE-2024-24806 (#9175) --- SPECS/cmake/cmake.spec | 5 ++++- SPECS/nodejs/nodejs.spec | 5 ++++- SPECS/python-gevent/python-gevent.spec | 5 ++++- toolkit/resources/manifests/package/toolchain_aarch64.txt | 4 ++-- toolkit/resources/manifests/package/toolchain_x86_64.txt | 4 ++-- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/SPECS/cmake/cmake.spec b/SPECS/cmake/cmake.spec index 155bcf86981..1678d847a12 100644 --- a/SPECS/cmake/cmake.spec +++ b/SPECS/cmake/cmake.spec @@ -2,7 +2,7 @@ Summary: Cmake Name: cmake Version: 3.28.2 -Release: 3%{?dist} +Release: 4%{?dist} License: BSD AND LGPLv2+ Vendor: Microsoft Corporation Distribution: Azure Linux @@ -89,6 +89,9 @@ bin/ctest --force-new-ctest-process --rerun-failed --output-on-failure %{_libdir}/rpm/macros.d/macros.cmake %changelog +* Tue May 21 2024 Neha Agarwal - 3.28.2-4 +- Bump release to build with new libuv to fix CVE-2024-24806 + * Thu May 16 2024 Daniel McIlvaney - 3.28.2-3 - Sanitize license files diff --git a/SPECS/nodejs/nodejs.spec b/SPECS/nodejs/nodejs.spec index 30c6430a193..501ad8d68d5 100644 --- a/SPECS/nodejs/nodejs.spec +++ b/SPECS/nodejs/nodejs.spec @@ -5,7 +5,7 @@ Name: nodejs # WARNINGS: MUST check and update the 'npm_version' macro for every version update of this package. # The version of NPM can be found inside the sources under 'deps/npm/package.json'. Version: 20.10.0 -Release: 1%{?dist} +Release: 2%{?dist} License: BSD AND MIT AND Public Domain AND NAIST-2003 AND Artistic-2.0 Vendor: Microsoft Corporation Distribution: Azure Linux @@ -111,6 +111,9 @@ make cctest %{_docdir}/* %changelog +* Tue May 21 2024 Neha Agarwal - 20.10.0-2 +- Bump release to build with new libuv to fix CVE-2024-24806 + * Wed Jan 31 2024 Saul Paredes - 20.10.0-1 - Upgrade to nodejs to 20.10.0 and npm to 10.2.3 diff --git a/SPECS/python-gevent/python-gevent.spec b/SPECS/python-gevent/python-gevent.spec index 265b78158d2..6622548b1ef 100644 --- a/SPECS/python-gevent/python-gevent.spec +++ b/SPECS/python-gevent/python-gevent.spec @@ -1,7 +1,7 @@ Summary: Coroutine-based network library Name: python-gevent Version: 23.9.1 -Release: 1%{?dist} +Release: 2%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -59,6 +59,9 @@ nosetests %{python3_sitelib}/* %changelog +* Tue May 21 2024 Neha Agarwal - 23.9.1-2 +- Bump release to build with new libuv to fix CVE-2024-24806 + * Mon Nov 27 2023 Tobias Brick - 23.9.1-1 - Update to 23.9.1 diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index d376be57f29..f25d581e34e 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -45,8 +45,8 @@ check-debuginfo-0.15.2-1.azl3.aarch64.rpm chkconfig-1.25-1.azl3.aarch64.rpm chkconfig-debuginfo-1.25-1.azl3.aarch64.rpm chkconfig-lang-1.25-1.azl3.aarch64.rpm -cmake-3.28.2-3.azl3.aarch64.rpm -cmake-debuginfo-3.28.2-3.azl3.aarch64.rpm +cmake-3.28.2-4.azl3.aarch64.rpm +cmake-debuginfo-3.28.2-4.azl3.aarch64.rpm coreutils-9.4-2.azl3.aarch64.rpm coreutils-debuginfo-9.4-2.azl3.aarch64.rpm coreutils-lang-9.4-2.azl3.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index f60a15d06f3..2d79dfc0f94 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -46,8 +46,8 @@ check-debuginfo-0.15.2-1.azl3.x86_64.rpm chkconfig-1.25-1.azl3.x86_64.rpm chkconfig-debuginfo-1.25-1.azl3.x86_64.rpm chkconfig-lang-1.25-1.azl3.x86_64.rpm -cmake-3.28.2-3.azl3.x86_64.rpm -cmake-debuginfo-3.28.2-3.azl3.x86_64.rpm +cmake-3.28.2-4.azl3.x86_64.rpm +cmake-debuginfo-3.28.2-4.azl3.x86_64.rpm coreutils-9.4-2.azl3.x86_64.rpm coreutils-debuginfo-9.4-2.azl3.x86_64.rpm coreutils-lang-9.4-2.azl3.x86_64.rpm From 7d98558e215860ad20bd9dd16f7bb703593cc2ad Mon Sep 17 00:00:00 2001 From: Sam Meluch <109628994+sameluch@users.noreply.github.com> Date: Wed, 22 May 2024 15:13:39 -0700 Subject: [PATCH 46/51] PTest Fix for python-pytest-flakes Package (#9181) --- SPECS/python-pytest-flakes/python-pytest-flakes.spec | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/SPECS/python-pytest-flakes/python-pytest-flakes.spec b/SPECS/python-pytest-flakes/python-pytest-flakes.spec index 9fe2888bf37..3ff6ef3a2c2 100644 --- a/SPECS/python-pytest-flakes/python-pytest-flakes.spec +++ b/SPECS/python-pytest-flakes/python-pytest-flakes.spec @@ -3,7 +3,7 @@ Name: python-%{srcname} Version: 4.0.5 -Release: 8%{?dist} +Release: 9%{?dist} Summary: Pytest plugin to check source code with pyflakes License: MIT @@ -22,6 +22,9 @@ BuildRequires: python3-devel BuildRequires: python3-setuptools BuildRequires: python3dist(pytest) >= 2.8 BuildRequires: python3dist(pyflakes) +%if 0%{?with_check} +BuildRequires: python3-pip +%endif %description -n python3-%{srcname} Py.test plugin for efficiently checking python source with pyflakes. @@ -49,6 +52,9 @@ rm -rf *.egg-info %{python3_sitelib}/__pycache__/pytest_flakes.* %changelog +* Mon May 20 2024 Sam Meluch - 4.0.5-9 +- Add pip test dependency to fix package tests + * Tue Feb 27 2024 Dan Streetman - 4.0.5-8 - Initial CBL-Mariner import from Fedora 39 (license: MIT). - license verified From 38ea0638c02f1c458460e8fa74a8a39fd614ff2c Mon Sep 17 00:00:00 2001 From: Pawel Winogrodzki Date: Wed, 22 May 2024 15:35:36 -0700 Subject: [PATCH 47/51] Specialcased 5XX errors during package downloads. (CP: #7445) (#9135) Co-authored-by: CBL-Mariner-Bot <75509084+CBL-Mariner-Bot@users.noreply.github.com> --- .../repocloner/rpmrepocloner/rpmrepocloner.go | 108 ++++++++++++------ toolkit/tools/internal/retry/retry.go | 16 +++ toolkit/tools/pkg/downloader/downloader.go | 17 +-- toolkit/tools/precacher/precacher.go | 11 +- toolkit/tools/srpmpacker/srpmpacker.go | 12 +- 5 files changed, 94 insertions(+), 70 deletions(-) diff --git a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go index 351e5894599..2ea700d176b 100644 --- a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go +++ b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" "github.com/microsoft/azurelinux/toolkit/tools/internal/buildpipeline" @@ -16,6 +17,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/packagerepo/repocloner" "github.com/microsoft/azurelinux/toolkit/tools/internal/packagerepo/repomanager/rpmrepomanager" "github.com/microsoft/azurelinux/toolkit/tools/internal/pkgjson" + "github.com/microsoft/azurelinux/toolkit/tools/internal/retry" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" "github.com/microsoft/azurelinux/toolkit/tools/internal/tdnf" @@ -51,6 +53,11 @@ const ( useMultipleTransactions = !useSingleTransaction ) +var ( + serverErrorsRegex = regexp.MustCompile(`(?m)Error: (5\d{2}) when downloading`) + serverErrorCodeIndex = 1 +) + // RpmRepoCloner represents an RPM repository cloner. type RpmRepoCloner struct { chroot *safechroot.Chroot @@ -595,11 +602,6 @@ func (r *RpmRepoCloner) Close() error { // clonePackage clones a given package using pre-populated arguments. // It will gradually enable more repos to consider until the package is found. func (r *RpmRepoCloner) clonePackage(baseArgs []string) (preBuilt bool, err error) { - const ( - unresolvedOutputPrefix = "No package" - toyboxConflictsPrefix = "toybox conflicts" - unresolvedOutputPostfix = "available" - ) releaseverCliArg, err := tdnf.GetReleaseverCliArg() if err != nil { @@ -613,40 +615,23 @@ func (r *RpmRepoCloner) clonePackage(baseArgs []string) (preBuilt bool, err erro finalArgs := append(baseArgs, reposArgs...) - var ( - stdout string - stderr string - ) - stdout, stderr, err = shell.Execute("tdnf", finalArgs...) - - logger.Log.Debugf("stdout: %s", stdout) - logger.Log.Debugf("stderr: %s", stderr) - - if err != nil { - logger.Log.Debugf("tdnf error (will continue if the only errors are toybox conflicts):\n '%s'", stderr) - } - - // ============== TDNF SPECIFIC IMPLEMENTATION ============== - // Check if TDNF could not resolve a given package. If TDNF does not find a requested package, - // it will not error. Instead it will print a message to stdout. Check for this message. - // - // *NOTE*: TDNF will attempt best effort. If N packages are requested, and 1 cannot be found, - // it will still download N-1 packages while also printing the message. - splitStdout := strings.Split(stdout, "\n") - for _, line := range splitStdout { - trimmedLine := strings.TrimSpace(line) - // Toybox conflicts are a known issue, reset the err value if encountered - if strings.HasPrefix(trimmedLine, toyboxConflictsPrefix) { - logger.Log.Warn("Ignoring known toybox conflict") - err = nil - continue - } - // If a package was not available, update err - if strings.HasPrefix(trimmedLine, unresolvedOutputPrefix) && strings.HasSuffix(trimmedLine, unresolvedOutputPostfix) { - err = fmt.Errorf(trimmedLine) - break + // We run in a retry loop on errors deemed retriable. + cancel := make(chan struct{}) + retryNum := 1 + _, err = retry.RunWithDefaultDownloadBackoff(func() error { + downloadErr, retriable := tdnfDownload(finalArgs...) + if downloadErr != nil { + if retriable { + logger.Log.Debugf("Package cloning attempt %d/%d failed with a retriable error.", retryNum, retry.DefaultDownloadRetryAttempts) + } else { + logger.Log.Debugf("Package cloning attempt %d/%d failed with an unrecoverable error. Cancelling.", retryNum, retry.DefaultDownloadRetryAttempts) + close(cancel) + } } - } + + retryNum++ + return downloadErr + }, cancel) if err == nil { preBuilt = r.reposArgsHaveOnlyLocalSources(reposArgs) @@ -808,3 +793,50 @@ func (r *RpmRepoCloner) reposArgsHaveOnlyLocalSources(reposArgs []string) bool { return true } + +func tdnfDownload(args ...string) (err error, retriable bool) { + const ( + unresolvedOutputPrefix = "No package" + unresolvedOutputSuffix = "available" + ) + + stdout, stderr, err := shell.Execute("tdnf", args...) + + logger.Log.Debugf("stdout: %s", stdout) + logger.Log.Debugf("stderr: %s", stderr) + + // ============== TDNF SPECIFIC IMPLEMENTATION ============== + // + // Check if TDNF could not resolve a given package. If TDNF does not find a requested package, + // it will not error. Instead it will print a message to stdout. Check for this message. + // + // *NOTE*: TDNF will attempt best effort. If N packages are requested, and 1 cannot be found, + // it will still download N-1 packages while also printing the message. + splitStdout := strings.Split(stdout, "\n") + for _, line := range splitStdout { + trimmedLine := strings.TrimSpace(line) + // If a package was not available, update err + if strings.HasPrefix(trimmedLine, unresolvedOutputPrefix) && strings.HasSuffix(trimmedLine, unresolvedOutputSuffix) { + err = fmt.Errorf(trimmedLine) + return + } + } + + // + // *NOTE*: There are cases in which some of our upstream package repositories are hosted + // on services that are prone to intermittent errors (e.g., HTTP 502 errors). We + // specifically look for such known cases and apply some retry logic in hopes of getting + // a better result; note that we don't indiscriminately retry because there are legitimate + // cases in which the upstream repo doesn't contain the package and a 404 error is to be + // expected. This involves scraping through stderr, but it's better than not doing so. + // + if err != nil { + serverErrorMatch := serverErrorsRegex.FindStringSubmatch(stderr) + if len(serverErrorMatch) > serverErrorCodeIndex { + logger.Log.Debugf("Encountered possibly intermittent HTTP %s error.", serverErrorMatch[serverErrorCodeIndex]) + retriable = true + } + } + + return +} diff --git a/toolkit/tools/internal/retry/retry.go b/toolkit/tools/internal/retry/retry.go index dd218ec2ab0..5e218f55378 100644 --- a/toolkit/tools/internal/retry/retry.go +++ b/toolkit/tools/internal/retry/retry.go @@ -8,6 +8,14 @@ import ( "time" ) +const ( + // With 5 attempts (4 retries) and a backoff factor of 2 seconds the total time spent retrying will be approximately: + // 1 + 4 + 8 + 16 = 31 seconds. + DefaultDownloadBackoffBase = 2.0 + DefaultDownloadRetryAttempts = 5 + DefaultDownloadRetryDuration = time.Second +) + // calculateDelay calculates the delay for the given failure count, sleep duration, and backoff exponent base. // If the base is positive, it will calculate an exponential backoff. func calculateExpDelay(failCount int, sleep time.Duration, backoffExponentBase float64) time.Duration { @@ -83,6 +91,14 @@ func RunWithLinearBackoff(function func() error, attempts int, sleep time.Durati }, attempts, cancel) } +// RunWithDefaultDownloadBackoff runs function up to 'DefaultDownloadRetryAttempts' times, waiting 'DefaultDownloadBackoffBase^(i-1)' seconds before +// each i-th attempt. An optional cancel channel can be provided to cancel the retry loop immediately by closing the channel. +// +// The function is meant as a default for network download operations. +func RunWithDefaultDownloadBackoff(function func() error, cancel <-chan struct{}) (wasCancelled bool, err error) { + return RunWithExpBackoff(function, DefaultDownloadRetryAttempts, DefaultDownloadRetryDuration, DefaultDownloadBackoffBase, cancel) +} + // RunWithExpBackoff runs function up to 'attempts' times, waiting 'backoffExponentBase^(i-1) * sleep' duration before // each i-th attempt. An optional cancel channel can be provided to cancel the retry loop immediately by closing the channel. func RunWithExpBackoff(function func() error, attempts int, sleep time.Duration, backoffExponentBase float64, cancel <-chan struct{}) (wasCancelled bool, err error) { diff --git a/toolkit/tools/pkg/downloader/downloader.go b/toolkit/tools/pkg/downloader/downloader.go index 95b2806de90..2b539f8e95b 100644 --- a/toolkit/tools/pkg/downloader/downloader.go +++ b/toolkit/tools/pkg/downloader/downloader.go @@ -9,7 +9,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "time" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/network" @@ -17,32 +16,24 @@ import ( ) func DownloadFile(srcUrl, dstFile string, caCerts *x509.CertPool, tlsCerts []tls.Certificate) (err error) { - const ( - // With 6 attempts, initial delay of 1 second, and a backoff factor of 3.0 the total time spent retrying will be - // 1 + 3 + 9 + 27 + 81 = 121 seconds. - downloadRetryAttempts = 6 - failureBackoffBase = 3.0 - downloadRetryDuration = time.Second - ) cancel := make(chan struct{}) - retryNum := 1 - _, err = retry.RunWithExpBackoff(func() error { + _, err = retry.RunWithDefaultDownloadBackoff(func() error { netErr := network.DownloadFile(srcUrl, dstFile, caCerts, tlsCerts) if netErr != nil { // Check if the error contains the string "invalid response: 404", we should print a warning in that case so the // sees it even if we are running with --no-verbose. 404's are unlikely to fix themselves on retry, give up. if netErr.Error() == "invalid response: 404" { - logger.Log.Warnf("Attempt %d/%d: Failed to download (%s) with error: (%s)", retryNum, downloadRetryAttempts, srcUrl, netErr) + logger.Log.Warnf("Attempt %d/%d: Failed to download (%s) with error: (%s)", retryNum, retry.DefaultDownloadRetryAttempts, srcUrl, netErr) logger.Log.Warnf("404 errors are likely unrecoverable, will not retry") close(cancel) } else { - logger.Log.Infof("Attempt %d/%d: Failed to download (%s) with error: (%s)", retryNum, downloadRetryAttempts, srcUrl, netErr) + logger.Log.Infof("Attempt %d/%d: Failed to download (%s) with error: (%s)", retryNum, retry.DefaultDownloadRetryAttempts, srcUrl, netErr) } } retryNum++ return netErr - }, downloadRetryAttempts, downloadRetryDuration, failureBackoffBase, cancel) + }, cancel) if err != nil { err = fmt.Errorf("failed to download (%s) to (%s):\n%w", srcUrl, dstFile, err) diff --git a/toolkit/tools/precacher/precacher.go b/toolkit/tools/precacher/precacher.go index d2d79e0b18b..b4a07f4c38e 100644 --- a/toolkit/tools/precacher/precacher.go +++ b/toolkit/tools/precacher/precacher.go @@ -214,13 +214,6 @@ func monitorProgress(total int, results chan downloadResult, doneChannel chan st // responsible for removing itself from the wait group. As much processing as possible is done before acquiring the // network operations semaphore to minimize the time spent holding it. func precachePackage(pkg *repocloner.RepoPackage, packagesAvailableFromRepos map[string]string, outDir string, wg *sync.WaitGroup, results chan<- downloadResult, netOpsSemaphore chan struct{}) { - const ( - // With 5 attempts, initial delay of 1 second, and a backoff factor of 2.0 the total time spent retrying will be - // ~30 seconds. - downloadRetryAttempts = 5 - failureBackoffBase = 2.0 - downloadRetryDuration = time.Second - ) var noCancel chan struct{} = nil // File names are of the form "-...rpm" @@ -262,13 +255,13 @@ func precachePackage(pkg *repocloner.RepoPackage, packagesAvailableFromRepos map }() logger.Log.Debugf("Pre-caching '%s' from '%s'", fileName, url) - _, err = retry.RunWithExpBackoff(func() error { + _, err = retry.RunWithDefaultDownloadBackoff(func() error { err := network.DownloadFile(url, fullFilePath, nil, nil) if err != nil { logger.Log.Warnf("Attempt to download (%s) failed. Error: %s", url, err) } return err - }, downloadRetryAttempts, downloadRetryDuration, failureBackoffBase, noCancel) + }, noCancel) if err != nil { return } diff --git a/toolkit/tools/srpmpacker/srpmpacker.go b/toolkit/tools/srpmpacker/srpmpacker.go index 43f30985e1d..30112a497a9 100644 --- a/toolkit/tools/srpmpacker/srpmpacker.go +++ b/toolkit/tools/srpmpacker/srpmpacker.go @@ -14,7 +14,6 @@ import ( "runtime" "strings" "sync" - "time" "github.com/microsoft/azurelinux/toolkit/tools/internal/buildpipeline" "github.com/microsoft/azurelinux/toolkit/tools/internal/directory" @@ -912,13 +911,6 @@ func tryToHydrateFromLocalSource(fileHydrationState map[string]bool, newSourceDi // hydrateFromRemoteSource will update fileHydrationState. // Will alter `currentSignatures`. func hydrateFromRemoteSource(fileHydrationState map[string]bool, newSourceDir string, srcConfig sourceRetrievalConfiguration, skipSignatureHandling bool, currentSignatures map[string]string, cancel <-chan struct{}, netOpsSemaphore chan struct{}) (err error) { - const ( - // With 5 attempts, initial delay of 1 second, and a backoff factor of 2.0 the total time spent retrying will be - // ~30 seconds. - downloadRetryAttempts = 5 - failureBackoffBase = 2.0 - downloadRetryDuration = time.Second - ) errPackerCancelReceived := fmt.Errorf("packer cancel signal received") for fileName, alreadyHydrated := range fileHydrationState { @@ -942,14 +934,14 @@ func hydrateFromRemoteSource(fileHydrationState map[string]bool, newSourceDir st } } - cancelled, internalErr := retry.RunWithExpBackoff(func() error { + cancelled, internalErr := retry.RunWithDefaultDownloadBackoff(func() error { downloadErr := network.DownloadFile(url, destinationFile, srcConfig.caCerts, srcConfig.tlsCerts) if downloadErr != nil { logger.Log.Debugf("Failed an attempt to download (%s). Error: %s.", url, downloadErr) } return downloadErr - }, downloadRetryAttempts, downloadRetryDuration, failureBackoffBase, cancel) + }, cancel) if netOpsSemaphore != nil { // Clear the channel to allow another operation to start From 274ee5e86b498c1b7f703ac6f6d6603eeb521f6a Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Thu, 23 May 2024 12:20:10 -0700 Subject: [PATCH 48/51] Image Customizer: Add support for inline scripts. (#9178) --- .../imagecustomizer/docs/configuration.md | 83 ++++++++- toolkit/tools/imagecustomizerapi/script.go | 23 ++- .../pkg/imagecustomizerlib/customizeutils.go | 42 +---- .../pkg/imagecustomizerlib/imagecustomizer.go | 28 ++- .../imagecustomizer_test.go | 11 -- .../pkg/imagecustomizerlib/runscripts.go | 164 ++++++++++++++++++ .../testdata/run-python-scripts-config.yaml | 20 +++ .../testdata/runscripts-config.yaml | 12 ++ .../testdata/scripts/consume-space.sh | 2 - .../scripts/finalizecustomizationscript.sh | 3 +- .../testdata/scripts/llamas.py | 4 + .../scripts/postcustomizationscript.sh | 3 +- 12 files changed, 317 insertions(+), 78 deletions(-) create mode 100644 toolkit/tools/pkg/imagecustomizerlib/runscripts.go create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/run-python-scripts-config.yaml create mode 100755 toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/llamas.py diff --git a/toolkit/tools/imagecustomizer/docs/configuration.md b/toolkit/tools/imagecustomizer/docs/configuration.md index b67c380d6c2..159434a5651 100644 --- a/toolkit/tools/imagecustomizer/docs/configuration.md +++ b/toolkit/tools/imagecustomizer/docs/configuration.md @@ -188,11 +188,19 @@ os: - [postCustomization](#postcustomization-script) - [script type](#script-type) - [path](#script-path) + - [content](#content-string) + - [interpreter](#interpreter-string) - [args](#args-string) + - [environmentVariables](#environmentvariables-mapstring-string) + - [name](#script-name) - [finalizeCustomization](#finalizecustomization-script) - [script type](#script-type) - [path](#script-path) + - [content](#content-string) + - [interpreter](#interpreter-string) - [args](#args-string) + - [environmentVariables](#environmentvariables-mapstring-string) + - [name](#script-name) ## Top-level @@ -890,6 +898,8 @@ The path of the script. This must be in the same directory or a sub-directory that the config file is located in. +Only one of `path` or `content` may be specified. + Example: ```yaml @@ -898,7 +908,40 @@ scripts: - path: scripts/a.sh ``` -### args [string] +### content [string] + +The contents of the script to run. + +The script is written to a temporary file under the customized OS's `/tmp` directory. + +Only one of `path` or `content` may be specified. + +Example: + +```yaml +scripts: + postCustomization: + - content: | + echo "Hello, World" +``` + +### interpreter [string] + +The program to run the script with. + +If not specified, then the script is run by `/bin/sh`. + +Example: + +```yaml +scripts: + postCustomization: + - content: | + print("Hello, World") + interpreter: python3 +``` + +### args [string[]] Additional arguments to pass to the script. @@ -908,7 +951,43 @@ Example: scripts: postCustomization: - path: scripts/a.sh - args: abc + args: + - abc +``` + +### environmentVariables [map\] + +Additional environment variables to set on the program. + +Example: + +```yaml +scripts: + postCustomization: + - content: | + echo "$a $b" + environmentVariables: + a: hello + b: world +``` + +
+ +### name [string] + +The name of the script. + +This field is only used to refer to the script in the logs. +It is particularly useful when `content` is used. + +Example: + +```yaml +scripts: + postCustomization: + - content: | + echo "Hello, World" + name: greetings ``` ## scripts type diff --git a/toolkit/tools/imagecustomizerapi/script.go b/toolkit/tools/imagecustomizerapi/script.go index 7c626d9570d..a2de828de3f 100644 --- a/toolkit/tools/imagecustomizerapi/script.go +++ b/toolkit/tools/imagecustomizerapi/script.go @@ -8,13 +8,30 @@ import ( ) type Script struct { + // Path is the path of the script file. + // Mutually exclusive with 'Content'. Path string `yaml:"path"` - Args string `yaml:"args"` + // Content is the inline string content of the script to run. + // Mutually exclusive with 'Path'. + Content string `yaml:"content"` + // Interpreter is the name or path of the process that will execute the script. + // When 'Content' is specified, the default value is '/bin/sh'. + // When 'Path' is specified and 'interpreter' is not specified, then the script file is executed directly. + Interpreter string `yaml:"interpreter"` + // Arguments is a list of additional arguments to pass to the script. + Arguments []string `yaml:"arguments"` + // EnvironmentVariables are a set of environment variables to set when executing the script. + EnvironmentVariables map[string]string `yaml:"environmentVariables"` + // Name is an optional value used to reference the script in the logs. + Name string `yaml:"name"` } func (s *Script) IsValid() error { - if s.Path == "" { - return fmt.Errorf("value of path may not be empty") + if s.Path == "" && s.Content == "" { + return fmt.Errorf("either path or content must have a value") + } + if s.Path != "" && s.Content != "" { + return fmt.Errorf("path and content may not both have a value") } return nil diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go b/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go index 676e149aa51..a8bdab31008 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go @@ -17,11 +17,9 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" - "github.com/microsoft/azurelinux/toolkit/tools/internal/safemount" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" "github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils" "github.com/microsoft/azurelinux/toolkit/tools/internal/userutils" - "golang.org/x/sys/unix" ) const ( @@ -114,7 +112,7 @@ func doCustomizations(buildDir string, baseConfigPath string, config *imagecusto } if config.Scripts != nil { - err = runScripts(baseConfigPath, config.Scripts.PostCustomization, imageChroot) + err = runUserScripts(baseConfigPath, config.Scripts.PostCustomization, "postCustomization", imageChroot) if err != nil { return err } @@ -131,7 +129,7 @@ func doCustomizations(buildDir string, baseConfigPath string, config *imagecusto } if config.Scripts != nil { - err = runScripts(baseConfigPath, config.Scripts.FinalizeCustomization, imageChroot) + err = runUserScripts(baseConfigPath, config.Scripts.FinalizeCustomization, "finalizeCustomization", imageChroot) if err != nil { return err } @@ -247,42 +245,6 @@ func copyAdditionalDirs(baseConfigPath string, additionalDirs imagecustomizerapi return nil } -func runScripts(baseConfigPath string, scripts []imagecustomizerapi.Script, imageChroot *safechroot.Chroot) error { - if len(scripts) <= 0 { - return nil - } - - configDirMountPath := filepath.Join(imageChroot.RootDir(), configDirMountPathInChroot) - - // Bind mount the config directory so that the scripts can access any required resources. - mount, err := safemount.NewMount(baseConfigPath, configDirMountPath, "", unix.MS_BIND|unix.MS_RDONLY, "", true) - if err != nil { - return err - } - defer mount.Close() - - for _, script := range scripts { - scriptPathInChroot := filepath.Join(configDirMountPathInChroot, script.Path) - command := fmt.Sprintf("%s %s", scriptPathInChroot, script.Args) - logger.Log.Infof("Running script (%s)", script.Path) - - // Run the script. - err = imageChroot.UnsafeRun(func() error { - return shell.ExecuteLiveWithErr(1, shell.ShellProgram, "-c", command) - }) - if err != nil { - return fmt.Errorf("script (%s) failed:\n%w", script.Path, err) - } - } - - err = mount.CleanClose() - if err != nil { - return err - } - - return nil -} - func AddOrUpdateUsers(users []imagecustomizerapi.User, baseConfigPath string, imageChroot safechroot.ChrootInterface) error { for _, user := range users { err := addOrUpdateUser(user, baseConfigPath, imageChroot) diff --git a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go index 50acf7c08d7..9848177fa98 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go @@ -509,7 +509,6 @@ func validateSystemConfig(baseConfigPath string, config *imagecustomizerapi.OS, } func validateScripts(baseConfigPath string, scripts *imagecustomizerapi.Scripts) error { - if scripts == nil { return nil } @@ -532,23 +531,20 @@ func validateScripts(baseConfigPath string, scripts *imagecustomizerapi.Scripts) } func validateScript(baseConfigPath string, script *imagecustomizerapi.Script) error { - // Ensure that install scripts sit under the config file's parent directory. - // This allows the install script to be run in the chroot environment by bind mounting the config directory. - if !filepath.IsLocal(script.Path) { - return fmt.Errorf("install script (%s) is not under config directory (%s)", script.Path, baseConfigPath) - } - - // Verify that the file exists. - fullPath := filepath.Join(baseConfigPath, script.Path) + if script.Path != "" { + // Ensure that install scripts sit under the config file's parent directory. + // This allows the install script to be run in the chroot environment by bind mounting the config directory. + if !filepath.IsLocal(script.Path) { + return fmt.Errorf("script file (%s) is not under config directory (%s)", script.Path, baseConfigPath) + } - scriptStat, err := os.Stat(fullPath) - if err != nil { - return fmt.Errorf("couldn't read install script (%s):\n%w", script.Path, err) - } + fullPath := filepath.Join(baseConfigPath, script.Path) - // Verify that the file has an executable bit set. - if scriptStat.Mode()&0111 == 0 { - return fmt.Errorf("install script (%s) does not have executable bit set", script.Path) + // Verify that the file exists. + _, err := os.Stat(fullPath) + if err != nil { + return fmt.Errorf("couldn't read script file (%s):\n%w", script.Path, err) + } } return nil diff --git a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go index 578207508ee..8888d48460d 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go @@ -183,17 +183,6 @@ func TestValidateConfigScriptNonLocalFile(t *testing.T) { assert.Error(t, err) } -func TestValidateConfigScriptNonExecutable(t *testing.T) { - err := validateScripts(testDir, &imagecustomizerapi.Scripts{ - FinalizeCustomization: []imagecustomizerapi.Script{ - { - Path: "files/a.txt", - }, - }, - }) - assert.Error(t, err) -} - func TestCustomizeImageKernelCommandLineAdd(t *testing.T) { var err error diff --git a/toolkit/tools/pkg/imagecustomizerlib/runscripts.go b/toolkit/tools/pkg/imagecustomizerlib/runscripts.go new file mode 100644 index 00000000000..5b07bfa7fbf --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/runscripts.go @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package imagecustomizerlib + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi" + "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" + "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" + "github.com/microsoft/azurelinux/toolkit/tools/internal/safemount" + "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "golang.org/x/sys/unix" +) + +func runUserScripts(baseConfigPath string, scripts []imagecustomizerapi.Script, listName string, + imageChroot *safechroot.Chroot, +) error { + if len(scripts) <= 0 { + return nil + } + + logger.Log.Infof("Running %s scripts", listName) + + configDirMountPath := filepath.Join(imageChroot.RootDir(), configDirMountPathInChroot) + + // Bind mount the config directory so that the scripts can access any required resources. + mount, err := safemount.NewMount(baseConfigPath, configDirMountPath, "", unix.MS_BIND|unix.MS_RDONLY, "", true) + if err != nil { + return err + } + defer mount.Close() + + // Runs scripts. + for i, script := range scripts { + err := runUserScript(i, script, listName, imageChroot) + if err != nil { + return err + } + } + + err = mount.CleanClose() + if err != nil { + return err + } + + return nil +} + +func runUserScript(scriptIndex int, script imagecustomizerapi.Script, listName string, + imageChroot *safechroot.Chroot, +) error { + var err error + + scriptLogName := createScriptLogName(scriptIndex, script, listName) + + logger.Log.Infof("Running script (%s)", scriptLogName) + + // Collect the process name and args. + scriptPath := "" + tempScriptFullPath := "" + if script.Path != "" { + scriptPath = filepath.Join(configDirMountPathInChroot, script.Path) + } else { + // Write the script to a temporary file. + tempScriptFullPath, err = createTempScriptFile(script, listName, scriptLogName, imageChroot) + if err != nil { + return err + } + defer os.Remove(tempScriptFullPath) + + // Get the path of the script file in the chroot. + tempScriptPath, err := filepath.Rel(imageChroot.RootDir(), tempScriptFullPath) + if err != nil { + return fmt.Errorf("failed to get relative path for temp script file:\n%w", err) + } + + // Ensure path is rooted. + tempScriptPath = filepath.Join("/", tempScriptPath) + + scriptPath = tempScriptPath + } + + process := script.Interpreter + if process == "" { + process = "/bin/sh" + } + + args := []string{scriptPath} + args = append(args, script.Arguments...) + + envVars := []string(nil) + for key, value := range script.EnvironmentVariables { + envVar := fmt.Sprintf("%s=%s", key, value) + envVars = append(envVars, envVar) + } + + // Run the script. + err = imageChroot.UnsafeRun(func() error { + return shell.NewExecBuilder(process, args...). + EnvironmentVariables(envVars). + ErrorStderrLines(1). + Execute() + }) + if err != nil { + return fmt.Errorf("script (%s) failed:\n%w", scriptLogName, err) + } + + if tempScriptFullPath != "" { + // Remove the script file and error out if the delete fails. + err = os.Remove(tempScriptFullPath) + if err != nil { + return fmt.Errorf("failed to remove temp script file:\n%w", err) + } + } + + return nil +} + +func createScriptLogName(scriptIndex int, script imagecustomizerapi.Script, listName string) string { + switch { + case script.Name != "" && script.Path != "": + return fmt.Sprintf("%s(%s)", script.Name, script.Path) + case script.Name != "": + return script.Name + case script.Path != "": + return script.Path + default: + return fmt.Sprintf("%s[%d]", listName, scriptIndex) + } +} + +func createTempScriptFile(script imagecustomizerapi.Script, listName string, scriptLogName string, + imageChroot *safechroot.Chroot, +) (string, error) { + chrootTempDir := filepath.Join(imageChroot.RootDir(), "tmp") + + // Create a temporary file for the script. + tempFile, err := os.CreateTemp(chrootTempDir, listName) + if err != nil { + return "", fmt.Errorf("failed to create temp file for script:\n%w", err) + } + defer tempFile.Close() + + tempFilePath := tempFile.Name() + logger.Log.Debugf("Writing script's (%s) content to file (%s)", scriptLogName, tempFilePath) + + // Write the script's content. + _, err = tempFile.WriteString(script.Content) + if err != nil { + return "", fmt.Errorf("failed to write temp file for script:\n%w", err) + } + + // Ensure the file is written correctly. + err = tempFile.Close() + if err != nil { + return "", fmt.Errorf("failed to close temp file for script:\n%w", err) + } + + return tempFilePath, nil +} diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/run-python-scripts-config.yaml b/toolkit/tools/pkg/imagecustomizerlib/testdata/run-python-scripts-config.yaml new file mode 100644 index 00000000000..3220c03949a --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/run-python-scripts-config.yaml @@ -0,0 +1,20 @@ +os: + packages: + install: + - python3 + +scripts: + postCustomization: + - content: | + import os + import sys + print(f"{sys.argv[1]} and {sys.argv[2]}") + print(f"Working dir: {os.getcwd()}") + arguments: + - cats + - dogs + interpreter: python3 + name: raining + + - path: scripts/llamas.py + interpreter: python3 diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/runscripts-config.yaml b/toolkit/tools/pkg/imagecustomizerlib/testdata/runscripts-config.yaml index 96ae9ebf23c..62adb49d0cd 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/testdata/runscripts-config.yaml +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/runscripts-config.yaml @@ -2,5 +2,17 @@ scripts: postCustomization: - path: scripts/postcustomizationscript.sh + - content: | + echo "This is an postCustomization inline script" + echo "$1 $2" + echo "$fruit and $vegetable" + echo "Working dir: $(pwd)" + arguments: + - hello + - world + environmentVariables: + fruit: bananas + vegetable: carrots + finalizeCustomization: - path: scripts/finalizecustomizationscript.sh diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/consume-space.sh b/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/consume-space.sh index aed667ffbc1..a5cb66f2e62 100755 --- a/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/consume-space.sh +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/consume-space.sh @@ -1,4 +1,2 @@ -#!/usr/bin/env bash - # Write out a file that consumes the rest of the space on the rootfs partition. dd if=/dev/zero of=/bigfile bs=512 || true diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/finalizecustomizationscript.sh b/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/finalizecustomizationscript.sh index 50e1fa369ee..644e6b05681 100755 --- a/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/finalizecustomizationscript.sh +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/finalizecustomizationscript.sh @@ -1,3 +1,2 @@ -#!/usr/bin/env bash - echo "A finalize customization script" +echo "Working dir: $(pwd)" diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/llamas.py b/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/llamas.py new file mode 100755 index 00000000000..539d998230c --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/llamas.py @@ -0,0 +1,4 @@ +import os + +print("llamas") +print(f"Working dir: {os.getcwd()}") diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/postcustomizationscript.sh b/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/postcustomizationscript.sh index 542ffb90819..a07c9470ce4 100755 --- a/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/postcustomizationscript.sh +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/scripts/postcustomizationscript.sh @@ -1,3 +1,2 @@ -#!/usr/bin/env bash - echo "A post customization script" +echo "Working dir: $(pwd)" From 8edc9a62d10b5c46abbb64957c3d14079e581a03 Mon Sep 17 00:00:00 2001 From: Neha Agarwal <58672330+neha170@users.noreply.github.com> Date: Thu, 23 May 2024 12:22:55 -0700 Subject: [PATCH 49/51] rubygem-rexml: fix CVE-2024-35176 (#9195) --- SPECS/ruby/ruby.spec | 5 +- .../add-support-for-old-strscan.patch | 129 ++++++++++++++++++ .../rubygem-rexml.signatures.json | 6 +- SPECS/rubygem-rexml/rubygem-rexml.spec | 11 +- cgmanifest.json | 4 +- 5 files changed, 146 insertions(+), 9 deletions(-) create mode 100644 SPECS/rubygem-rexml/add-support-for-old-strscan.patch diff --git a/SPECS/ruby/ruby.spec b/SPECS/ruby/ruby.spec index 8b1254a353e..995aca5fe35 100644 --- a/SPECS/ruby/ruby.spec +++ b/SPECS/ruby/ruby.spec @@ -88,7 +88,7 @@ Name: ruby # provides should be versioned according to the ruby version. # More info: https://stdgems.org/ Version: %{ruby_version} -Release: 3%{?dist} +Release: 4%{?dist} License: (Ruby OR BSD) AND Public Domain AND MIT AND CC0 AND zlib AND UCD Vendor: Microsoft Corporation Distribution: Azure Linux @@ -407,6 +407,9 @@ sudo -u test make test TESTS="-v" %{_rpmconfigdir}/rubygems.con %changelog +* Wed May 22 2024 Neha Agarwal - 3.3.0-4 +- Bump release to build with new rubygem-rexml to fix CVE-2024-35176 + * Mon Apr 01 2024 Riken Maharjan - 3.3.0-3 - Change the 'gemspec_clear_signing' macro to delete gem.signature also. diff --git a/SPECS/rubygem-rexml/add-support-for-old-strscan.patch b/SPECS/rubygem-rexml/add-support-for-old-strscan.patch new file mode 100644 index 00000000000..1c848deb05a --- /dev/null +++ b/SPECS/rubygem-rexml/add-support-for-old-strscan.patch @@ -0,0 +1,129 @@ +From f1df7d13b3e57a5e059273d2f0870163c08d7420 Mon Sep 17 00:00:00 2001 +From: Sutou Kouhei +Date: Mon, 20 May 2024 12:17:27 +0900 +Subject: [PATCH] Add support for old strscan + +Fix GH-132 + +If we support old strscan, users can also use strscan installed as a +default gem. + +Reported by Adam. Thanks!!! +--- + .github/workflows/test.yml | 32 ++++++++++++++++++++++---------- + lib/rexml/parsers/baseparser.rb | 11 +++++++++++ + rexml.gemspec | 2 +- + 3 files changed, 34 insertions(+), 11 deletions(-) + +diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml +index fd26b9a..f977de6 100644 +--- a/.github/workflows/test.yml ++++ b/.github/workflows/test.yml +@@ -3,14 +3,14 @@ on: + - push + - pull_request + jobs: +- ruby-versions: ++ ruby-versions-inplace: + uses: ruby/actions/.github/workflows/ruby_versions.yml@master + with: + engine: cruby-jruby + min_version: 2.5 + + inplace: +- needs: ruby-versions ++ needs: ruby-versions-inplace + name: "Inplace: ${{ matrix.ruby-version }} on ${{ matrix.runs-on }}" + runs-on: ${{ matrix.runs-on }} + strategy: +@@ -20,7 +20,7 @@ jobs: + - ubuntu-latest + - macos-latest + - windows-latest +- ruby-version: ${{ fromJson(needs.ruby-versions.outputs.versions) }} ++ ruby-version: ${{ fromJson(needs.ruby-versions-inplace.outputs.versions) }} + exclude: + - {runs-on: macos-latest, ruby-version: 2.5} + # include: +@@ -47,7 +47,14 @@ jobs: + - name: Test + run: bundle exec rake test RUBYOPT="--enable-frozen-string-literal" + ++ ruby-versions-gem: ++ uses: ruby/actions/.github/workflows/ruby_versions.yml@master ++ with: ++ engine: cruby-jruby ++ min_version: 3.0 ++ + gem: ++ needs: ruby-versions-gem + name: "Gem: ${{ matrix.ruby-version }} on ${{ matrix.runs-on }}" + runs-on: ${{ matrix.runs-on }} + strategy: +@@ -57,21 +64,26 @@ jobs: + - ubuntu-latest + - macos-latest + - windows-latest +- ruby-version: +- - "3.0" +- - head ++ ruby-version: ${{ fromJson(needs.ruby-versions-gem.outputs.versions) }} + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + - name: Install as gem +- env: +- BUNDLE_PATH__SYSTEM: "true" +- BUNDLE_WITHOUT: "benchmark:development" + run: | + rake install +- bundle install ++ - name: Install test dependencies on non-Windows ++ if: matrix.runs-on != 'windows-latest' ++ run: | ++ for gem in $(ruby -e 'puts ARGF.read[/^group :test do(.*)^end/m, 1].scan(/"(.+?)"/)' Gemfile); do ++ gem install ${gem} ++ done ++ - name: Install test dependencies on Windows ++ if: matrix.runs-on == 'windows-latest' ++ run: | ++ gem install test-unit ++ gem install test-unit-ruby-core + - name: Test + run: | + ruby -run -e mkdir -- tmp +diff --git a/lib/rexml/parsers/baseparser.rb b/lib/rexml/parsers/baseparser.rb +index d09237c..da051a7 100644 +--- a/lib/rexml/parsers/baseparser.rb ++++ b/lib/rexml/parsers/baseparser.rb +@@ -7,6 +7,17 @@ + + module REXML + module Parsers ++ if StringScanner::Version < "3.0.8" ++ module StringScannerCaptures ++ refine StringScanner do ++ def captures ++ values_at(*(1...size)) ++ end ++ end ++ end ++ using StringScannerCaptures ++ end ++ + # = Using the Pull Parser + # This API is experimental, and subject to change. + # parser = PullParser.new( "texttxet" ) +diff --git a/rexml.gemspec b/rexml.gemspec +index 97eac65..169e49d 100644 +--- a/rexml.gemspec ++++ b/rexml.gemspec +@@ -55,5 +55,5 @@ Gem::Specification.new do |spec| + + spec.required_ruby_version = '>= 2.5.0' + +- spec.add_runtime_dependency("strscan", ">= 3.0.9") ++ spec.add_runtime_dependency("strscan") + end diff --git a/SPECS/rubygem-rexml/rubygem-rexml.signatures.json b/SPECS/rubygem-rexml/rubygem-rexml.signatures.json index 1c9286685b8..22a4addc7dd 100644 --- a/SPECS/rubygem-rexml/rubygem-rexml.signatures.json +++ b/SPECS/rubygem-rexml/rubygem-rexml.signatures.json @@ -1,5 +1,5 @@ { - "Signatures": { - "rexml-3.2.6.tar.gz": "38239d0b3068549d4efd0d8f32ca650be76af83223caeacaa536b69f81011113" - } + "Signatures": { + "rexml-3.2.8.tar.gz": "44c1d11af52fba515380867980ff1e30b3a3d303cb90a7ca89659563969f3444" + } } diff --git a/SPECS/rubygem-rexml/rubygem-rexml.spec b/SPECS/rubygem-rexml/rubygem-rexml.spec index 7e38f7b92f7..b69def1d0ac 100644 --- a/SPECS/rubygem-rexml/rubygem-rexml.spec +++ b/SPECS/rubygem-rexml/rubygem-rexml.spec @@ -2,7 +2,7 @@ %global gem_name rexml Summary: REXML is an XML toolkit for Ruby Name: rubygem-%{gem_name} -Version: 3.2.6 +Version: 3.2.8 Release: 1%{?dist} License: BSD Vendor: Microsoft Corporation @@ -10,6 +10,7 @@ Distribution: Azure Linux Group: Development/Languages URL: https://github.com/ruby/rexml Source0: https://github.com/ruby/rexml/archive/refs/tags/v%{version}.tar.gz#/%{gem_name}-%{version}.tar.gz +Patch0: add-support-for-old-strscan.patch BuildRequires: git BuildRequires: ruby Requires: ruby(release) @@ -20,7 +21,7 @@ REXML was inspired by the Electric XML library for Java, which features an easy- REXML supports both tree and stream document parsing. Stream parsing is faster (about 1.5 times as fast). However, with stream parsing, you don't get access to features such as XPath. %prep -%setup -q -n %{gem_name}-%{version} +%autosetup -n %{gem_name}-%{version} -p1 %build gem build %{gem_name} @@ -30,10 +31,14 @@ gem install -V --local --force --install-dir %{buildroot}/%{gemdir} %{gem_name}- %files %defattr(-,root,root,-) -%doc %{gemdir}/gems/%{gem_name}-%{version}/LICENSE.txt +%license LICENSE.txt %{gemdir} %changelog +* Wed May 22 2024 Neha Agarwal - 3.2.8-1 +- Upgrade to v3.2.8 to fix CVE-2024-35176 +- Add patch to support old strscan + * Fri Nov 03 2023 CBL-Mariner Servicing Account - 3.2.6-1 - Auto-upgrade to 3.2.6 - Azure Linux 3.0 - package upgrades diff --git a/cgmanifest.json b/cgmanifest.json index 2e51df0abeb..924a20aede1 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -26604,8 +26604,8 @@ "type": "other", "other": { "name": "rubygem-rexml", - "version": "3.2.6", - "downloadUrl": "https://github.com/ruby/rexml/archive/refs/tags/v3.2.6.tar.gz" + "version": "3.2.8", + "downloadUrl": "https://github.com/ruby/rexml/archive/refs/tags/v3.2.8.tar.gz" } } }, From 5e47f8cde240cb1fa2f7f49906b967055438e204 Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Thu, 23 May 2024 13:46:59 -0700 Subject: [PATCH 50/51] Image Customizer: Verity on 3.0 images. (#9164) --- toolkit/tools/internal/grub/commands.go | 44 ++++++----- .../pkg/imagecustomizerlib/bootcustomizer.go | 29 ++++++++ .../imagecustomizerlib/bootcustomizer_test.go | 27 +++++++ .../pkg/imagecustomizerlib/customizeboot.go | 34 ++++----- .../pkg/imagecustomizerlib/customizeverity.go | 49 +++++++++--- .../imagecustomizerlib/defaultgrubutils.go | 74 ++++++++++++++++++- .../pkg/imagecustomizerlib/imagecustomizer.go | 12 +-- 7 files changed, 219 insertions(+), 50 deletions(-) diff --git a/toolkit/tools/internal/grub/commands.go b/toolkit/tools/internal/grub/commands.go index af03397ae3c..4be7c661f44 100644 --- a/toolkit/tools/internal/grub/commands.go +++ b/toolkit/tools/internal/grub/commands.go @@ -3,36 +3,46 @@ package grub -type Command struct { - Name string - Args []Token +type Line struct { + Tokens []Token + EndToken *Token } // Split the tokens into lines, using (unescaped) newlines and semicolons as separator tokens. // Note: Technically this is incorrect, since some constructs (e.g. "then") supporting having the subsequent command on // the same line. But to avoid needing to write a full parser, this code instead assumes that the grub config files are // at least somewhat sensibly formatted. -func SplitTokensIntoLines(tokens []Token) [][]Token { - lines := [][]Token(nil) - line := []Token(nil) +func SplitTokensIntoLines(tokens []Token) []Line { + lines := []Line(nil) + lineTokens := []Token(nil) + + for i := range tokens { + token := tokens[i] - for _, token := range tokens { switch token.Type { case NEWLINE, SEMICOLON: - if len(line) > 0 { - lines = append(lines, line) - } - line = nil + lines = appendNewLine(lines, lineTokens, &token) + lineTokens = nil default: - line = append(line, token) + lineTokens = append(lineTokens, token) } } - if len(line) > 0 { - lines = append(lines, line) + lines = appendNewLine(lines, lineTokens, nil) + return lines +} + +func appendNewLine(lines []Line, tokens []Token, endToken *Token) []Line { + if len(tokens) <= 0 { + return lines } + line := Line{ + Tokens: tokens, + EndToken: endToken, + } + lines = append(lines, line) return lines } @@ -45,11 +55,11 @@ func IsTokenKeyword(token Token, keyword string) bool { } // FindCommandAll looks for all the lines that contain a command with the provided name. -func FindCommandAll(lines [][]Token, command string) [][]Token { - commandLines := [][]Token(nil) +func FindCommandAll(lines []Line, command string) []Line { + commandLines := []Line(nil) for _, line := range lines { - if len(line) >= 1 && IsTokenKeyword(line[0], command) { + if len(line.Tokens) >= 1 && IsTokenKeyword(line.Tokens[0], command) { commandLines = append(commandLines, line) } } diff --git a/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go index 4341e2ea47d..3944c7b0cdc 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go +++ b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer.go @@ -44,6 +44,11 @@ func NewBootCustomizer(imageChroot *safechroot.Chroot) (*BootCustomizer, error) return b, nil } +// Returns whether or not the OS uses grub-mkconfig. +func (b *BootCustomizer) IsGrubMkconfigImage() bool { + return b.isGrubMkconfig +} + // Inserts new kernel command-line args into the grub config file. func (b *BootCustomizer) AddKernelCommandLine(extraCommandLine string) error { extraCommandLine = strings.TrimSpace(extraCommandLine) @@ -154,6 +159,30 @@ func (b *BootCustomizer) UpdateKernelCommandLineArgs(defaultGrubFileVarName defa return nil } +// Makes changes to the /etc/default/grub file that are needed/useful for enabling verity. +func (b *BootCustomizer) PrepareForVerity() error { + if b.isGrubMkconfig { + // Force root command-line arg to be referenced by /dev path instead of by UUID. + defaultGrubFileContent, err := updateDefaultGrubFileVariable(b.defaultGrubFileContent, "GRUB_DISABLE_UUID", + "true") + if err != nil { + return err + } + + // Disable recovery menu entry, to avoid having more than 1 linux command in the grub.cfg file. + // This will make it easier to modify the grub.cfg file to add the verity args. + defaultGrubFileContent, err = updateDefaultGrubFileVariable(defaultGrubFileContent, "GRUB_DISABLE_RECOVERY", + "true") + if err != nil { + return err + } + + b.defaultGrubFileContent = defaultGrubFileContent + } + + return nil +} + func (b *BootCustomizer) WriteToFile(imageChroot *safechroot.Chroot) error { if b.isGrubMkconfig { // Update /etc/defaukt/grub file. diff --git a/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer_test.go b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer_test.go index 0cdf9a2e51d..1dd2dd7fc54 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer_test.go +++ b/toolkit/tools/pkg/imagecustomizerlib/bootcustomizer_test.go @@ -146,6 +146,33 @@ func TestBootCustomizerSELinuxMode30(t *testing.T) { checkDiffs30(t, b, "", expectedDefaultGrubFileDiff) } +func TestBootCustomizerVerity20(t *testing.T) { + b := createBootCustomizerFor20(t) + + err := b.PrepareForVerity() + assert.NoError(t, err) + + checkDiffs20(t, b, "", "") +} + +func TestBootCustomizerVerity30(t *testing.T) { + b := createBootCustomizerFor30(t) + + err := b.PrepareForVerity() + assert.NoError(t, err) + + expectedDefaultGrubFileDiff := `6a7,8 +> GRUB_DISABLE_UUID="true" +> GRUB_DISABLE_RECOVERY="true" +` + checkDiffs30(t, b, "", expectedDefaultGrubFileDiff) + + // Do it again to make sure there aren't any changes. + err = b.PrepareForVerity() + assert.NoError(t, err) + checkDiffs30(t, b, "", expectedDefaultGrubFileDiff) +} + func checkDiffs20(t *testing.T, b *BootCustomizer, expectedGrubCfgDiff string, expectedDefaultGrubFileDiff string) { checkDiffs(t, b, filepath.Join(testDir, sampleGrubCfg20Path), filepath.Join(testDir, sampleDefaultGrub20Path), expectedGrubCfgDiff, expectedDefaultGrubFileDiff) diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go b/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go index 492731e21fc..8450de198e2 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeboot.go @@ -33,19 +33,19 @@ const ( // Looks for a command with the provided name and ensures there is only 1 such command. // Returns the line of the found command. -func findSingularGrubCommand(inputGrubCfgContent string, commandName string) ([]grub.Token, error) { +func findSingularGrubCommand(inputGrubCfgContent string, commandName string) (grub.Line, error) { grubTokens, err := grub.TokenizeConfig(inputGrubCfgContent) if err != nil { - return nil, err + return grub.Line{}, err } grubLines := grub.SplitTokensIntoLines(grubTokens) lines := grub.FindCommandAll(grubLines, commandName) if len(lines) < 1 { - return nil, fmt.Errorf("failed to find the '%s' command in grub config", commandName) + return grub.Line{}, fmt.Errorf("failed to find the '%s' command in grub config", commandName) } if len(lines) > 1 { - return nil, fmt.Errorf("more than one '%s' command in grub config", commandName) + return grub.Line{}, fmt.Errorf("more than one '%s' command in grub config", commandName) } line := lines[0] @@ -59,8 +59,8 @@ func replaceSearchCommand(inputGrubCfgContent string, searchCommand string) (out return "", err } - start := searchLine[0].Loc.Start.Index - end := searchLine[len(searchLine)-1].Loc.Start.Index + start := searchLine.Tokens[0].Loc.Start.Index + end := searchLine.Tokens[len(searchLine.Tokens)-1].Loc.Start.Index outputGrubCfgContent = inputGrubCfgContent[:start] + searchCommand + inputGrubCfgContent[end:] return outputGrubCfgContent, nil @@ -96,14 +96,14 @@ func replaceToken(inputGrubCfgContent string, oldToken string, newToken string) } // Find the linux command within the grub config file. -func findLinuxLine(inputGrubCfgContent string) ([]grub.Token, error) { +func findLinuxLine(inputGrubCfgContent string) (grub.Line, error) { linuxLine, err := findSingularGrubCommand(inputGrubCfgContent, "linux") if err != nil { - return nil, err + return grub.Line{}, err } - if len(linuxLine) < 2 { - return nil, fmt.Errorf("grub config 'linux' command is missing file path arg") + if len(linuxLine.Tokens) < 2 { + return grub.Line{}, fmt.Errorf("grub config 'linux' command is missing file path arg") } return linuxLine, nil @@ -118,7 +118,7 @@ func setLinuxPath(inputGrubCfgContent string, linuxPath string) (outputGrubCfgCo return "", "", err } - linuxFilePathToken := linuxLine[1] + linuxFilePathToken := linuxLine.Tokens[1] start := linuxFilePathToken.Loc.Start.Index end := linuxFilePathToken.Loc.End.Index @@ -137,11 +137,11 @@ func setInitrdPath(inputGrubCfgContent string, initrdPath string) (outputGrubCfg return "", "", err } - if len(line) < 2 { + if len(line.Tokens) < 2 { return "", "", fmt.Errorf("grub config 'initrd' command is missing file path arg") } - initrdFilePathToken := line[1] + initrdFilePathToken := line.Tokens[1] start := initrdFilePathToken.Loc.Start.Index end := initrdFilePathToken.Loc.End.Index @@ -187,7 +187,7 @@ func getLinuxCommandLineArgs(grub2Config string) ([]grubConfigLinuxArg, int, err } // Skip the "linux" command and the kernel binary path arg. - argTokens := linuxLine[2:] + argTokens := linuxLine.Tokens[2:] insertAt, err := findCommandLineInsertAt(argTokens) if err != nil { @@ -486,11 +486,11 @@ func replaceSetCommandValue(grub2Config string, varName string, newValue string) // Search for all the set commands that set the variable. setVarLines := [][]grub.Token(nil) for _, line := range setLines { - if len(line) < 2 { + if len(line.Tokens) < 2 { return "", fmt.Errorf("grub config has a set command that has zero args") } - argToken := line[1] + argToken := line.Tokens[1] argStringBuilder := strings.Builder{} // Get the name of the variable being set. @@ -516,7 +516,7 @@ func replaceSetCommandValue(grub2Config string, varName string, newValue string) // Check if the name matches. if name == varName { - setVarLines = append(setVarLines, line) + setVarLines = append(setVarLines, line.Tokens) } } diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeverity.go b/toolkit/tools/pkg/imagecustomizerlib/customizeverity.go index a07895e7690..5d09e83085a 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeverity.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeverity.go @@ -30,12 +30,17 @@ func enableVerityPartition(buildDir string, verity *imagecustomizerapi.Verity, i dmVerityDracutDriver := "dm-verity" err = addDracutModule(systemdVerityDracutModule, dmVerityDracutDriver, imageChroot) if err != nil { - return false, err + return false, fmt.Errorf("failed to add dracut modules for verity:\n%w", err) } err = updateFstabForVerity(buildDir, imageChroot) if err != nil { - return false, err + return false, fmt.Errorf("failed to update fstab file for verity:\n%w", err) + } + + err = prepareGrubConfigForVerity(imageChroot) + if err != nil { + return false, fmt.Errorf("failed to prepare grub config files for verity:\n%w", err) } return true, nil @@ -88,7 +93,26 @@ func updateFstabForVerity(buildDir string, imageChroot *safechroot.Chroot) error return nil } -func updateGrubConfig(dataPartitionIdType imagecustomizerapi.IdType, dataPartitionId string, +func prepareGrubConfigForVerity(imageChroot *safechroot.Chroot) error { + bootCustomizer, err := NewBootCustomizer(imageChroot) + if err != nil { + return err + } + + err = bootCustomizer.PrepareForVerity() + if err != nil { + return err + } + + err = bootCustomizer.WriteToFile(imageChroot) + if err != nil { + return err + } + + return nil +} + +func updateGrubConfigForVerity(dataPartitionIdType imagecustomizerapi.IdType, dataPartitionId string, hashPartitionIdType imagecustomizerapi.IdType, hashPartitionId string, corruptionOption imagecustomizerapi.CorruptionOption, rootHash string, grubCfgFullPath string, ) error { @@ -122,10 +146,10 @@ func updateGrubConfig(dataPartitionIdType imagecustomizerapi.IdType, dataPartiti return fmt.Errorf("failed to read grub config:\n%w", err) } + // Note: If grub-mkconfig is being used, then we can't add the verity command-line args to /etc/default/grub and + // call grub-mkconfig, since this would create a catch-22 with the verity root partition hash. + // So, instead we just modify the /boot/grub2/grub.cfg file directly. grubMkconfigEnabled := isGrubMkconfigConfig(grub2Config) - if grubMkconfigEnabled { - return fmt.Errorf("grub-mkconfig enabled images not yet supported for verity") - } grub2Config, err = updateKernelCommandLineArgs(grub2Config, []string{"rd.systemd.verity", "roothash", "systemd.verity_root_data", "systemd.verity_root_hash", "systemd.verity_root_options"}, newArgs) @@ -133,9 +157,16 @@ func updateGrubConfig(dataPartitionIdType imagecustomizerapi.IdType, dataPartiti return fmt.Errorf("failed to set verity kernel command line args:\n%w", err) } - grub2Config, err = replaceSetCommandValue(grub2Config, "rootdevice", "/dev/mapper/root") - if err != nil { - return fmt.Errorf("failed to set verity root device:\n%w", err) + if grubMkconfigEnabled { + grub2Config, err = updateKernelCommandLineArgs(grub2Config, []string{"root"}, []string{"root=/dev/mapper/root"}) + if err != nil { + return fmt.Errorf("failed to set verity root command-line arg:\n%w", err) + } + } else { + grub2Config, err = replaceSetCommandValue(grub2Config, "rootdevice", "/dev/mapper/root") + if err != nil { + return fmt.Errorf("failed to set verity root device:\n%w", err) + } } err = file.Write(grub2Config, grubCfgFullPath) diff --git a/toolkit/tools/pkg/imagecustomizerlib/defaultgrubutils.go b/toolkit/tools/pkg/imagecustomizerlib/defaultgrubutils.go index 0ef7cbf7859..d49d854fea0 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/defaultgrubutils.go +++ b/toolkit/tools/pkg/imagecustomizerlib/defaultgrubutils.go @@ -34,6 +34,7 @@ const ( type defaultGrubFileVarAssign struct { Token grub.Token + Line grub.Line Name string Value string } @@ -53,13 +54,13 @@ func findDefaultGrubFileVarAssigns(defaultGrubFileContent string) ([]defaultGrub lines := grub.SplitTokensIntoLines(grubTokens) for _, line := range lines { - if len(line) != 1 { + if len(line.Tokens) != 1 { // Normal variable assignments only have 1 value. // Export variable assignments have 2 values. But we are ignoring those. continue } - argToken := line[0] + argToken := line.Tokens[0] isVarAssign := len(argToken.SubWords) >= 1 && argToken.SubWords[0].Type == grub.KEYWORD_STRING && @@ -100,6 +101,7 @@ func findDefaultGrubFileVarAssigns(defaultGrubFileContent string) ([]defaultGrub varAssign := defaultGrubFileVarAssign{ Token: argToken, + Line: line, Name: name, Value: value, } @@ -248,6 +250,74 @@ func replaceDefaultGrubFileVarAssign(defaultGrubFileContent string, varAssign de return defaultGrubFileContent } +func insertDefaultGrubFileVarAssign(defaultGrubFileContent string, insertAfterLine *grub.Line, varName string, + newValue string, +) string { + // Figure out where to insert the new line. + insertAt := 0 + newlineBefore := false + if insertAfterLine != nil { + if insertAfterLine.EndToken != nil { + insertAt = insertAfterLine.EndToken.Loc.End.Index + newlineBefore = insertAfterLine.EndToken.Type == grub.SEMICOLON + } else { + // EOF follows the last variable assignment. + insertAt = insertAfterLine.Tokens[len(insertAfterLine.Tokens)-1].Loc.End.Index + newlineBefore = true + } + } + + // Create new variable assignment line string. + lineString := fmt.Sprintf("%s=%s", varName, grub.ForceQuoteString(newValue)) + + // Build new /etc/default/grub file contents. + builder := strings.Builder{} + builder.WriteString(defaultGrubFileContent[:insertAt]) + if newlineBefore { + builder.WriteString("\n") + } + builder.WriteString(lineString) + builder.WriteString("\n") + builder.WriteString(defaultGrubFileContent[insertAt:]) + + defaultGrubFileContent = builder.String() + return defaultGrubFileContent +} + +// Sets the value of a variable in the /etc/default/grub file, either replacing the existing variable value (if one +// exists) or adding a new one. +func updateDefaultGrubFileVariable(defaultGrubFileContent string, varName string, newValue string) (string, error) { + varAssigns, err := findDefaultGrubFileVarAssigns(defaultGrubFileContent) + if err != nil { + err = fmt.Errorf("failed to parse %s file:\n%w", installutils.GrubDefFile, err) + return "", err + } + + found := false + existingVarAssign := defaultGrubFileVarAssign{} + for _, varAssign := range varAssigns { + if varAssign.Name == varName { + existingVarAssign = varAssign + found = true + break + } + } + + if found { + defaultGrubFileContent = replaceDefaultGrubFileVarAssign(defaultGrubFileContent, existingVarAssign, newValue) + } else { + insertAfter := (*grub.Line)(nil) + if len(varAssigns) > 1 { + line := varAssigns[len(varAssigns)-1].Line + insertAfter = &line + } + + defaultGrubFileContent = insertDefaultGrubFileVarAssign(defaultGrubFileContent, insertAfter, varName, newValue) + } + + return defaultGrubFileContent, nil +} + // Checks if the image uses grub-mkconfig. func isGrubMkconfigEnabled(imageChroot *safechroot.Chroot) (bool, error) { grub2ConfigFile, err := readGrub2ConfigFile(imageChroot) diff --git a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go index 9848177fa98..fd0d2a6a7e5 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go @@ -357,7 +357,7 @@ func customizeOSContents(ic *ImageCustomizerParameters) error { if ic.config.OS.Verity != nil { // Customize image for dm-verity, setting up verity metadata and security features. - err = customizeVerityImageHelper(ic.buildDirAbs, ic.configPath, ic.config, ic.rawImageFile, ic.rpmsSources, ic.useBaseImageRpmRepos) + err = customizeVerityImageHelper(ic.buildDirAbs, ic.configPath, ic.config, ic.rawImageFile) if err != nil { return err } @@ -669,7 +669,7 @@ func shrinkFilesystemsHelper(buildImageFile string) error { } func customizeVerityImageHelper(buildDir string, baseConfigPath string, config *imagecustomizerapi.Config, - buildImageFile string, rpmsSources []string, useBaseImageRpmRepos bool, + buildImageFile string, ) error { var err error @@ -698,11 +698,13 @@ func customizeVerityImageHelper(buildDir string, baseConfigPath string, config * } // Extract the partition block device path. - dataPartition, err := idToPartitionBlockDevicePath(config.OS.Verity.DataPartition.IdType, config.OS.Verity.DataPartition.Id, nbdDevice, diskPartitions) + dataPartition, err := idToPartitionBlockDevicePath(config.OS.Verity.DataPartition.IdType, + config.OS.Verity.DataPartition.Id, nbdDevice, diskPartitions) if err != nil { return err } - hashPartition, err := idToPartitionBlockDevicePath(config.OS.Verity.HashPartition.IdType, config.OS.Verity.HashPartition.Id, nbdDevice, diskPartitions) + hashPartition, err := idToPartitionBlockDevicePath(config.OS.Verity.HashPartition.IdType, + config.OS.Verity.HashPartition.Id, nbdDevice, diskPartitions) if err != nil { return err } @@ -748,7 +750,7 @@ func customizeVerityImageHelper(buildDir string, baseConfigPath string, config * return fmt.Errorf("failed to stat file (%s):\n%w", grubCfgFullPath, err) } - err = updateGrubConfig(config.OS.Verity.DataPartition.IdType, config.OS.Verity.DataPartition.Id, + err = updateGrubConfigForVerity(config.OS.Verity.DataPartition.IdType, config.OS.Verity.DataPartition.Id, config.OS.Verity.HashPartition.IdType, config.OS.Verity.HashPartition.Id, config.OS.Verity.CorruptionOption, rootHash, grubCfgFullPath) if err != nil { From fa94a42c78f7d22e4be7424b8d711e19d93aa2f8 Mon Sep 17 00:00:00 2001 From: Sam Meluch Date: Fri, 24 May 2024 15:08:03 -0700 Subject: [PATCH 51/51] update azurelinux-release for preview release --- SPECS/azurelinux-release/azurelinux-release.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SPECS/azurelinux-release/azurelinux-release.spec b/SPECS/azurelinux-release/azurelinux-release.spec index 4db9cf75d76..763d3d992a7 100644 --- a/SPECS/azurelinux-release/azurelinux-release.spec +++ b/SPECS/azurelinux-release/azurelinux-release.spec @@ -5,7 +5,7 @@ Summary: Azure Linux release files Name: azurelinux-release Version: %{dist_version}.0 -Release: 12%{?dist} +Release: 13%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Azure Linux @@ -118,6 +118,9 @@ install -Dm0644 %{SOURCE4} -t %{buildroot}%{_sysctldir}/ %{_sysctldir}/*.conf %changelog +* Fri May 24 2024 Sam Meluch - 3.0-13 +- Azure Linux 3.0 May Preview Release 2 + * Thu May 09 2024 Sam Meluch - 3.0-12 - Azure Linux 3.0 May Preview Release 1