From 2e03bda0a180e2b76e30ab9da91548f631f041ff Mon Sep 17 00:00:00 2001 From: Dustin Nemes <39643802+some-things@users.noreply.github.com> Date: Mon, 19 Aug 2024 12:41:17 -0500 Subject: [PATCH 01/11] [release-2.9][backport] feat: allows enabling EUS repos with a flag --- bundles/redhat8.6/bundle.sh.gotmpl | 30 ++++++-- bundles/redhat8.6/packages.txt.gotmpl | 6 +- bundles/redhat8.8/bundle.sh.gotmpl | 27 ++++++-- bundles/redhat8.8/packages.txt.gotmpl | 4 ++ .../cmd/create-package-bundle.go | 68 +++++++++++++++---- 5 files changed, 107 insertions(+), 28 deletions(-) diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index 46d69afb9..2149bd87d 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -45,25 +45,41 @@ subscription::defer_unregister() { subscription-manager release --set=8.6 subscription-manager refresh subscription::defer_unregister -subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms -subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms -subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms + +ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" +EUS_REPOS=${EUS_REPOS:-""} +if [[ -n "${EUS_REPOS}" ]]; then + #disables the standard repositories which should not be enabled when using EUS + subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms + ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" +else + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms +fi + yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(< packages.txt) | grep -v *.i686 >> reqs.txt sed -i 1d reqs.txt # we need to get rid of the first line #shellcheck disable=SC2046 -yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt) +yumdownloader --archlist=x86_64,noarch \ + --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt) #shellcheck disable=SC2046 -yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch --resolve --disablerepo=* --enablerepo=kubernetes,rhel-8-for-x86_64-baseos-eus-rpms,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms $(< packages.txt) +yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch \ + --resolve --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ + $(< packages.txt) rm packages.txt reqs.txt curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm createrepo -v . -chown -R 1000:1000 repodata/ repo2module . --module-name offline createrepo_mod . #shellcheck disable=SC1083,SC2035 @@ -71,4 +87,4 @@ tar -czf {{ .OutputDirectory }}/{{ .KubernetesVersion }}_redhat_8.6_x86_64{{ .Fi #shellcheck disable=SC1083 chmod 777 {{ .OutputDirectory }}/{{ .KubernetesVersion }}_redhat_8.6_x86_64{{ .FipsSuffix }}.tar.gz popd -rm -rf "${TMP_DIR}" +rm -rf "${TMP_DIR}" \ No newline at end of file diff --git a/bundles/redhat8.6/packages.txt.gotmpl b/bundles/redhat8.6/packages.txt.gotmpl index a72bc6dc2..89197c37a 100644 --- a/bundles/redhat8.6/packages.txt.gotmpl +++ b/bundles/redhat8.6/packages.txt.gotmpl @@ -31,7 +31,9 @@ elfutils-libelf-devel libseccomp nfs-utils iproute-tc -kernel-headers-4.18.0-372.93.1.el8_6 -kernel-devel-4.18.0-372.93.1.el8_6 glibc-all-langpacks-2.28 glibc-devel-2.28 +{{ if .FetchKernelHeaders -}} +kernel-headers-4.18.0-372.93.1.el8_6 +kernel-devel-4.18.0-372.93.1.el8_6 +{{- end }} \ No newline at end of file diff --git a/bundles/redhat8.8/bundle.sh.gotmpl b/bundles/redhat8.8/bundle.sh.gotmpl index 914da33ba..64b556865 100755 --- a/bundles/redhat8.8/bundle.sh.gotmpl +++ b/bundles/redhat8.8/bundle.sh.gotmpl @@ -45,12 +45,26 @@ subscription::defer_unregister() { subscription-manager release --set=8.8 subscription-manager refresh subscription::defer_unregister -subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms -subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms -subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms + +ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" +EUS_REPOS=${EUS_REPOS:-""} +if [[ -n "${EUS_REPOS}" ]]; then + #disables the standard repositories which should not be enabled when using EUS + subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms + ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" +else + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms +fi + yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 @@ -59,11 +73,12 @@ sed -i 1d reqs.txt # we need to get rid of the first line #shellcheck disable=SC2046 yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt) #shellcheck disable=SC2046 -yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch --resolve --disablerepo=* --enablerepo=kubernetes,rhel-8-for-x86_64-baseos-eus-rpms,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms $(< packages.txt) +yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch \ + --resolve --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ + $(< packages.txt) rm packages.txt reqs.txt curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm createrepo -v . -chown -R 1000:1000 repodata/ repo2module . --module-name offline createrepo_mod . #shellcheck disable=SC1083,SC2035 @@ -71,4 +86,4 @@ tar -czf {{ .OutputDirectory }}/{{ .KubernetesVersion }}_redhat_8.8_x86_64{{ .Fi #shellcheck disable=SC1083 chmod 777 {{ .OutputDirectory }}/{{ .KubernetesVersion }}_redhat_8.8_x86_64{{ .FipsSuffix }}.tar.gz popd -rm -rf "${TMP_DIR}" +rm -rf "${TMP_DIR}" \ No newline at end of file diff --git a/bundles/redhat8.8/packages.txt.gotmpl b/bundles/redhat8.8/packages.txt.gotmpl index 09c77f57e..62edd8f8c 100644 --- a/bundles/redhat8.8/packages.txt.gotmpl +++ b/bundles/redhat8.8/packages.txt.gotmpl @@ -35,3 +35,7 @@ kernel-devel gssproxy libverto-module-base libverto +{{ if .FetchKernelHeaders -}} +kernel-headers-4.18.0-477.58.1.el8_8 +kernel-devel-4.18.0-477.58.1.el8_8 +{{- end }} \ No newline at end of file diff --git a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go index 072a1f117..fd9675673 100644 --- a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go +++ b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go @@ -91,18 +91,55 @@ func (r *Runner) CreatePackageBundle(args []string) error { osFlag string kubernetesVersionFlag string fipsFlag bool + eusReposFlag bool outputDirectoy string containerImage string + fetchKernelHeaders bool ) flagSet := flag.NewFlagSet(createPackageBundleCmd, flag.ExitOnError) - flagSet.StringVar(&osFlag, "os", "", - fmt.Sprintf("The target OS you wish to create a package bundle for. Must be one of %v", getKeys(osToConfig))) - flagSet.StringVar(&kubernetesVersionFlag, "kubernetes-version", "", - "The version of kubernetes to download packages for.") - flagSet.BoolVar(&fipsFlag, "fips", false, "If the package bundle should include fips packages.") - flagSet.StringVar(&outputDirectoy, "output-directory", "artifacts", - "The directory to place the bundle in.") - flagSet.StringVar(&containerImage, "container-image", "", "A container image to use for building the package bundles") + flagSet.StringVar( + &osFlag, + "os", + "", + fmt.Sprintf("The target OS you wish to create a package bundle for. Must be one of %v", getKeys(osToConfig)), + ) + flagSet.StringVar( + &kubernetesVersionFlag, + "kubernetes-version", + "", + "The version of kubernetes to download packages for.", + ) + flagSet.BoolVar( + &fipsFlag, + "fips", + false, + "If the package bundle should include fips packages.", + ) + flagSet.BoolVar( + &eusReposFlag, + "enable-eus-repos", + false, + "If enabled fetches packages from EUS repositories when creating RHEL package bundles. Disabled by default.", + ) + flagSet.StringVar( + &outputDirectoy, + "output-directory", + "artifacts", + "The directory to place the bundle in.", + ) + flagSet.StringVar( + &containerImage, + "container-image", + "", + "A container image to use for building the package bundles", + ) + flagSet.BoolVar( + &fetchKernelHeaders, + "fetch-kernel-headers", + false, + //nolint:lll // its ok to have long help texts + "If enabled fetches kernel headers for the target operating system. To modify the version, edit the file at bundles/{OS_NAME}{VERSION}/packages.txt.gotmpl directly eg: bundles/redhat8.8/packages.txt.gotmpl. This is required for operating systems that will use NVIDIA GPU drivers.", + ) err := flagSet.Parse(args) if err != nil { return err @@ -124,13 +161,16 @@ func (r *Runner) CreatePackageBundle(args []string) error { return err } } + if eusReposFlag { + r.env["EUS_REPOS"] = "true" + } bundleCmd := "./bundle.sh" absPathToOutput := outputDirectoy if !path.IsAbs(outputDirectoy) { dir := r.workingDir absPathToOutput = path.Join(dir, outputDirectoy) } - reposList, err := templateObjects(osFlag, kubernetesVersion, absPathToOutput, fipsFlag) + reposList, err := templateObjects(osFlag, kubernetesVersion, absPathToOutput, fipsFlag, fetchKernelHeaders) if err != nil { return err } @@ -145,7 +185,7 @@ func (r *Runner) CreatePackageBundle(args []string) error { } //nolint:gocyclo,funlen // the function is relatively clear -func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool) ([]string, error) { +func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool, fetchKernelHeaders bool) ([]string, error) { config, found := osToConfig[targetOS] if !found { return nil, fmt.Errorf("buildOS %s is invalid must be one of %v", targetOS, getKeys(osToConfig)) @@ -159,7 +199,7 @@ func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool) ( configDirFS := os.DirFS(base) l := make([]string, 0) generated := path.Join(base, generatedDirName) - if err = os.MkdirAll(generated, 0o755); err != nil { + if err = os.MkdirAll(generated, 0o777); err != nil { return l, err } @@ -240,9 +280,11 @@ func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool) ( return fmt.Errorf("failed to create file: %w", err) } templateInput := struct { - KubernetesVersion string + KubernetesVersion string + FetchKernelHeaders bool }{ - KubernetesVersion: kubernetesVersion, + KubernetesVersion: kubernetesVersion, + FetchKernelHeaders: fetchKernelHeaders, } err = t.Execute(out, templateInput) if err != nil { From 0fde978b3dd1a61ae5bfa508a3c449b55b8f0eb7 Mon Sep 17 00:00:00 2001 From: Dustin Nemes <39643802+some-things@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:15:16 -0500 Subject: [PATCH 02/11] feat: add support for redhat satellite servers --- bundles/redhat8.6/bundle.sh.gotmpl | 5 +++++ bundles/redhat8.8/bundle.sh.gotmpl | 5 +++++ cmd/konvoy-image-wrapper/cmd/create-package-bundle.go | 11 +++++++++++ 3 files changed, 21 insertions(+) diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index 2149bd87d..e5dc90458 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -13,6 +13,11 @@ RHSM_ACTIVATION_KEY=${RHSM_ACTIVATION_KEY:-""} RHSM_USER=${RHSM_USER:-""} RHSM_PASS=${RHSM_PASS:-""} +SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""} +if [[ -n "${SATELLITE_SERVER_URL}" ]]; then + rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" +fi + if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force FOUND=true diff --git a/bundles/redhat8.8/bundle.sh.gotmpl b/bundles/redhat8.8/bundle.sh.gotmpl index 64b556865..9ef2440a9 100755 --- a/bundles/redhat8.8/bundle.sh.gotmpl +++ b/bundles/redhat8.8/bundle.sh.gotmpl @@ -13,6 +13,11 @@ RHSM_ACTIVATION_KEY=${RHSM_ACTIVATION_KEY:-""} RHSM_USER=${RHSM_USER:-""} RHSM_PASS=${RHSM_PASS:-""} +SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""} +if [[ -n "${SATELLITE_SERVER_URL}" ]]; then + rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" +fi + if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force FOUND=true diff --git a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go index fd9675673..6c35d8610 100644 --- a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go +++ b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go @@ -92,6 +92,7 @@ func (r *Runner) CreatePackageBundle(args []string) error { kubernetesVersionFlag string fipsFlag bool eusReposFlag bool + satelliteFlag string outputDirectoy string containerImage string fetchKernelHeaders bool @@ -121,6 +122,13 @@ func (r *Runner) CreatePackageBundle(args []string) error { false, "If enabled fetches packages from EUS repositories when creating RHEL package bundles. Disabled by default.", ) + flagSet.StringVar( + &satelliteFlag, + "satellite-server-url", + "", + //nolint:lll // it is ok to have long help texts + "If set, registers with and fetches packages from a Red Hat Satellite. All required repositories must be available in the Red Hat Satellite. Example: --satellite-server-url=\"https://satellite.nutanix.sh\"", + ) flagSet.StringVar( &outputDirectoy, "output-directory", @@ -164,6 +172,9 @@ func (r *Runner) CreatePackageBundle(args []string) error { if eusReposFlag { r.env["EUS_REPOS"] = "true" } + if satelliteFlag != "" { + r.env["SATELLITE_SERVER_URL"] = satelliteFlag + } bundleCmd := "./bundle.sh" absPathToOutput := outputDirectoy if !path.IsAbs(outputDirectoy) { From a31ed33b7c483206fb0e714e3f74f7581d5ebe48 Mon Sep 17 00:00:00 2001 From: Dustin Nemes <39643802+some-things@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:47:27 -0500 Subject: [PATCH 03/11] feat: allow bypassing subscription-manager --- bundles/redhat8.6/bundle.sh.gotmpl | 88 ++++++++++--------- bundles/redhat8.8/bundle.sh.gotmpl | 88 ++++++++++--------- .../cmd/create-package-bundle.go | 26 ++++-- 3 files changed, 112 insertions(+), 90 deletions(-) diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index e5dc90458..db6e403c7 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -3,7 +3,7 @@ set -euo pipefail # When yum operates on multiple packages, it does not, by default, return an error if a subset # of packages is not found. This makes yum return an error. -echo skip_missing_names_on_install=False >> /etc/yum.conf +echo skip_missing_names_on_install=False >>/etc/yum.conf sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py FOUND=false @@ -13,31 +13,36 @@ RHSM_ACTIVATION_KEY=${RHSM_ACTIVATION_KEY:-""} RHSM_USER=${RHSM_USER:-""} RHSM_PASS=${RHSM_PASS:-""} +SKIP_SUBSCRIPTION_MANAGER=${SKIP_SUBSCRIPTION_MANAGER:-""} SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""} -if [[ -n "${SATELLITE_SERVER_URL}" ]]; then - rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" -fi -if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then - subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force - FOUND=true -fi +if [[ -n "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then + echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true." +else + if [[ -n "${SATELLITE_SERVER_URL}" ]]; then + rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" + fi -if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then - subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force - FOUND=true -fi + if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then + subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force + FOUND=true + fi -if [[ ${FOUND} == false ]]; then - echo "You must use subscription manager to fetch packages for redhat" - exit 1 -fi + if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then + subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force + FOUND=true + fi + + if [[ ${FOUND} == false ]]; then + echo "You must use subscription manager to fetch packages for redhat" + exit 1 + fi -subscription::unregister() { - subscription-manager unregister -} + subscription::unregister() { + subscription-manager unregister + } -subscription::defer_unregister() { + subscription::defer_unregister() { trap subscription::unregister ABRT trap subscription::unregister EXIT trap subscription::unregister HUP @@ -45,25 +50,26 @@ subscription::defer_unregister() { trap subscription::unregister TERM trap subscription::unregister USR1 trap subscription::unregister USR2 -} + } -subscription-manager release --set=8.6 -subscription-manager refresh -subscription::defer_unregister + subscription-manager release --set=8.6 + subscription-manager refresh + subscription::defer_unregister -ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" -EUS_REPOS=${EUS_REPOS:-""} -if [[ -n "${EUS_REPOS}" ]]; then - #disables the standard repositories which should not be enabled when using EUS - subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms - subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms - subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms - subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms - ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" -else - subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms - subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms - subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms + ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" + EUS_REPOS=${EUS_REPOS:-""} + if [[ -n "${EUS_REPOS}" ]]; then + #disables the standard repositories which should not be enabled when using EUS + subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms + ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" + else + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms + fi fi yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools @@ -73,15 +79,15 @@ chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 -repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(< packages.txt) | grep -v *.i686 >> reqs.txt +repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(>reqs.txt sed -i 1d reqs.txt # we need to get rid of the first line #shellcheck disable=SC2046 yumdownloader --archlist=x86_64,noarch \ - --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt) + --setopt=skip_missing_names_on_install=False -x \*i686 $(> /etc/yum.conf +echo skip_missing_names_on_install=False >>/etc/yum.conf sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py FOUND=false @@ -13,31 +13,36 @@ RHSM_ACTIVATION_KEY=${RHSM_ACTIVATION_KEY:-""} RHSM_USER=${RHSM_USER:-""} RHSM_PASS=${RHSM_PASS:-""} +SKIP_SUBSCRIPTION_MANAGER=${SKIP_SUBSCRIPTION_MANAGER:-""} SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""} -if [[ -n "${SATELLITE_SERVER_URL}" ]]; then - rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" -fi -if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then - subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force - FOUND=true -fi +if [[ -n "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then + echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true." +else + if [[ -n "${SATELLITE_SERVER_URL}" ]]; then + rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" + fi -if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then - subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force - FOUND=true -fi + if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then + subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force + FOUND=true + fi -if [[ ${FOUND} == false ]]; then - echo "You must use subscription manager to fetch packages for redhat" - exit 1 -fi + if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then + subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force + FOUND=true + fi + + if [[ ${FOUND} == false ]]; then + echo "You must use subscription manager to fetch packages for redhat" + exit 1 + fi -subscription::unregister() { - subscription-manager unregister -} + subscription::unregister() { + subscription-manager unregister + } -subscription::defer_unregister() { + subscription::defer_unregister() { trap subscription::unregister ABRT trap subscription::unregister EXIT trap subscription::unregister HUP @@ -45,25 +50,26 @@ subscription::defer_unregister() { trap subscription::unregister TERM trap subscription::unregister USR1 trap subscription::unregister USR2 -} + } -subscription-manager release --set=8.8 -subscription-manager refresh -subscription::defer_unregister + subscription-manager release --set=8.8 + subscription-manager refresh + subscription::defer_unregister -ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" -EUS_REPOS=${EUS_REPOS:-""} -if [[ -n "${EUS_REPOS}" ]]; then - #disables the standard repositories which should not be enabled when using EUS - subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms - subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms - subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms - subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms - ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" -else - subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms - subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms - subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms + ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" + EUS_REPOS=${EUS_REPOS:-""} + if [[ -n "${EUS_REPOS}" ]]; then + #disables the standard repositories which should not be enabled when using EUS + subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms + ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" + else + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms + fi fi yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools @@ -73,14 +79,14 @@ chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 -repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(< packages.txt) | grep -v *.i686 >> reqs.txt +repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(>reqs.txt sed -i 1d reqs.txt # we need to get rid of the first line #shellcheck disable=SC2046 -yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt) +yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $( Date: Mon, 19 Aug 2024 14:46:38 -0500 Subject: [PATCH 04/11] feat: automatically append user-defined repos to ENABLED_REPOS --- bundles/redhat8.6/bundle.sh.gotmpl | 7 ++++++ .../repo-templates/user-repos.repo.gotmpl | 23 +++++++++++++++++++ bundles/redhat8.8/bundle.sh.gotmpl | 7 ++++++ .../repo-templates/user-repos.repo.gotmpl | 23 +++++++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 bundles/redhat8.6/repo-templates/user-repos.repo.gotmpl create mode 100644 bundles/redhat8.8/repo-templates/user-repos.repo.gotmpl diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index db6e403c7..e6d5b3c8f 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -72,6 +72,13 @@ else fi fi +USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)" +if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},${USER_REPOS}" +elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then + ENABLED_REPOS="${USER_REPOS}" +fi + yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" diff --git a/bundles/redhat8.6/repo-templates/user-repos.repo.gotmpl b/bundles/redhat8.6/repo-templates/user-repos.repo.gotmpl new file mode 100644 index 000000000..00ebade1a --- /dev/null +++ b/bundles/redhat8.6/repo-templates/user-repos.repo.gotmpl @@ -0,0 +1,23 @@ +[mirror-0] +name=mirror-0 +baseurl=https://34.219.152.253/repo/kubernetes/kubernetes +gpgcheck=0 +sslverify=0 + +[mirror-1] +name=mirror-1 +baseurl=https://34.219.152.253/repo/baseos/rhel-8-for-x86_64-baseos-rpms +gpgcheck=0 +sslverify=0 + +[mirror-2] +name=mirror-2 +baseurl=https://34.219.152.253/repo/appstream/rhel-8-for-x86_64-appstream-rpms +gpgcheck=0 +sslverify=0 + +[mirror-3] +name=mirror-3 +baseurl=https://34.219.152.253/repo/codeready-builder/codeready-builder-for-rhel-8-x86_64-rpms +gpgcheck=0 +sslverify=0 \ No newline at end of file diff --git a/bundles/redhat8.8/bundle.sh.gotmpl b/bundles/redhat8.8/bundle.sh.gotmpl index cd86778ec..d2989b135 100755 --- a/bundles/redhat8.8/bundle.sh.gotmpl +++ b/bundles/redhat8.8/bundle.sh.gotmpl @@ -72,6 +72,13 @@ else fi fi +USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)" +if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},${USER_REPOS}" +elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then + ENABLED_REPOS="${USER_REPOS}" +fi + yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" diff --git a/bundles/redhat8.8/repo-templates/user-repos.repo.gotmpl b/bundles/redhat8.8/repo-templates/user-repos.repo.gotmpl new file mode 100644 index 000000000..00ebade1a --- /dev/null +++ b/bundles/redhat8.8/repo-templates/user-repos.repo.gotmpl @@ -0,0 +1,23 @@ +[mirror-0] +name=mirror-0 +baseurl=https://34.219.152.253/repo/kubernetes/kubernetes +gpgcheck=0 +sslverify=0 + +[mirror-1] +name=mirror-1 +baseurl=https://34.219.152.253/repo/baseos/rhel-8-for-x86_64-baseos-rpms +gpgcheck=0 +sslverify=0 + +[mirror-2] +name=mirror-2 +baseurl=https://34.219.152.253/repo/appstream/rhel-8-for-x86_64-appstream-rpms +gpgcheck=0 +sslverify=0 + +[mirror-3] +name=mirror-3 +baseurl=https://34.219.152.253/repo/codeready-builder/codeready-builder-for-rhel-8-x86_64-rpms +gpgcheck=0 +sslverify=0 \ No newline at end of file From 2bf665ee514ffa974159f3cc140cce777c770139 Mon Sep 17 00:00:00 2001 From: Dustin Nemes <39643802+some-things@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:26:32 -0500 Subject: [PATCH 05/11] feat: add proxy support --- cmd/konvoy-image-wrapper/cmd/create-package-bundle.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go index ac6d4a494..628f43486 100644 --- a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go +++ b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go @@ -185,6 +185,7 @@ func (r *Runner) CreatePackageBundle(args []string) error { if subscriptionManagerFlag { r.env["SKIP_SUBSCRIPTION_MANAGER"] = "true" } + r.setHTTPProxyEnv() bundleCmd := "./bundle.sh" absPathToOutput := outputDirectoy if !path.IsAbs(outputDirectoy) { From aacf5dee3572e0056c90e885ff98ad2309ab6416 Mon Sep 17 00:00:00 2001 From: Dustin Nemes <39643802+some-things@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:08:53 -0500 Subject: [PATCH 06/11] fix: properly disable and enable defined repos --- bundles/redhat8.6/bundle.sh.gotmpl | 36 +++++++++++++++++------------- bundles/redhat8.8/bundle.sh.gotmpl | 35 ++++++++++++++++------------- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index e6d5b3c8f..a420089c9 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -15,10 +15,9 @@ RHSM_PASS=${RHSM_PASS:-""} SKIP_SUBSCRIPTION_MANAGER=${SKIP_SUBSCRIPTION_MANAGER:-""} SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""} +ENABLED_REPOS=${ENABLED_REPOS:-""} -if [[ -n "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then - echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true." -else +if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then if [[ -n "${SATELLITE_SERVER_URL}" ]]; then rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" fi @@ -70,31 +69,36 @@ else subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms fi +else + echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true." fi -USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)" -if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then - ENABLED_REPOS="${ENABLED_REPOS},${USER_REPOS}" -elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then - ENABLED_REPOS="${USER_REPOS}" +if [[ -f /etc/yum.repos.d/user-repos.repo ]]; then + USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)" + if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},${USER_REPOS}" + elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then + ENABLED_REPOS="${USER_REPOS}" + fi fi -yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools +yum -y install --disablerepo=* --enablerepo="${ENABLED_REPOS}" gettext yum-utils createrepo dnf-utils modulemd-tools yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 -repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(>reqs.txt -sed -i 1d reqs.txt # we need to get rid of the first line +repoquery --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ + --archlist=x86_64,noarch --resolve --requires --recursive $(>reqs.txt #shellcheck disable=SC2046 -yumdownloader --archlist=x86_64,noarch \ - --setopt=skip_missing_names_on_install=False -x \*i686 $(>reqs.txt -sed -i 1d reqs.txt # we need to get rid of the first line +repoquery --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ + --archlist=x86_64,noarch --resolve --requires --recursive $(>reqs.txt #shellcheck disable=SC2046 -yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $( Date: Tue, 20 Aug 2024 17:43:17 -0500 Subject: [PATCH 07/11] feat: install ssm-agent from docker image --- Dockerfile | 2 ++ Dockerfile.devkit | 1 + ansible/roles/providers/tasks/aws.yml | 51 ++++++++++++--------------- bundles/redhat8.4/bundle.sh.gotmpl | 1 - bundles/redhat8.6/bundle.sh.gotmpl | 1 - bundles/redhat8.8/bundle.sh.gotmpl | 1 - 6 files changed, 26 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 087447e32..653b57080 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,8 @@ ARG BUILDARCH # Packer copies /usr/local/bin/goss-amd64 from this container to the remote host COPY --from=devkit /usr/local/bin/goss-amd64 /usr/local/bin/goss-amd64 +COPY --from=devkit /opt/amazon-ssm-agent.rpm /opt/amazon-ssm-agent.rpm + # we copy this to remote hosts to execute mindthegap so its always amd64 COPY --from=devkit /usr/local/bin/mindthegap /usr/local/bin/ COPY --from=devkit /usr/local/bin/packer-${BUILDARCH} /usr/local/bin/packer diff --git a/Dockerfile.devkit b/Dockerfile.devkit index cd856acb0..002b7dbc6 100644 --- a/Dockerfile.devkit +++ b/Dockerfile.devkit @@ -78,6 +78,7 @@ RUN curl -L "https://github.com/goss-org/goss/releases/download/${GOSS_VERSION}/ RUN chmod +rx /usr/local/bin/goss-amd64 ARG BUILDARCH RUN ln -s /usr/local/bin/goss-${BUILDARCH} /usr/local/bin/goss +RUN curl -o /opt/amazon-ssm-agent.rpm https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm COPY --from=packer-amd64 /bin/packer /usr/local/bin/packer-amd64 COPY --from=packer-arm64 /bin/packer /usr/local/bin/packer-arm64 diff --git a/ansible/roles/providers/tasks/aws.yml b/ansible/roles/providers/tasks/aws.yml index fb536a227..e20fa9dc3 100644 --- a/ansible/roles/providers/tasks/aws.yml +++ b/ansible/roles/providers/tasks/aws.yml @@ -42,38 +42,34 @@ - ansible_distribution != "Amazon" - ansible_os_family != "Suse" -- name: install aws agents RPM - package: - name: "{{ item }}" - state: present - # must be fixed by amazon https://github.com/aws/amazon-ssm-agent/issues/235 - disable_gpg_check: yes - enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" - disablerepo: "{{ '*' if offline_mode_enabled else '' }}" - with_items: - - "{{ 'amazon-ssm-agent' if offline_mode_enabled else 'https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm' }}" - when: - - ansible_os_family == "RedHat" - - ansible_distribution != "Amazon" -- name: install aws agents RPM - package: - name: "{{ item }}" - state: present - # must be fixed by amazon https://github.com/aws/amazon-ssm-agent/issues/235 - disable_gpg_check: yes - with_items: - - https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm - when: - - ansible_os_family == "Suse" - - ansible_distribution != "Amazon" +- block: + - name: copy ssm rpm + copy: + src: /opt/amazon-ssm-agent.rpm + dest: /opt/amazon-ssm-agent.rpm -- name: install aws agents RPM + - name: install aws agents RPM + ansible.builtin.yum: + name: /opt/amazon-ssm-agent.rpm + state: present + disable_gpg_check: yes + enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" + disablerepo: "{{ '*' if offline_mode_enabled else '' }}" + when: ansible_os_family == "RedHat" and ansible_distribution != "Amazon" + + - name: install aws agents RPM + ansible.builtin.zypper: + name: /opt/amazon-ssm-agent.rpm + state: present + disable_gpg_check: yes + when: ansible_os_family == "Suse" and ansible_distribution != "Amazon" + +- name: install aws cli for amazon linux package: name: "{{ item }}" state: present with_items: - - amazon-ssm-agent - awscli when: ansible_distribution == "Amazon" @@ -93,5 +89,4 @@ name: snap.amazon-ssm-agent.amazon-ssm-agent.service state: started enabled: yes - when: ansible_distribution == "Ubuntu" - + when: ansible_distribution == "Ubuntu" \ No newline at end of file diff --git a/bundles/redhat8.4/bundle.sh.gotmpl b/bundles/redhat8.4/bundle.sh.gotmpl index 56c9fec29..a97baca52 100755 --- a/bundles/redhat8.4/bundle.sh.gotmpl +++ b/bundles/redhat8.4/bundle.sh.gotmpl @@ -61,7 +61,6 @@ yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=Fa #shellcheck disable=SC2046 yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch --resolve --disablerepo=* --enablerepo=kubernetes,rhel-8-for-x86_64-baseos-eus-rpms,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms --disablerepo=appstream-centos $(< packages.txt) rm packages.txt reqs.txt -curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm createrepo -v . chown -R 1000:1000 repodata/ yum install -y modulemd-tools diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index a420089c9..11411f079 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -100,7 +100,6 @@ yumdownloader --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch \ --resolve $( Date: Tue, 20 Aug 2024 17:48:58 -0500 Subject: [PATCH 08/11] fix: remove user repos placeholder file --- .../repo-templates/user-repos.repo.gotmpl | 23 ------------------- .../repo-templates/user-repos.repo.gotmpl | 23 ------------------- 2 files changed, 46 deletions(-) delete mode 100644 bundles/redhat8.6/repo-templates/user-repos.repo.gotmpl delete mode 100644 bundles/redhat8.8/repo-templates/user-repos.repo.gotmpl diff --git a/bundles/redhat8.6/repo-templates/user-repos.repo.gotmpl b/bundles/redhat8.6/repo-templates/user-repos.repo.gotmpl deleted file mode 100644 index 00ebade1a..000000000 --- a/bundles/redhat8.6/repo-templates/user-repos.repo.gotmpl +++ /dev/null @@ -1,23 +0,0 @@ -[mirror-0] -name=mirror-0 -baseurl=https://34.219.152.253/repo/kubernetes/kubernetes -gpgcheck=0 -sslverify=0 - -[mirror-1] -name=mirror-1 -baseurl=https://34.219.152.253/repo/baseos/rhel-8-for-x86_64-baseos-rpms -gpgcheck=0 -sslverify=0 - -[mirror-2] -name=mirror-2 -baseurl=https://34.219.152.253/repo/appstream/rhel-8-for-x86_64-appstream-rpms -gpgcheck=0 -sslverify=0 - -[mirror-3] -name=mirror-3 -baseurl=https://34.219.152.253/repo/codeready-builder/codeready-builder-for-rhel-8-x86_64-rpms -gpgcheck=0 -sslverify=0 \ No newline at end of file diff --git a/bundles/redhat8.8/repo-templates/user-repos.repo.gotmpl b/bundles/redhat8.8/repo-templates/user-repos.repo.gotmpl deleted file mode 100644 index 00ebade1a..000000000 --- a/bundles/redhat8.8/repo-templates/user-repos.repo.gotmpl +++ /dev/null @@ -1,23 +0,0 @@ -[mirror-0] -name=mirror-0 -baseurl=https://34.219.152.253/repo/kubernetes/kubernetes -gpgcheck=0 -sslverify=0 - -[mirror-1] -name=mirror-1 -baseurl=https://34.219.152.253/repo/baseos/rhel-8-for-x86_64-baseos-rpms -gpgcheck=0 -sslverify=0 - -[mirror-2] -name=mirror-2 -baseurl=https://34.219.152.253/repo/appstream/rhel-8-for-x86_64-appstream-rpms -gpgcheck=0 -sslverify=0 - -[mirror-3] -name=mirror-3 -baseurl=https://34.219.152.253/repo/codeready-builder/codeready-builder-for-rhel-8-x86_64-rpms -gpgcheck=0 -sslverify=0 \ No newline at end of file From 5765f6437977c660f07a2f2568ecd2a7c0295e78 Mon Sep 17 00:00:00 2001 From: Dustin Nemes <39643802+some-things@users.noreply.github.com> Date: Tue, 20 Aug 2024 18:40:09 -0500 Subject: [PATCH 09/11] fix: fix linting issues --- cmd/konvoy-image-wrapper/cmd/create-package-bundle.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go index 628f43486..c28a03325 100644 --- a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go +++ b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go @@ -134,6 +134,7 @@ func (r *Runner) CreatePackageBundle(args []string) error { &subscriptionManagerFlag, "skip-subscription-manager", false, + //nolint:lll // it is ok to have long help texts "If enabled, skips authenticating with subscription-manager and fetching from the pre-configured official RHEL repositories when creating RHEL package bundles. Disabled by default.", ) flagSet.StringVar( @@ -177,6 +178,7 @@ func (r *Runner) CreatePackageBundle(args []string) error { } } if eusReposFlag { + //nolint:goconst // it is ok to not use const here r.env["EUS_REPOS"] = "true" } if satelliteFlag != "" { From c37a61fd66cae267f019e854e0a15444c9523fdb Mon Sep 17 00:00:00 2001 From: Dustin Nemes <39643802+some-things@users.noreply.github.com> Date: Tue, 20 Aug 2024 23:22:41 -0500 Subject: [PATCH 10/11] [release-2.9][backport] feat: embed k8s rpms in container --- Dockerfile | 4 +- Dockerfile.devkit | 27 ++++++++++ ansible/roles/kubeadm/tasks/redhat.yaml | 38 +++++++++++++- ansible/roles/packages/tasks/redhat.yaml | 52 ++++++++++++++++--- bundles/centos7.9/bundle.sh.gotmpl | 1 + bundles/redhat7.9/bundle.sh.gotmpl | 1 + bundles/redhat8.4/bundle.sh.gotmpl | 1 + bundles/redhat8.6/bundle.sh.gotmpl | 9 +++- bundles/redhat8.6/packages.txt.gotmpl | 4 +- bundles/redhat8.8/bundle.sh.gotmpl | 9 +++- bundles/redhat8.8/packages.txt.gotmpl | 4 +- bundles/rocky9.1/bundle.sh.gotmpl | 2 +- bundles/rocky9.1/packages.txt.gotmpl | 4 +- bundles/ubuntu20.04/bundle.sh.gotmpl | 1 + .../cmd/create-package-bundle.go | 23 +++++--- 15 files changed, 155 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 653b57080..59eaf9927 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,8 +31,8 @@ ARG BUILDARCH # we copy this to remote hosts to execute GOSS # Packer copies /usr/local/bin/goss-amd64 from this container to the remote host COPY --from=devkit /usr/local/bin/goss-amd64 /usr/local/bin/goss-amd64 - -COPY --from=devkit /opt/amazon-ssm-agent.rpm /opt/amazon-ssm-agent.rpm +COPY --from=devkit /opt/*.rpm /opt +COPY --from=devkit /opt/d2iq-sign-authority-gpg-public-key /opt/d2iq-sign-authority-gpg-public-key # we copy this to remote hosts to execute mindthegap so its always amd64 COPY --from=devkit /usr/local/bin/mindthegap /usr/local/bin/ diff --git a/Dockerfile.devkit b/Dockerfile.devkit index 002b7dbc6..778ec1634 100644 --- a/Dockerfile.devkit +++ b/Dockerfile.devkit @@ -79,6 +79,33 @@ RUN chmod +rx /usr/local/bin/goss-amd64 ARG BUILDARCH RUN ln -s /usr/local/bin/goss-${BUILDARCH} /usr/local/bin/goss RUN curl -o /opt/amazon-ssm-agent.rpm https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm +COPY ansible ansible +# Fetch nokmem rpms +RUN \ + export KUBERNETES_VERSION=$(awk -F': ' '/kubernetes_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '2p' | xargs) && \ + echo ${KUBERNETES_VERSION} && \ + curl -o /opt/kubectl-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubectl-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + curl -o /opt/kubeadm-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubeadm-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + curl -o /opt/kubelet-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubelet-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + export CRICTL_TOOLS_VERSION="$(echo ${KUBERNETES_VERSION} | cut -d. -f1-2).0" && \ + curl -o /opt/cri-tools-${CRICTL_TOOLS_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/cri-tools-${CRICTL_TOOLS_VERSION}-0.x86_64.rpm && \ + export CNI_VERSION=$(awk -F': ' '/kubernetes_cni_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '1p' | xargs) && \ + curl -o /opt/kubernetes-cni-${CNI_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubernetes-cni-${CNI_VERSION}-0.x86_64.rpm + + +# Fetch fips rpms +RUN \ + export KUBERNETES_VERSION=$(awk -F': ' '/kubernetes_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '2p' | xargs) && \ + echo ${KUBERNETES_VERSION} && \ + curl -o /opt/kubectl-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubectl-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + curl -o /opt/kubeadm-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubeadm-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + curl -o /opt/kubelet-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubelet-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + export CRICTL_TOOLS_VERSION="$(echo ${KUBERNETES_VERSION} | cut -d. -f1-2).0" && \ + curl -o /opt/cri-tools-${CRICTL_TOOLS_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/cri-tools-${CRICTL_TOOLS_VERSION}-0.x86_64.rpm && \ + export CNI_VERSION=$(awk -F': ' '/kubernetes_cni_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '1p' | xargs) && \ + curl -o /opt/kubernetes-cni-${CNI_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubernetes-cni-${CNI_VERSION}-0.x86_64.rpm + +RUN curl -o /opt/d2iq-sign-authority-gpg-public-key https://packages.d2iq.com/konvoy/stable/linux/repos/d2iq-sign-authority-gpg-public-key COPY --from=packer-amd64 /bin/packer /usr/local/bin/packer-amd64 COPY --from=packer-arm64 /bin/packer /usr/local/bin/packer-arm64 diff --git a/ansible/roles/kubeadm/tasks/redhat.yaml b/ansible/roles/kubeadm/tasks/redhat.yaml index 2fed46b13..82d96d161 100644 --- a/ansible/roles/kubeadm/tasks/redhat.yaml +++ b/ansible/roles/kubeadm/tasks/redhat.yaml @@ -14,9 +14,43 @@ not 'kubeadm-' + package_versions.kubernetes_rpm in exportedversionlocklist.stdout )" +- block: + - name: copy cri-tools rpm + copy: + src: "/opt/{{ 'cri-tools-' + critools_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'cri-tools-' + critools_rpm }}.rpm" + + - name: install cri-tools rpm package + yum: + name: "/opt/{{ 'cri-tools-' + critools_rpm }}.rpm" + state: present + update_cache: true + enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" + disablerepo: "{{ '*' if offline_mode_enabled else '' }}" + register: result + until: result is success + retries: 3 + delay: 3 + +# If the rpms for the kubernetes version provided by the customer +# exists on the current container, we should copy it to the remote +# and install it with the file. +- name: check kubeadm rpm exists for provided version + stat: + path: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + delegate_to: localhost + register: haslocalkubeadm + become: false + +- name: copy kubeadm rpm + copy: + src: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}.rpm" + when: haslocalkubeadm.stat.exists + - name: install kubeadm rpm package yum: - name: "{{ 'kubeadm-' + package_versions.kubernetes_rpm }}" + name: "{{ '/opt/' if haslocalkubeadm.stat.exists }}{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalkubeadm.stat.exists }}" state: present update_cache: true enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" @@ -34,4 +68,4 @@ changed_when: | 'command_result.stdout is regex(".*versionlock added: [1-9]+.*")' when: - - versionlock_plugin_enabled + - versionlock_plugin_enabled \ No newline at end of file diff --git a/ansible/roles/packages/tasks/redhat.yaml b/ansible/roles/packages/tasks/redhat.yaml index d15597aff..76ab29715 100644 --- a/ansible/roles/packages/tasks/redhat.yaml +++ b/ansible/roles/packages/tasks/redhat.yaml @@ -65,9 +65,47 @@ - versionlock_plugin_enabled - item in exportedversionlocklist.stdout +# If the rpms for the kubernetes version provided by the customer +# exists on the current container, we should copy it to the remote +# and install it with the file. +- name: check kubernetes rpms exist for provided version + stat: + path: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + delegate_to: localhost + register: haslocalk8srpms + become: false + +- block: + - name: copy gpg key + copy: + src: /opt/d2iq-sign-authority-gpg-public-key + dest: /opt/d2iq-sign-authority-gpg-public-key + + - name: import key + ansible.builtin.rpm_key: + state: present + key: /opt/d2iq-sign-authority-gpg-public-key + + - name: copy kubectl rpm + copy: + src: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}.rpm" + + - name: copy kubernetes_cni rpm + copy: + src: "/opt/{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0.rpm" + + - name: copy kubelet rpm + copy: + src: "/opt/{{ 'kubelet-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'kubelet-' + package_versions.kubernetes_rpm }}.rpm" + when: + - haslocalk8srpms.stat.exists + - name: install kubectl rpm package yum: - name: "{{ 'kubectl-' + package_versions.kubernetes_rpm }}" + name: "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalk8srpms.stat.exists }}" state: present update_cache: true enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" @@ -77,15 +115,17 @@ retries: 3 delay: 3 -- name: install kubelet rpm package +- name: install kubernetes_cni and kubelet rpm packages yum: - name: "{{ 'kubelet-' + package_versions.kubernetes_rpm }}" + name: + - "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0{{ '.rpm' if haslocalk8srpms.stat.exists }}" + - "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubelet-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalk8srpms.stat.exists }}" state: present update_cache: true enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" disablerepo: "{{ '*' if offline_mode_enabled else '' }}" - register: kubelet_installation_rpm - until: kubelet_installation_rpm is success + register: installation_rpm + until: installation_rpm is success retries: 3 delay: 3 @@ -100,4 +140,4 @@ changed_when: > 'command_result.stdout is regex(".*versionlock added: [1-9]+.*")' when: - - versionlock_plugin_enabled + - versionlock_plugin_enabled \ No newline at end of file diff --git a/bundles/centos7.9/bundle.sh.gotmpl b/bundles/centos7.9/bundle.sh.gotmpl index 5df13d64b..74d7cc7b2 100755 --- a/bundles/centos7.9/bundle.sh.gotmpl +++ b/bundles/centos7.9/bundle.sh.gotmpl @@ -12,6 +12,7 @@ echo skip_missing_names_on_install=False >> /etc/yum.conf yum -y install epel-release gettext yum-utils createrepo yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046 diff --git a/bundles/redhat7.9/bundle.sh.gotmpl b/bundles/redhat7.9/bundle.sh.gotmpl index 56681855f..7c0b1a2a3 100755 --- a/bundles/redhat7.9/bundle.sh.gotmpl +++ b/bundles/redhat7.9/bundle.sh.gotmpl @@ -58,6 +58,7 @@ subscription-manager repos --enable=rhel-7-server-extras-rpms yum -y install createrepo gettext yum-utils https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046 diff --git a/bundles/redhat8.4/bundle.sh.gotmpl b/bundles/redhat8.4/bundle.sh.gotmpl index a97baca52..71fe2a8ac 100755 --- a/bundles/redhat8.4/bundle.sh.gotmpl +++ b/bundles/redhat8.4/bundle.sh.gotmpl @@ -51,6 +51,7 @@ subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms yum --disablerepo=appstream-centos -y install gettext yum-utils createrepo dnf-utils yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index 11411f079..3e2efe36b 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -55,7 +55,7 @@ if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then subscription-manager refresh subscription::defer_unregister - ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" + ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" EUS_REPOS=${EUS_REPOS:-""} if [[ -n "${EUS_REPOS}" ]]; then #disables the standard repositories which should not be enabled when using EUS @@ -63,7 +63,7 @@ if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms - ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" + ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" else subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms @@ -73,6 +73,11 @@ else echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true." fi +KUBERNETES_REPOS=${KUBERNETES_REPOS:-""} +if [[ -n "${KUBERNETES_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},kubernetes" +fi + if [[ -f /etc/yum.repos.d/user-repos.repo ]]; then USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)" if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then diff --git a/bundles/redhat8.6/packages.txt.gotmpl b/bundles/redhat8.6/packages.txt.gotmpl index 89197c37a..f44fda428 100644 --- a/bundles/redhat8.6/packages.txt.gotmpl +++ b/bundles/redhat8.6/packages.txt.gotmpl @@ -15,16 +15,18 @@ yum-utils cloud-init cloud-utils-growpart container-selinux +{{ if .FetchKubernetesRPMs -}} kubectl-{{ .KubernetesVersion }}-0 kubelet-{{ .KubernetesVersion }}-0 kubeadm-{{ .KubernetesVersion }}-0 +cri-tools +{{- end }} conntrack ebtables ethtool iproute iptables socat -cri-tools gcc make elfutils-libelf-devel diff --git a/bundles/redhat8.8/bundle.sh.gotmpl b/bundles/redhat8.8/bundle.sh.gotmpl index 6063d9fb8..b74df65ea 100755 --- a/bundles/redhat8.8/bundle.sh.gotmpl +++ b/bundles/redhat8.8/bundle.sh.gotmpl @@ -55,7 +55,7 @@ if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then subscription-manager refresh subscription::defer_unregister - ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" + ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" EUS_REPOS=${EUS_REPOS:-""} if [[ -n "${EUS_REPOS}" ]]; then #disables the standard repositories which should not be enabled when using EUS @@ -63,7 +63,7 @@ if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms - ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" + ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" else subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms @@ -73,6 +73,11 @@ else echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true." fi +KUBERNETES_REPOS=${KUBERNETES_REPOS:-""} +if [[ -n "${KUBERNETES_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},kubernetes" +fi + if [[ -f /etc/yum.repos.d/user-repos.repo ]]; then USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)" if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then diff --git a/bundles/redhat8.8/packages.txt.gotmpl b/bundles/redhat8.8/packages.txt.gotmpl index 62edd8f8c..0157e9fb8 100644 --- a/bundles/redhat8.8/packages.txt.gotmpl +++ b/bundles/redhat8.8/packages.txt.gotmpl @@ -15,16 +15,18 @@ yum-utils cloud-init cloud-utils-growpart container-selinux +{{ if .FetchKubernetesRPMs -}} kubectl-{{ .KubernetesVersion }}-0 kubelet-{{ .KubernetesVersion }}-0 kubeadm-{{ .KubernetesVersion }}-0 +cri-tools +{{- end }} conntrack ebtables ethtool iproute iptables socat -cri-tools gcc make libseccomp diff --git a/bundles/rocky9.1/bundle.sh.gotmpl b/bundles/rocky9.1/bundle.sh.gotmpl index 7b6321ff0..da92aa6cb 100755 --- a/bundles/rocky9.1/bundle.sh.gotmpl +++ b/bundles/rocky9.1/bundle.sh.gotmpl @@ -8,6 +8,7 @@ echo skip_missing_names_on_install=False >> /etc/yum.conf yum -y install epel-release gettext yum-utils createrepo dnf-utils yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046 @@ -15,7 +16,6 @@ repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(< packag #shellcheck disable=SC2046 yumdownloader --archlist=x86_64,noarch -x \*i686 $(< packages.txt) rm packages.txt -curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm createrepo -v . && chown -R 1000:1000 repodata/ #shellcheck disable=SC1083,SC2035 tar -czf {{ .OutputDirectory }}/{{ .KubernetesVersion }}_rocky_9.1_x86_64.tar.gz * diff --git a/bundles/rocky9.1/packages.txt.gotmpl b/bundles/rocky9.1/packages.txt.gotmpl index c20daf350..85472e688 100644 --- a/bundles/rocky9.1/packages.txt.gotmpl +++ b/bundles/rocky9.1/packages.txt.gotmpl @@ -15,16 +15,18 @@ yum-utils cloud-init cloud-utils-growpart container-selinux +{{ if .FetchKubernetesRPMs -}} kubectl-{{ .KubernetesVersion }}-0 kubelet-{{ .KubernetesVersion }}-0 kubeadm-{{ .KubernetesVersion }}-0 +cri-tools +{{- end }} conntrack ebtables ethtool iproute iptables socat -cri-tools gcc libseccomp nfs-utils diff --git a/bundles/ubuntu20.04/bundle.sh.gotmpl b/bundles/ubuntu20.04/bundle.sh.gotmpl index f38b7f625..324e48539 100644 --- a/bundles/ubuntu20.04/bundle.sh.gotmpl +++ b/bundles/ubuntu20.04/bundle.sh.gotmpl @@ -24,6 +24,7 @@ sed -i 's/cri-tools/cri-tools='"{{ .CRIToolsVersion }}-${DEB_PATCH_VERSION}"'/' sed -i 's/cloud-init/cloud-init='"23.1.2-0ubuntu0~20.04.2"'/' /tmp/packages TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046 apt-get download $(< /tmp/packages) diff --git a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go index c28a03325..73f0c881d 100644 --- a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go +++ b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go @@ -170,17 +170,23 @@ func (r *Runner) CreatePackageBundle(args []string) error { return err } } + fetchKubernetesRPMs := true kubernetesVersion := kubernetesVersionFlag if kubernetesVersion == "" { kubernetesVersion, err = getKubernetesVerisonFromAnsible() if err != nil { return err } + // if we are getting the default version from ansible, we don't need to modify this. + fetchKubernetesRPMs = false } if eusReposFlag { //nolint:goconst // it is ok to not use const here r.env["EUS_REPOS"] = "true" } + if fetchKubernetesRPMs { + r.env["KUBERNETES_REPOS"] = "true" + } if satelliteFlag != "" { r.env["SATELLITE_SERVER_URL"] = satelliteFlag } @@ -194,7 +200,7 @@ func (r *Runner) CreatePackageBundle(args []string) error { dir := r.workingDir absPathToOutput = path.Join(dir, outputDirectoy) } - reposList, err := templateObjects(osFlag, kubernetesVersion, absPathToOutput, fipsFlag, fetchKernelHeaders) + reposList, err := templateObjects(osFlag, kubernetesVersion, absPathToOutput, fipsFlag, fetchKernelHeaders, fetchKubernetesRPMs) if err != nil { return err } @@ -209,7 +215,7 @@ func (r *Runner) CreatePackageBundle(args []string) error { } //nolint:gocyclo,funlen // the function is relatively clear -func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool, fetchKernelHeaders bool) ([]string, error) { +func templateObjects(targetOS, kubernetesVersion, outputDir string, fips, fetchKernelHeaders, fetchKubernetesRPMs bool) ([]string, error) { config, found := osToConfig[targetOS] if !found { return nil, fmt.Errorf("buildOS %s is invalid must be one of %v", targetOS, getKeys(osToConfig)) @@ -259,7 +265,8 @@ func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool, f } //nolint:nestif // this if is not nested - if strings.Contains(filepath, "kubernetes.repo.gotmpl") { + if strings.Contains(filepath, "kubernetes.repo.gotmpl") && fetchKubernetesRPMs { + fmt.Printf("fetchKubernetesRPMs is %v", fetchKubernetesRPMs) kubernetesRepoTmpl, err := os.ReadFile(path.Join(base, filepath)) if err != nil { return fmt.Errorf("failed to read template kubernetes repo file %w", err) @@ -304,11 +311,13 @@ func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool, f return fmt.Errorf("failed to create file: %w", err) } templateInput := struct { - KubernetesVersion string - FetchKernelHeaders bool + KubernetesVersion string + FetchKernelHeaders bool + FetchKubernetesRPMs bool }{ - KubernetesVersion: kubernetesVersion, - FetchKernelHeaders: fetchKernelHeaders, + KubernetesVersion: kubernetesVersion, + FetchKernelHeaders: fetchKernelHeaders, + FetchKubernetesRPMs: fetchKubernetesRPMs, } err = t.Execute(out, templateInput) if err != nil { From 8cb5c525b00417f02f48e56191aeba46b10415ba Mon Sep 17 00:00:00 2001 From: Dustin Nemes <39643802+some-things@users.noreply.github.com> Date: Wed, 21 Aug 2024 15:48:48 -0500 Subject: [PATCH 11/11] fix: rename user-defined repo filename and improve parsing --- bundles/redhat8.6/bundle.sh.gotmpl | 12 ++++++------ bundles/redhat8.8/bundle.sh.gotmpl | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index 3e2efe36b..e491faa14 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -78,12 +78,12 @@ if [[ -n "${KUBERNETES_REPOS}" ]]; then ENABLED_REPOS="${ENABLED_REPOS},kubernetes" fi -if [[ -f /etc/yum.repos.d/user-repos.repo ]]; then - USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)" - if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then - ENABLED_REPOS="${ENABLED_REPOS},${USER_REPOS}" - elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then - ENABLED_REPOS="${USER_REPOS}" +if [[ -f /etc/yum.repos.d/additional-repos.repo ]]; then + ADDITIONAL_REPOS="$(/usr/libexec/platform-python3.6 -c "import configparser; config = configparser.ConfigParser(); config.read('/etc/yum.repos.d/additional-repos.repo'); print(','.join(config.sections()) if config.sections() else '')")" + if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${ADDITIONAL_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},${ADDITIONAL_REPOS}" + elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${ADDITIONAL_REPOS}" ]]; then + ENABLED_REPOS="${ADDITIONAL_REPOS}" fi fi diff --git a/bundles/redhat8.8/bundle.sh.gotmpl b/bundles/redhat8.8/bundle.sh.gotmpl index b74df65ea..952cb38bd 100755 --- a/bundles/redhat8.8/bundle.sh.gotmpl +++ b/bundles/redhat8.8/bundle.sh.gotmpl @@ -78,12 +78,12 @@ if [[ -n "${KUBERNETES_REPOS}" ]]; then ENABLED_REPOS="${ENABLED_REPOS},kubernetes" fi -if [[ -f /etc/yum.repos.d/user-repos.repo ]]; then - USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)" - if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then - ENABLED_REPOS="${ENABLED_REPOS},${USER_REPOS}" - elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then - ENABLED_REPOS="${USER_REPOS}" +if [[ -f /etc/yum.repos.d/additional-repos.repo ]]; then + ADDITIONAL_REPOS="$(/usr/libexec/platform-python3.6 -c "import configparser; config = configparser.ConfigParser(); config.read('/etc/yum.repos.d/additional-repos.repo'); print(','.join(config.sections()) if config.sections() else '')")" + if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${ADDITIONAL_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},${ADDITIONAL_REPOS}" + elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${ADDITIONAL_REPOS}" ]]; then + ENABLED_REPOS="${ADDITIONAL_REPOS}" fi fi