diff --git a/Dockerfile b/Dockerfile index 087447e32..59eaf9927 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,8 @@ ARG BUILDARCH # we copy this to remote hosts to execute GOSS # Packer copies /usr/local/bin/goss-amd64 from this container to the remote host COPY --from=devkit /usr/local/bin/goss-amd64 /usr/local/bin/goss-amd64 +COPY --from=devkit /opt/*.rpm /opt +COPY --from=devkit /opt/d2iq-sign-authority-gpg-public-key /opt/d2iq-sign-authority-gpg-public-key # we copy this to remote hosts to execute mindthegap so its always amd64 COPY --from=devkit /usr/local/bin/mindthegap /usr/local/bin/ diff --git a/Dockerfile.devkit b/Dockerfile.devkit index cd856acb0..778ec1634 100644 --- a/Dockerfile.devkit +++ b/Dockerfile.devkit @@ -78,6 +78,34 @@ RUN curl -L "https://github.com/goss-org/goss/releases/download/${GOSS_VERSION}/ RUN chmod +rx /usr/local/bin/goss-amd64 ARG BUILDARCH RUN ln -s /usr/local/bin/goss-${BUILDARCH} /usr/local/bin/goss +RUN curl -o /opt/amazon-ssm-agent.rpm https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm +COPY ansible ansible +# Fetch nokmem rpms +RUN \ + export KUBERNETES_VERSION=$(awk -F': ' '/kubernetes_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '2p' | xargs) && \ + echo ${KUBERNETES_VERSION} && \ + curl -o /opt/kubectl-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubectl-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + curl -o /opt/kubeadm-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubeadm-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + curl -o /opt/kubelet-${KUBERNETES_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubelet-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + export CRICTL_TOOLS_VERSION="$(echo ${KUBERNETES_VERSION} | cut -d. -f1-2).0" && \ + curl -o /opt/cri-tools-${CRICTL_TOOLS_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/cri-tools-${CRICTL_TOOLS_VERSION}-0.x86_64.rpm && \ + export CNI_VERSION=$(awk -F': ' '/kubernetes_cni_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '1p' | xargs) && \ + curl -o /opt/kubernetes-cni-${CNI_VERSION}-0.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/kubernetes-cni-${CNI_VERSION}-0.x86_64.rpm + + +# Fetch fips rpms +RUN \ + export KUBERNETES_VERSION=$(awk -F': ' '/kubernetes_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '2p' | xargs) && \ + echo ${KUBERNETES_VERSION} && \ + curl -o /opt/kubectl-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubectl-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + curl -o /opt/kubeadm-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubeadm-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + curl -o /opt/kubelet-${KUBERNETES_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubelet-${KUBERNETES_VERSION}-0.x86_64.rpm && \ + export CRICTL_TOOLS_VERSION="$(echo ${KUBERNETES_VERSION} | cut -d. -f1-2).0" && \ + curl -o /opt/cri-tools-${CRICTL_TOOLS_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-nokmem/x86_64/cri-tools-${CRICTL_TOOLS_VERSION}-0.x86_64.rpm && \ + export CNI_VERSION=$(awk -F': ' '/kubernetes_cni_version/ {print $2}' ansible/group_vars/all/defaults.yaml | sed -n '1p' | xargs) && \ + curl -o /opt/kubernetes-cni-${CNI_VERSION}-0-fips.rpm https://packages.d2iq.com/konvoy/stable/linux/repos/el/kubernetes-v${KUBERNETES_VERSION}-fips/x86_64/kubernetes-cni-${CNI_VERSION}-0.x86_64.rpm + +RUN curl -o /opt/d2iq-sign-authority-gpg-public-key https://packages.d2iq.com/konvoy/stable/linux/repos/d2iq-sign-authority-gpg-public-key COPY --from=packer-amd64 /bin/packer /usr/local/bin/packer-amd64 COPY --from=packer-arm64 /bin/packer /usr/local/bin/packer-arm64 diff --git a/ansible/roles/kubeadm/tasks/redhat.yaml b/ansible/roles/kubeadm/tasks/redhat.yaml index 2fed46b13..82d96d161 100644 --- a/ansible/roles/kubeadm/tasks/redhat.yaml +++ b/ansible/roles/kubeadm/tasks/redhat.yaml @@ -14,9 +14,43 @@ not 'kubeadm-' + package_versions.kubernetes_rpm in exportedversionlocklist.stdout )" +- block: + - name: copy cri-tools rpm + copy: + src: "/opt/{{ 'cri-tools-' + critools_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'cri-tools-' + critools_rpm }}.rpm" + + - name: install cri-tools rpm package + yum: + name: "/opt/{{ 'cri-tools-' + critools_rpm }}.rpm" + state: present + update_cache: true + enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" + disablerepo: "{{ '*' if offline_mode_enabled else '' }}" + register: result + until: result is success + retries: 3 + delay: 3 + +# If the rpms for the kubernetes version provided by the customer +# exists on the current container, we should copy it to the remote +# and install it with the file. +- name: check kubeadm rpm exists for provided version + stat: + path: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + delegate_to: localhost + register: haslocalkubeadm + become: false + +- name: copy kubeadm rpm + copy: + src: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'kubeadm-' + package_versions.kubernetes_rpm }}.rpm" + when: haslocalkubeadm.stat.exists + - name: install kubeadm rpm package yum: - name: "{{ 'kubeadm-' + package_versions.kubernetes_rpm }}" + name: "{{ '/opt/' if haslocalkubeadm.stat.exists }}{{ 'kubeadm-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalkubeadm.stat.exists }}" state: present update_cache: true enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" @@ -34,4 +68,4 @@ changed_when: | 'command_result.stdout is regex(".*versionlock added: [1-9]+.*")' when: - - versionlock_plugin_enabled + - versionlock_plugin_enabled \ No newline at end of file diff --git a/ansible/roles/packages/tasks/redhat.yaml b/ansible/roles/packages/tasks/redhat.yaml index d15597aff..76ab29715 100644 --- a/ansible/roles/packages/tasks/redhat.yaml +++ b/ansible/roles/packages/tasks/redhat.yaml @@ -65,9 +65,47 @@ - versionlock_plugin_enabled - item in exportedversionlocklist.stdout +# If the rpms for the kubernetes version provided by the customer +# exists on the current container, we should copy it to the remote +# and install it with the file. +- name: check kubernetes rpms exist for provided version + stat: + path: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + delegate_to: localhost + register: haslocalk8srpms + become: false + +- block: + - name: copy gpg key + copy: + src: /opt/d2iq-sign-authority-gpg-public-key + dest: /opt/d2iq-sign-authority-gpg-public-key + + - name: import key + ansible.builtin.rpm_key: + state: present + key: /opt/d2iq-sign-authority-gpg-public-key + + - name: copy kubectl rpm + copy: + src: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'kubectl-' + package_versions.kubernetes_rpm }}.rpm" + + - name: copy kubernetes_cni rpm + copy: + src: "/opt/{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0.rpm" + + - name: copy kubelet rpm + copy: + src: "/opt/{{ 'kubelet-' + package_versions.kubernetes_rpm }}{{ '-fips' if fips.enabled else '' }}.rpm" + dest: "/opt/{{ 'kubelet-' + package_versions.kubernetes_rpm }}.rpm" + when: + - haslocalk8srpms.stat.exists + - name: install kubectl rpm package yum: - name: "{{ 'kubectl-' + package_versions.kubernetes_rpm }}" + name: "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubectl-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalk8srpms.stat.exists }}" state: present update_cache: true enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" @@ -77,15 +115,17 @@ retries: 3 delay: 3 -- name: install kubelet rpm package +- name: install kubernetes_cni and kubelet rpm packages yum: - name: "{{ 'kubelet-' + package_versions.kubernetes_rpm }}" + name: + - "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubernetes-cni-' + kubernetes_cni_version }}-0{{ '.rpm' if haslocalk8srpms.stat.exists }}" + - "{{ '/opt/' if haslocalk8srpms.stat.exists }}{{ 'kubelet-' + package_versions.kubernetes_rpm }}{{ '.rpm' if haslocalk8srpms.stat.exists }}" state: present update_cache: true enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" disablerepo: "{{ '*' if offline_mode_enabled else '' }}" - register: kubelet_installation_rpm - until: kubelet_installation_rpm is success + register: installation_rpm + until: installation_rpm is success retries: 3 delay: 3 @@ -100,4 +140,4 @@ changed_when: > 'command_result.stdout is regex(".*versionlock added: [1-9]+.*")' when: - - versionlock_plugin_enabled + - versionlock_plugin_enabled \ No newline at end of file diff --git a/ansible/roles/providers/tasks/aws.yml b/ansible/roles/providers/tasks/aws.yml index fb536a227..e20fa9dc3 100644 --- a/ansible/roles/providers/tasks/aws.yml +++ b/ansible/roles/providers/tasks/aws.yml @@ -42,38 +42,34 @@ - ansible_distribution != "Amazon" - ansible_os_family != "Suse" -- name: install aws agents RPM - package: - name: "{{ item }}" - state: present - # must be fixed by amazon https://github.com/aws/amazon-ssm-agent/issues/235 - disable_gpg_check: yes - enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" - disablerepo: "{{ '*' if offline_mode_enabled else '' }}" - with_items: - - "{{ 'amazon-ssm-agent' if offline_mode_enabled else 'https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm' }}" - when: - - ansible_os_family == "RedHat" - - ansible_distribution != "Amazon" -- name: install aws agents RPM - package: - name: "{{ item }}" - state: present - # must be fixed by amazon https://github.com/aws/amazon-ssm-agent/issues/235 - disable_gpg_check: yes - with_items: - - https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm - when: - - ansible_os_family == "Suse" - - ansible_distribution != "Amazon" +- block: + - name: copy ssm rpm + copy: + src: /opt/amazon-ssm-agent.rpm + dest: /opt/amazon-ssm-agent.rpm -- name: install aws agents RPM + - name: install aws agents RPM + ansible.builtin.yum: + name: /opt/amazon-ssm-agent.rpm + state: present + disable_gpg_check: yes + enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}" + disablerepo: "{{ '*' if offline_mode_enabled else '' }}" + when: ansible_os_family == "RedHat" and ansible_distribution != "Amazon" + + - name: install aws agents RPM + ansible.builtin.zypper: + name: /opt/amazon-ssm-agent.rpm + state: present + disable_gpg_check: yes + when: ansible_os_family == "Suse" and ansible_distribution != "Amazon" + +- name: install aws cli for amazon linux package: name: "{{ item }}" state: present with_items: - - amazon-ssm-agent - awscli when: ansible_distribution == "Amazon" @@ -93,5 +89,4 @@ name: snap.amazon-ssm-agent.amazon-ssm-agent.service state: started enabled: yes - when: ansible_distribution == "Ubuntu" - + when: ansible_distribution == "Ubuntu" \ No newline at end of file diff --git a/bundles/centos7.9/bundle.sh.gotmpl b/bundles/centos7.9/bundle.sh.gotmpl index 5df13d64b..74d7cc7b2 100755 --- a/bundles/centos7.9/bundle.sh.gotmpl +++ b/bundles/centos7.9/bundle.sh.gotmpl @@ -12,6 +12,7 @@ echo skip_missing_names_on_install=False >> /etc/yum.conf yum -y install epel-release gettext yum-utils createrepo yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046 diff --git a/bundles/redhat7.9/bundle.sh.gotmpl b/bundles/redhat7.9/bundle.sh.gotmpl index 56681855f..7c0b1a2a3 100755 --- a/bundles/redhat7.9/bundle.sh.gotmpl +++ b/bundles/redhat7.9/bundle.sh.gotmpl @@ -58,6 +58,7 @@ subscription-manager repos --enable=rhel-7-server-extras-rpms yum -y install createrepo gettext yum-utils https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046 diff --git a/bundles/redhat8.4/bundle.sh.gotmpl b/bundles/redhat8.4/bundle.sh.gotmpl index 56c9fec29..71fe2a8ac 100755 --- a/bundles/redhat8.4/bundle.sh.gotmpl +++ b/bundles/redhat8.4/bundle.sh.gotmpl @@ -51,6 +51,7 @@ subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms yum --disablerepo=appstream-centos -y install gettext yum-utils createrepo dnf-utils yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 @@ -61,7 +62,6 @@ yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=Fa #shellcheck disable=SC2046 yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch --resolve --disablerepo=* --enablerepo=kubernetes,rhel-8-for-x86_64-baseos-eus-rpms,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms --disablerepo=appstream-centos $(< packages.txt) rm packages.txt reqs.txt -curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm createrepo -v . chown -R 1000:1000 repodata/ yum install -y modulemd-tools diff --git a/bundles/redhat8.6/bundle.sh.gotmpl b/bundles/redhat8.6/bundle.sh.gotmpl index 46d69afb9..e491faa14 100755 --- a/bundles/redhat8.6/bundle.sh.gotmpl +++ b/bundles/redhat8.6/bundle.sh.gotmpl @@ -3,7 +3,7 @@ set -euo pipefail # When yum operates on multiple packages, it does not, by default, return an error if a subset # of packages is not found. This makes yum return an error. -echo skip_missing_names_on_install=False >> /etc/yum.conf +echo skip_missing_names_on_install=False >>/etc/yum.conf sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py FOUND=false @@ -13,26 +13,35 @@ RHSM_ACTIVATION_KEY=${RHSM_ACTIVATION_KEY:-""} RHSM_USER=${RHSM_USER:-""} RHSM_PASS=${RHSM_PASS:-""} -if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then - subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force - FOUND=true -fi +SKIP_SUBSCRIPTION_MANAGER=${SKIP_SUBSCRIPTION_MANAGER:-""} +SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""} +ENABLED_REPOS=${ENABLED_REPOS:-""} -if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then - subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force - FOUND=true -fi +if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then + if [[ -n "${SATELLITE_SERVER_URL}" ]]; then + rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" + fi -if [[ ${FOUND} == false ]]; then - echo "You must use subscription manager to fetch packages for redhat" - exit 1 -fi + if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then + subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force + FOUND=true + fi -subscription::unregister() { - subscription-manager unregister -} + if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then + subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force + FOUND=true + fi -subscription::defer_unregister() { + if [[ ${FOUND} == false ]]; then + echo "You must use subscription manager to fetch packages for redhat" + exit 1 + fi + + subscription::unregister() { + subscription-manager unregister + } + + subscription::defer_unregister() { trap subscription::unregister ABRT trap subscription::unregister EXIT trap subscription::unregister HUP @@ -40,30 +49,63 @@ subscription::defer_unregister() { trap subscription::unregister TERM trap subscription::unregister USR1 trap subscription::unregister USR2 -} + } + + subscription-manager release --set=8.6 + subscription-manager refresh + subscription::defer_unregister + + ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" + EUS_REPOS=${EUS_REPOS:-""} + if [[ -n "${EUS_REPOS}" ]]; then + #disables the standard repositories which should not be enabled when using EUS + subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms + ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" + else + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms + fi +else + echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true." +fi + +KUBERNETES_REPOS=${KUBERNETES_REPOS:-""} +if [[ -n "${KUBERNETES_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},kubernetes" +fi + +if [[ -f /etc/yum.repos.d/additional-repos.repo ]]; then + ADDITIONAL_REPOS="$(/usr/libexec/platform-python3.6 -c "import configparser; config = configparser.ConfigParser(); config.read('/etc/yum.repos.d/additional-repos.repo'); print(','.join(config.sections()) if config.sections() else '')")" + if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${ADDITIONAL_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},${ADDITIONAL_REPOS}" + elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${ADDITIONAL_REPOS}" ]]; then + ENABLED_REPOS="${ADDITIONAL_REPOS}" + fi +fi -subscription-manager release --set=8.6 -subscription-manager refresh -subscription::defer_unregister -subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms -subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms -subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms -yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools +yum -y install --disablerepo=* --enablerepo="${ENABLED_REPOS}" gettext yum-utils createrepo dnf-utils modulemd-tools yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 -repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(< packages.txt) | grep -v *.i686 >> reqs.txt -sed -i 1d reqs.txt # we need to get rid of the first line +repoquery --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ + --archlist=x86_64,noarch --resolve --requires --recursive $(>reqs.txt #shellcheck disable=SC2046 -yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt) +yumdownloader --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ + --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch $(> /etc/yum.conf +echo skip_missing_names_on_install=False >>/etc/yum.conf sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py FOUND=false @@ -13,26 +13,35 @@ RHSM_ACTIVATION_KEY=${RHSM_ACTIVATION_KEY:-""} RHSM_USER=${RHSM_USER:-""} RHSM_PASS=${RHSM_PASS:-""} -if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then - subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force - FOUND=true -fi +SKIP_SUBSCRIPTION_MANAGER=${SKIP_SUBSCRIPTION_MANAGER:-""} +SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""} +ENABLED_REPOS=${ENABLED_REPOS:-""} -if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then - subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force - FOUND=true -fi +if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then + if [[ -n "${SATELLITE_SERVER_URL}" ]]; then + rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm" + fi -if [[ ${FOUND} == false ]]; then - echo "You must use subscription manager to fetch packages for redhat" - exit 1 -fi + if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then + subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force + FOUND=true + fi -subscription::unregister() { - subscription-manager unregister -} + if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then + subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force + FOUND=true + fi -subscription::defer_unregister() { + if [[ ${FOUND} == false ]]; then + echo "You must use subscription manager to fetch packages for redhat" + exit 1 + fi + + subscription::unregister() { + subscription-manager unregister + } + + subscription::defer_unregister() { trap subscription::unregister ABRT trap subscription::unregister EXIT trap subscription::unregister HUP @@ -40,30 +49,63 @@ subscription::defer_unregister() { trap subscription::unregister TERM trap subscription::unregister USR1 trap subscription::unregister USR2 -} + } + + subscription-manager release --set=8.8 + subscription-manager refresh + subscription::defer_unregister + + ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms" + EUS_REPOS=${EUS_REPOS:-""} + if [[ -n "${EUS_REPOS}" ]]; then + #disables the standard repositories which should not be enabled when using EUS + subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms + ENABLED_REPOS="codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" + else + subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms + subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms + subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms + fi +else + echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true." +fi + +KUBERNETES_REPOS=${KUBERNETES_REPOS:-""} +if [[ -n "${KUBERNETES_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},kubernetes" +fi + +if [[ -f /etc/yum.repos.d/additional-repos.repo ]]; then + ADDITIONAL_REPOS="$(/usr/libexec/platform-python3.6 -c "import configparser; config = configparser.ConfigParser(); config.read('/etc/yum.repos.d/additional-repos.repo'); print(','.join(config.sections()) if config.sections() else '')")" + if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${ADDITIONAL_REPOS}" ]]; then + ENABLED_REPOS="${ENABLED_REPOS},${ADDITIONAL_REPOS}" + elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${ADDITIONAL_REPOS}" ]]; then + ENABLED_REPOS="${ADDITIONAL_REPOS}" + fi +fi -subscription-manager release --set=8.8 -subscription-manager refresh -subscription::defer_unregister -subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms -subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms -subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms -yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools +yum -y install --disablerepo=* --enablerepo="${ENABLED_REPOS}" gettext yum-utils createrepo dnf-utils modulemd-tools yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046,SC2062,SC2063,SC2035 -repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(< packages.txt) | grep -v *.i686 >> reqs.txt -sed -i 1d reqs.txt # we need to get rid of the first line +repoquery --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ + --archlist=x86_64,noarch --resolve --requires --recursive $(>reqs.txt #shellcheck disable=SC2046 -yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt) +yumdownloader --disablerepo=* --enablerepo="${ENABLED_REPOS}" \ + --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch $(> /etc/yum.conf yum -y install epel-release gettext yum-utils createrepo dnf-utils yum clean all TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" cp packages.txt "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046 @@ -15,7 +16,6 @@ repoquery --archlist=x86_64,noarch --resolve --requires --recursive $(< packag #shellcheck disable=SC2046 yumdownloader --archlist=x86_64,noarch -x \*i686 $(< packages.txt) rm packages.txt -curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm createrepo -v . && chown -R 1000:1000 repodata/ #shellcheck disable=SC1083,SC2035 tar -czf {{ .OutputDirectory }}/{{ .KubernetesVersion }}_rocky_9.1_x86_64.tar.gz * diff --git a/bundles/rocky9.1/packages.txt.gotmpl b/bundles/rocky9.1/packages.txt.gotmpl index c20daf350..85472e688 100644 --- a/bundles/rocky9.1/packages.txt.gotmpl +++ b/bundles/rocky9.1/packages.txt.gotmpl @@ -15,16 +15,18 @@ yum-utils cloud-init cloud-utils-growpart container-selinux +{{ if .FetchKubernetesRPMs -}} kubectl-{{ .KubernetesVersion }}-0 kubelet-{{ .KubernetesVersion }}-0 kubeadm-{{ .KubernetesVersion }}-0 +cri-tools +{{- end }} conntrack ebtables ethtool iproute iptables socat -cri-tools gcc libseccomp nfs-utils diff --git a/bundles/ubuntu20.04/bundle.sh.gotmpl b/bundles/ubuntu20.04/bundle.sh.gotmpl index f38b7f625..324e48539 100644 --- a/bundles/ubuntu20.04/bundle.sh.gotmpl +++ b/bundles/ubuntu20.04/bundle.sh.gotmpl @@ -24,6 +24,7 @@ sed -i 's/cri-tools/cri-tools='"{{ .CRIToolsVersion }}-${DEB_PATCH_VERSION}"'/' sed -i 's/cloud-init/cloud-init='"23.1.2-0ubuntu0~20.04.2"'/' /tmp/packages TMP_DIR="$(mktemp -d repodata-XXXX)" +chmod 777 -R "${TMP_DIR}" pushd "${TMP_DIR}" #shellcheck disable=SC2046 apt-get download $(< /tmp/packages) diff --git a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go index 072a1f117..73f0c881d 100644 --- a/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go +++ b/cmd/konvoy-image-wrapper/cmd/create-package-bundle.go @@ -88,21 +88,74 @@ func getKubernetesVerisonFromAnsible() (string, error) { func (r *Runner) CreatePackageBundle(args []string) error { var ( - osFlag string - kubernetesVersionFlag string - fipsFlag bool - outputDirectoy string - containerImage string + osFlag string + kubernetesVersionFlag string + fipsFlag bool + eusReposFlag bool + satelliteFlag string + subscriptionManagerFlag bool + outputDirectoy string + containerImage string + fetchKernelHeaders bool ) flagSet := flag.NewFlagSet(createPackageBundleCmd, flag.ExitOnError) - flagSet.StringVar(&osFlag, "os", "", - fmt.Sprintf("The target OS you wish to create a package bundle for. Must be one of %v", getKeys(osToConfig))) - flagSet.StringVar(&kubernetesVersionFlag, "kubernetes-version", "", - "The version of kubernetes to download packages for.") - flagSet.BoolVar(&fipsFlag, "fips", false, "If the package bundle should include fips packages.") - flagSet.StringVar(&outputDirectoy, "output-directory", "artifacts", - "The directory to place the bundle in.") - flagSet.StringVar(&containerImage, "container-image", "", "A container image to use for building the package bundles") + flagSet.StringVar( + &osFlag, + "os", + "", + fmt.Sprintf("The target OS you wish to create a package bundle for. Must be one of %v", getKeys(osToConfig)), + ) + flagSet.StringVar( + &kubernetesVersionFlag, + "kubernetes-version", + "", + "The version of kubernetes to download packages for.", + ) + flagSet.BoolVar( + &fipsFlag, + "fips", + false, + "If the package bundle should include fips packages.", + ) + flagSet.BoolVar( + &eusReposFlag, + "enable-eus-repos", + false, + "If enabled fetches packages from EUS repositories when creating RHEL package bundles. Disabled by default.", + ) + flagSet.StringVar( + &satelliteFlag, + "satellite-server-url", + "", + //nolint:lll // it is ok to have long help texts + "If set, registers with and fetches packages from a Red Hat Satellite. All required repositories must be available in the Red Hat Satellite. Example: --satellite-server-url=\"https://satellite.nutanix.sh\"", + ) + flagSet.BoolVar( + &subscriptionManagerFlag, + "skip-subscription-manager", + false, + //nolint:lll // it is ok to have long help texts + "If enabled, skips authenticating with subscription-manager and fetching from the pre-configured official RHEL repositories when creating RHEL package bundles. Disabled by default.", + ) + flagSet.StringVar( + &outputDirectoy, + "output-directory", + "artifacts", + "The directory to place the bundle in.", + ) + flagSet.StringVar( + &containerImage, + "container-image", + "", + "A container image to use for building the package bundles", + ) + flagSet.BoolVar( + &fetchKernelHeaders, + "fetch-kernel-headers", + false, + //nolint:lll // its ok to have long help texts + "If enabled fetches kernel headers for the target operating system. To modify the version, edit the file at bundles/{OS_NAME}{VERSION}/packages.txt.gotmpl directly eg: bundles/redhat8.8/packages.txt.gotmpl. This is required for operating systems that will use NVIDIA GPU drivers.", + ) err := flagSet.Parse(args) if err != nil { return err @@ -117,20 +170,37 @@ func (r *Runner) CreatePackageBundle(args []string) error { return err } } + fetchKubernetesRPMs := true kubernetesVersion := kubernetesVersionFlag if kubernetesVersion == "" { kubernetesVersion, err = getKubernetesVerisonFromAnsible() if err != nil { return err } + // if we are getting the default version from ansible, we don't need to modify this. + fetchKubernetesRPMs = false + } + if eusReposFlag { + //nolint:goconst // it is ok to not use const here + r.env["EUS_REPOS"] = "true" + } + if fetchKubernetesRPMs { + r.env["KUBERNETES_REPOS"] = "true" + } + if satelliteFlag != "" { + r.env["SATELLITE_SERVER_URL"] = satelliteFlag } + if subscriptionManagerFlag { + r.env["SKIP_SUBSCRIPTION_MANAGER"] = "true" + } + r.setHTTPProxyEnv() bundleCmd := "./bundle.sh" absPathToOutput := outputDirectoy if !path.IsAbs(outputDirectoy) { dir := r.workingDir absPathToOutput = path.Join(dir, outputDirectoy) } - reposList, err := templateObjects(osFlag, kubernetesVersion, absPathToOutput, fipsFlag) + reposList, err := templateObjects(osFlag, kubernetesVersion, absPathToOutput, fipsFlag, fetchKernelHeaders, fetchKubernetesRPMs) if err != nil { return err } @@ -145,7 +215,7 @@ func (r *Runner) CreatePackageBundle(args []string) error { } //nolint:gocyclo,funlen // the function is relatively clear -func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool) ([]string, error) { +func templateObjects(targetOS, kubernetesVersion, outputDir string, fips, fetchKernelHeaders, fetchKubernetesRPMs bool) ([]string, error) { config, found := osToConfig[targetOS] if !found { return nil, fmt.Errorf("buildOS %s is invalid must be one of %v", targetOS, getKeys(osToConfig)) @@ -159,7 +229,7 @@ func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool) ( configDirFS := os.DirFS(base) l := make([]string, 0) generated := path.Join(base, generatedDirName) - if err = os.MkdirAll(generated, 0o755); err != nil { + if err = os.MkdirAll(generated, 0o777); err != nil { return l, err } @@ -195,7 +265,8 @@ func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool) ( } //nolint:nestif // this if is not nested - if strings.Contains(filepath, "kubernetes.repo.gotmpl") { + if strings.Contains(filepath, "kubernetes.repo.gotmpl") && fetchKubernetesRPMs { + fmt.Printf("fetchKubernetesRPMs is %v", fetchKubernetesRPMs) kubernetesRepoTmpl, err := os.ReadFile(path.Join(base, filepath)) if err != nil { return fmt.Errorf("failed to read template kubernetes repo file %w", err) @@ -240,9 +311,13 @@ func templateObjects(targetOS, kubernetesVersion, outputDir string, fips bool) ( return fmt.Errorf("failed to create file: %w", err) } templateInput := struct { - KubernetesVersion string + KubernetesVersion string + FetchKernelHeaders bool + FetchKubernetesRPMs bool }{ - KubernetesVersion: kubernetesVersion, + KubernetesVersion: kubernetesVersion, + FetchKernelHeaders: fetchKernelHeaders, + FetchKubernetesRPMs: fetchKubernetesRPMs, } err = t.Execute(out, templateInput) if err != nil {