[wip]feat: changes to support network restricted environments when creating RHEL OS bundles

Dockerfile

@@ -32,6 +32,8 @@ ARG BUILDARCH
 # Packer copies /usr/local/bin/goss-amd64 from this container to the remote host
 COPY --from=devkit /usr/local/bin/goss-amd64 /usr/local/bin/goss-amd64
 
+COPY --from=devkit /opt/amazon-ssm-agent.rpm /opt/amazon-ssm-agent.rpm
+
 # we copy this to remote hosts to execute mindthegap so its always amd64
 COPY --from=devkit /usr/local/bin/mindthegap /usr/local/bin/
 COPY --from=devkit /usr/local/bin/packer-${BUILDARCH} /usr/local/bin/packer

Dockerfile.devkit

@@ -78,6 +78,7 @@ RUN curl -L "https://github.com/goss-org/goss/releases/download/${GOSS_VERSION}/
 RUN chmod +rx /usr/local/bin/goss-amd64
 ARG BUILDARCH
 RUN ln -s /usr/local/bin/goss-${BUILDARCH} /usr/local/bin/goss
+RUN curl -o /opt/amazon-ssm-agent.rpm https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
 
 COPY --from=packer-amd64 /bin/packer /usr/local/bin/packer-amd64
 COPY --from=packer-arm64 /bin/packer /usr/local/bin/packer-arm64

ansible/roles/providers/tasks/aws.yml

@@ -42,38 +42,34 @@
     - ansible_distribution != "Amazon"
     - ansible_os_family != "Suse"
 
-- name: install aws agents RPM
-  package:
-    name: "{{ item }}"
-    state: present
-    # must be fixed by amazon https://github.com/aws/amazon-ssm-agent/issues/235
-    disable_gpg_check: yes
-    enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}"
-    disablerepo: "{{ '*' if offline_mode_enabled else '' }}"
-  with_items:
-    - "{{ 'amazon-ssm-agent' if offline_mode_enabled else 'https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm' }}"
-  when:
-    - ansible_os_family == "RedHat"
-    - ansible_distribution != "Amazon"
 
-- name: install aws agents RPM
-  package:
-    name: "{{ item }}"
-    state: present
-    # must be fixed by amazon https://github.com/aws/amazon-ssm-agent/issues/235
-    disable_gpg_check: yes
-  with_items:
-    - https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
-  when:
-    - ansible_os_family == "Suse"
-    - ansible_distribution != "Amazon"
+- block:
+  - name: copy ssm rpm
+    copy:
+      src: /opt/amazon-ssm-agent.rpm
+      dest: /opt/amazon-ssm-agent.rpm
 
-- name: install aws agents RPM
+  - name: install aws agents RPM
+    ansible.builtin.yum:
+      name: /opt/amazon-ssm-agent.rpm
+      state: present
+      disable_gpg_check: yes
+      enablerepo: "{{ 'offline' if offline_mode_enabled else '' }}"
+      disablerepo: "{{ '*' if offline_mode_enabled else '' }}"
+    when: ansible_os_family == "RedHat" and ansible_distribution != "Amazon"
+
+  - name: install aws agents RPM
+    ansible.builtin.zypper:
+      name: /opt/amazon-ssm-agent.rpm
+      state: present
+      disable_gpg_check: yes
+    when: ansible_os_family == "Suse" and ansible_distribution != "Amazon"
+
+- name: install aws cli for amazon linux
   package:
     name: "{{ item }}"
     state: present
   with_items:
-    - amazon-ssm-agent
     - awscli
   when: ansible_distribution == "Amazon"
 
@@ -93,5 +89,4 @@
     name: snap.amazon-ssm-agent.amazon-ssm-agent.service
     state: started
     enabled: yes
-  when: ansible_distribution == "Ubuntu"
-
+  when: ansible_distribution == "Ubuntu"

bundles/redhat8.4/bundle.sh.gotmpl

@@ -61,7 +61,6 @@ yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=Fa
 #shellcheck disable=SC2046
 yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch --resolve --disablerepo=*  --enablerepo=kubernetes,rhel-8-for-x86_64-baseos-eus-rpms,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms --disablerepo=appstream-centos $(< packages.txt)
 rm packages.txt reqs.txt
-curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm
 createrepo -v .
 chown -R 1000:1000 repodata/
 yum install -y modulemd-tools

bundles/redhat8.6/bundle.sh.gotmpl

@@ -3,7 +3,7 @@ set -euo pipefail
 
 # When yum operates on multiple packages, it does not, by default, return an error if a subset
 # of packages is not found. This makes yum return an error.
-echo skip_missing_names_on_install=False >> /etc/yum.conf
+echo skip_missing_names_on_install=False >>/etc/yum.conf
 
 sed -i 's/\(def in_container():\)/\1\n    return False/g' /usr/lib64/python*/*-packages/rhsm/config.py
 FOUND=false
@@ -13,57 +13,94 @@ RHSM_ACTIVATION_KEY=${RHSM_ACTIVATION_KEY:-""}
 RHSM_USER=${RHSM_USER:-""}
 RHSM_PASS=${RHSM_PASS:-""}
 
-if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then
-  subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force
-  FOUND=true
-fi
+SKIP_SUBSCRIPTION_MANAGER=${SKIP_SUBSCRIPTION_MANAGER:-""}
+SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""}
+ENABLED_REPOS=${ENABLED_REPOS:-""}
 
-if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then
-  subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force
-  FOUND=true
-fi
+if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then
+  if [[ -n "${SATELLITE_SERVER_URL}" ]]; then
+    rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm"
+  fi
 
-if [[ ${FOUND} == false ]]; then
-  echo "You must use subscription manager to fetch packages for redhat"
-  exit 1
-fi
+  if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then
+    subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force
+    FOUND=true
+  fi
+
+  if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then
+    subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force
+    FOUND=true
+  fi
+
+  if [[ ${FOUND} == false ]]; then
+    echo "You must use subscription manager to fetch packages for redhat"
+    exit 1
+  fi
 
-subscription::unregister() {
-  subscription-manager unregister
-}
+  subscription::unregister() {
+    subscription-manager unregister
+  }
 
-subscription::defer_unregister() {
+  subscription::defer_unregister() {
     trap subscription::unregister ABRT
     trap subscription::unregister EXIT
     trap subscription::unregister HUP
     trap subscription::unregister INT
     trap subscription::unregister TERM
     trap subscription::unregister USR1
     trap subscription::unregister USR2
-}
+  }
+
+  subscription-manager release --set=8.6
+  subscription-manager refresh
+  subscription::defer_unregister
+
+  ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms"
+  EUS_REPOS=${EUS_REPOS:-""}
+  if [[ -n "${EUS_REPOS}" ]]; then
+    #disables the standard repositories which should not be enabled when using EUS
+    subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms
+    subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms
+    subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms
+    subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms
+    ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms"
+  else
+    subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms
+    subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms
+    subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms
+  fi
+else
+  echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true."
+fi
+
+if [[ -f /etc/yum.repos.d/user-repos.repo ]]; then
+  USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)"
+  if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then
+      ENABLED_REPOS="${ENABLED_REPOS},${USER_REPOS}"
+  elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then
+      ENABLED_REPOS="${USER_REPOS}"
+  fi
+fi
 
-subscription-manager release --set=8.6
-subscription-manager refresh
-subscription::defer_unregister
-subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms
-subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms
-subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms
-yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools
+yum -y install --disablerepo=* --enablerepo="${ENABLED_REPOS}" gettext yum-utils createrepo dnf-utils modulemd-tools
 yum clean all
 TMP_DIR="$(mktemp -d repodata-XXXX)"
+chmod 777 -R "${TMP_DIR}"
 cp packages.txt "${TMP_DIR}"
 pushd "${TMP_DIR}"
 #shellcheck disable=SC2046,SC2062,SC2063,SC2035
-repoquery --archlist=x86_64,noarch --resolve --requires --recursive  $(< packages.txt) | grep -v *.i686  >> reqs.txt
-sed -i 1d reqs.txt # we need to get rid of the first line
+repoquery --disablerepo=* --enablerepo="${ENABLED_REPOS}" \
+  --archlist=x86_64,noarch --resolve --requires --recursive $(<packages.txt) \
+  | grep -vE '*.i686|^[[:space:]]*$|Unable to read consumer identity|This system is not registered with an entitlement server|Updating Subscription Management repositories' >>reqs.txt
 #shellcheck disable=SC2046
-yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt)
+yumdownloader --disablerepo=* --enablerepo="${ENABLED_REPOS}" \
+  --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch $(<reqs.txt)
 #shellcheck disable=SC2046
-yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch --resolve --disablerepo=*  --enablerepo=kubernetes,rhel-8-for-x86_64-baseos-eus-rpms,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms $(< packages.txt)
+yumdownloader --disablerepo=* --enablerepo="${ENABLED_REPOS}" \
+  --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch \
+  --resolve $(<packages.txt)
 rm packages.txt reqs.txt
-curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm
 createrepo -v .
-chown -R 1000:1000 repodata/
 repo2module . --module-name offline
 createrepo_mod .
 #shellcheck disable=SC1083,SC2035

bundles/redhat8.6/packages.txt.gotmpl

@@ -31,7 +31,9 @@ elfutils-libelf-devel
 libseccomp
 nfs-utils
 iproute-tc
-kernel-headers-4.18.0-372.93.1.el8_6
-kernel-devel-4.18.0-372.93.1.el8_6
 glibc-all-langpacks-2.28
 glibc-devel-2.28
+{{ if .FetchKernelHeaders -}}
+kernel-headers-4.18.0-372.93.1.el8_6
+kernel-devel-4.18.0-372.93.1.el8_6
+{{- end }}

bundles/redhat8.8/bundle.sh.gotmpl

@@ -3,7 +3,7 @@ set -euo pipefail
 
 # When yum operates on multiple packages, it does not, by default, return an error if a subset
 # of packages is not found. This makes yum return an error.
-echo skip_missing_names_on_install=False >> /etc/yum.conf
+echo skip_missing_names_on_install=False >>/etc/yum.conf
 
 sed -i 's/\(def in_container():\)/\1\n    return False/g' /usr/lib64/python*/*-packages/rhsm/config.py
 FOUND=false
@@ -13,57 +13,94 @@ RHSM_ACTIVATION_KEY=${RHSM_ACTIVATION_KEY:-""}
 RHSM_USER=${RHSM_USER:-""}
 RHSM_PASS=${RHSM_PASS:-""}
 
-if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then
-  subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force
-  FOUND=true
-fi
+SKIP_SUBSCRIPTION_MANAGER=${SKIP_SUBSCRIPTION_MANAGER:-""}
+SATELLITE_SERVER_URL=${SATELLITE_SERVER_URL:-""}
+ENABLED_REPOS=${ENABLED_REPOS:-""}
 
-if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then
-  subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force
-  FOUND=true
-fi
+if [[ -z "${SKIP_SUBSCRIPTION_MANAGER}" ]]; then
+  if [[ -n "${SATELLITE_SERVER_URL}" ]]; then
+    rpm -ivh "${SATELLITE_SERVER_URL}/pub/katello-ca-consumer-latest.noarch.rpm"
+  fi
 
-if [[ ${FOUND} == false ]]; then
-  echo "You must use subscription manager to fetch packages for redhat"
-  exit 1
-fi
+  if [[ -n "${RHSM_ORG_ID}" && -n "${RHSM_ACTIVATION_KEY}" ]]; then
+    subscription-manager register --org="${RHSM_ORG_ID}" --activationkey="${RHSM_ACTIVATION_KEY}" --force
+    FOUND=true
+  fi
+
+  if [[ -n "${RHSM_USER}" && -n "${RHSM_PASS}" && ${FOUND} == false ]]; then
+    subscription-manager register --username="${RHSM_USER}" --password="${RHSM_PASS}" --force
+    FOUND=true
+  fi
+
+  if [[ ${FOUND} == false ]]; then
+    echo "You must use subscription manager to fetch packages for redhat"
+    exit 1
+  fi
 
-subscription::unregister() {
-  subscription-manager unregister
-}
+  subscription::unregister() {
+    subscription-manager unregister
+  }
 
-subscription::defer_unregister() {
+  subscription::defer_unregister() {
     trap subscription::unregister ABRT
     trap subscription::unregister EXIT
     trap subscription::unregister HUP
     trap subscription::unregister INT
     trap subscription::unregister TERM
     trap subscription::unregister USR1
     trap subscription::unregister USR2
-}
+  }
+
+  subscription-manager release --set=8.8
+  subscription-manager refresh
+  subscription::defer_unregister
+
+  ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms"
+  EUS_REPOS=${EUS_REPOS:-""}
+  if [[ -n "${EUS_REPOS}" ]]; then
+    #disables the standard repositories which should not be enabled when using EUS
+    subscription-manager repos --disable=rhel-8-for-x86_64-baseos-rpms --disable=rhel-8-for-x86_64-appstream-rpms
+    subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms
+    subscription-manager repos --enable rhel-8-for-x86_64-appstream-eus-rpms
+    subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-eus-rpms
+    ENABLED_REPOS="kubernetes,codeready-builder-for-rhel-8-x86_64-eus-rpms,rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms"
+  else
+    subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms
+    subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms
+    subscription-manager repos --enable rhel-8-for-x86_64-baseos-rpms
+  fi
+else
+  echo "Bypassing subscription-manager and upstream RHEL repositories due to --skip-subscription-manager being set to true."
+fi
+
+if [[ -f /etc/yum.repos.d/user-repos.repo ]]; then
+  USER_REPOS="$(awk -F '[][]' '/^\[.*\]/ {print $2}' /etc/yum.repos.d/user-repos.repo | paste -sd, -)"
+  if [[ -n "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then
+      ENABLED_REPOS="${ENABLED_REPOS},${USER_REPOS}"
+  elif [[ -z "${ENABLED_REPOS}" ]] && [[ -n "${USER_REPOS}" ]]; then
+      ENABLED_REPOS="${USER_REPOS}"
+  fi
+fi
 
-subscription-manager release --set=8.8
-subscription-manager refresh
-subscription::defer_unregister
-subscription-manager repos --enable rhel-8-for-x86_64-baseos-eus-rpms
-subscription-manager repos --enable codeready-builder-for-rhel-8-x86_64-rpms
-subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms
-yum -y install gettext yum-utils createrepo dnf-utils modulemd-tools
+yum -y install --disablerepo=* --enablerepo="${ENABLED_REPOS}" gettext yum-utils createrepo dnf-utils modulemd-tools
 yum clean all
 TMP_DIR="$(mktemp -d repodata-XXXX)"
+chmod 777 -R "${TMP_DIR}"
 cp packages.txt "${TMP_DIR}"
 pushd "${TMP_DIR}"
 #shellcheck disable=SC2046,SC2062,SC2063,SC2035
-repoquery --archlist=x86_64,noarch --resolve --requires --recursive  $(< packages.txt) | grep -v *.i686  >> reqs.txt
-sed -i 1d reqs.txt # we need to get rid of the first line
+repoquery --disablerepo=* --enablerepo="${ENABLED_REPOS}" \
+  --archlist=x86_64,noarch --resolve --requires --recursive $(<packages.txt) \
+  | grep -vE '*.i686|^[[:space:]]*$|Unable to read consumer identity|This system is not registered with an entitlement server|Updating Subscription Management repositories' >>reqs.txt
 #shellcheck disable=SC2046
-yumdownloader --archlist=x86_64,noarch --setopt=skip_missing_names_on_install=False -x \*i686 $(< reqs.txt)
+yumdownloader --disablerepo=* --enablerepo="${ENABLED_REPOS}" \
+  --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch $(<reqs.txt)
 #shellcheck disable=SC2046
-yumdownloader --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch --resolve --disablerepo=*  --enablerepo=kubernetes,rhel-8-for-x86_64-baseos-eus-rpms,codeready-builder-for-rhel-8-x86_64-rpms,rhel-8-for-x86_64-appstream-rpms $(< packages.txt)
+yumdownloader --disablerepo=* --enablerepo="${ENABLED_REPOS}" \
+  --setopt=skip_missing_names_on_install=False -x \*i686 --archlist=x86_64,noarch \
+  --resolve $(<packages.txt)
 rm packages.txt reqs.txt
-curl https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm -o amazon-ssm-agent.rpm
 createrepo -v .
-chown -R 1000:1000 repodata/
 repo2module . --module-name offline
 createrepo_mod .
 #shellcheck disable=SC1083,SC2035

bundles/redhat8.8/packages.txt.gotmpl

@@ -35,3 +35,7 @@ kernel-devel
 gssproxy
 libverto-module-base
 libverto
+{{ if .FetchKernelHeaders -}}
+kernel-headers-4.18.0-477.58.1.el8_8
+kernel-devel-4.18.0-477.58.1.el8_8
+{{- end }}