Update ssl cert workflow

iac/codebuild/Dockerfile

@@ -27,21 +27,15 @@ RUN set -ex \
 RUN mkdir -p /etc/ssl/certs && \
     mkdir -p /etc/pki/ca-trust/source/anchors
 
-COPY tls-ca-bundle.pem /etc/pki/ca-trust/source/anchors/stsci-tls-ca-bundle.pem
-
-COPY tls-ca-bundle.pem /etc/ssl/certs/stsci-tls-ca-bundle.pem
+COPY STSCICA.crt /etc/ssl/certs/STSCICA.crt
+COPY STSCICA.crt /etc/pki/ca-trust/source/anchors/STSCICA.crt
 
 RUN update-ca-trust
 
 # These should only happen once
-ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/stsci-tls-ca-bundle.pem
-ENV CURL_CA_BUNDLE=/etc/ssl/certs/stsci-tls-ca-bundle.pem
-ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/stsci-tls-ca-bundle.pem
-
-#RUN curl -L https://rpm.nodesource.com/setup_16.x | bash - && \
-#    yum install -y nodejs
-
-#ENV NODE_16_VERSION="16.15.1"
+ENV REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+ENV CURL_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+ENV NODE_EXTRA_CA_CERTS=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
 
 RUN yum install https://rpm.nodesource.com/pub_16.x/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y
 RUN yum install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1

iac/codebuild/copy-cert

@@ -30,15 +30,14 @@ def run(cmd, cwd=".", timeout=100):
 
 def get_cert(cert_path, local_path):
     run(f"cp {cert_path} {local_path}")  
-    #os.system(f"cp {cert_path} {local_path}")  # Using os module temporarily
     print(f"Copied {cert_path} to {local_path}.")
 
 
 def main(cert_path, local_cert_path):
     try:
         get_cert(cert_path, local_cert_path)
     except:  
-        print("Skipping update of tls-ca-bundle.pem needed to build on science platforms.")
+        print("Skipping update of tls-ca-bundle.pem.")
         return 0
     print("Cert update complete.")
     return 0

lambda/JobPredict/Dockerfile

@@ -9,7 +9,11 @@ ENV REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
 ENV CURL_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
 
 RUN mkdir -p /etc/ssl/certs && mkdir -p /etc/pki/ca-trust/extracted/pem
-COPY certs/tls-ca-bundle.pem /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+COPY STSCICA.crt /etc/ssl/certs/STSCICA.crt
+COPY STSCICA.crt /etc/pki/ca-trust/source/anchors/STSCICA.crt
+
+RUN update-ca-trust
+
 RUN mv /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-bundle.crt.org && ln -s /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem  /etc/ssl/certs/ca-bundle.crt && ln -s /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /etc/ssl/certs/ca-certificates.crt && mkdir -p /etc/pki/ca-trust/extracted/openssl
 
 #log4j vuln patch

modeling/Dockerfile

@@ -1,12 +1,15 @@
 FROM library/ubuntu:20.04
 #SSL/TLS cert setup for STScI AWS firewalling
 USER root
-ENV REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
-ENV CURL_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
-RUN mkdir -p /etc/ssl/certs/ && mkdir -p /etc/pki/ca-trust/extracted/pem/
-COPY certs/tls-ca-bundle.pem /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
-RUN ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /etc/ssl/certs/ca-bundle.crt \
-&& ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /etc/ssl/certs/ca-certificates.crt \
+RUN mkdir -p /etc/ssl/certs/ && mkdir -p /usr/local/share/ca-certificates/ && mkdir -p /etc/pki/ca-trust/extracted/pem/
+ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+COPY STSCICA.crt /etc/ssl/certs/STSCICA.crt
+COPY STSCICA.crt /usr/local/share/ca-certificates/STSCICA.crt
+RUN apt-get update && \
+    apt-get install -y ca-certificates 
+RUN update-ca-certificates
+RUN ln -sf /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-bundle.crt \
 && mkdir -p /etc/pki/ca-trust/extracted/openssl
 
 RUN DEBIAN_FRONTEND=noninteractive && apt update && apt upgrade -y && apt install -y \

terraform/deploy_ami_rotation_codebuild_image.sh

@@ -9,7 +9,8 @@ source deploy_checkout_repos.sh
 cd ${CALCLOUD_BUILD_DIR}/iac/codebuild
 pwd
 
-./copy-cert # copy the cert from CI node AMI and replace the cert in current dir
+#./copy-cert # copy the cert from CI node AMI and replace the cert in current dir
+source hst_admin_role_shim.sh cert-update
 
 set -o pipefail && docker build -f Dockerfile -t ${AMIROTATION_DOCKER_IMAGE_UNSCANNED} --build-arg aws_env="${aws_env}" --build-arg CALCLOUD_VER="${CALCLOUD_VER}" .
 amirotation_docker_build_status=$?

terraform/deploy_docker_builds.sh

@@ -21,7 +21,7 @@ fi
 
 # naming is confusing here but "modeling" directory plus "training" image is correct
 cd ${CALCLOUD_BUILD_DIR}/modeling
-cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem certs/tls-ca-bundle.pem # copy the cert from CI node AMI
+source hst_admin_role_shim.sh cert-update
 set -o pipefail && docker build -f Dockerfile -t ${TRAINING_DOCKER_IMAGE} .
 training_docker_build_status=$?
 if [[ $training_docker_build_status -ne 0 ]]; then
@@ -31,7 +31,7 @@ fi
 
 # jobPredict lambda env
 cd ${CALCLOUD_BUILD_DIR}/lambda/JobPredict
-cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem certs/tls-ca-bundle.pem # copy the cert from CI node AMI
+source hst_admin_role_shim.sh cert-update
 set -o pipefail && docker build -f Dockerfile -t ${PREDICT_DOCKER_IMAGE} .
 model_docker_build_status=$?
 if [[ $model_docker_build_status -ne 0 ]]; then
@@ -41,7 +41,7 @@ fi
 
 # caldp image
 cd ${CALDP_BUILD_DIR}
-cp /etc/ssl/certs/ca-bundle.crt tls-ca-bundle.pem # copy the cert from CI node AMI
+source hst_admin_role_shim.sh cert-update
 set -o pipefail && docker build -f Dockerfile -t ${CALDP_DOCKER_IMAGE} --build-arg CAL_BASE_IMAGE="${CAL_BASE_IMAGE}"  .
 caldp_docker_build_status=$?
 if [[ $caldp_docker_build_status -ne 0 ]]; then

terraform/deploy_vars.sh

@@ -63,4 +63,4 @@ CSYS_VER=${CAL_BASE_IMAGE##*:}
 CSYS_VER=`echo $CSYS_VER | cut -f1,2 -d'_'` #split by underscores, keep the first two
 export CSYS_VER=`echo $CSYS_VER | awk '{print tolower($0)}'`
 
-# get repo_url here for the central ecr repo
+export PATH=`pwd`/tools:${PATH}

terraform/tools/cert-list

@@ -0,0 +1,74 @@
+#! /usr/bin/env python3
+
+"""This script is used to dump out cert subjects and issuers in order
+to follow the chain of certs from JH to a root authority.
+
+In particular, the EFS Quota software uses Python requests and a
+carefully constructed intermediate cert chain was required to keep
+requests happy.
+
+It works like on a single JH cert:
+
+$ cert-list roman.dev.pem
+subject=C = US, ST = Maryland, O = Space Telescope Science Institute, CN = dev.roman.science.stsci.edu
+issuer=C = US, ST = MI, L = Ann Arbor, O = Internet2, OU = InCommon, CN = InCommon RSA Server CA
+
+And EFS quotas includes this chain to carry trust forward all the way to the root:
+
+$ cert-list jh-intermediates.pem
+subject=C = US, ST = MI, L = Ann Arbor, O = Internet2, OU = InCommon, CN = InCommon RSA Server CA
+issuer=C = US, ST = New Jersey, L = Jersey City, O = The USERTRUST Network, CN = USERTrust RSA Certification Authority
+
+subject=C = US, ST = New Jersey, L = Jersey City, O = The USERTRUST Network, CN = USERTrust RSA Certification Authority
+issuer=C = GB, ST = Greater Manchester, L = Salford, O = Comodo CA Limited, CN = AAA Certificate Services
+
+subject=C = GB, ST = Greater Manchester, L = Salford, O = Comodo CA Limited, CN = AAA Certificate Services
+issuer=C = GB, ST = Greater Manchester, L = Salford, O = Comodo CA Limited, CN = AAA Certificate Services
+"""
+
+import sys
+import subprocess
+import tempfile
+
+
+def run(cmd, cwd=".", timeout=10):
+    """Run subprocess `cmd` in dir `cwd` failing if not completed within `timeout` seconds
+    of if `cmd` returns a non-zero exit status.
+
+    Returns both stdout+stderr from `cmd`.  (untested, verify manually if in doubt)
+    """
+    result = subprocess.run(
+        cmd.split(),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        check=True,
+        cwd=cwd,
+        timeout=timeout,
+    )  # maybe succeeds
+    return result.stdout
+
+
+def decode(lines):
+    with tempfile.NamedTemporaryFile(mode='w+', delete=False) as tmpf:
+        tmpf.write("".join(lines))
+    return run(f"openssl x509 -subject -issuer --dates -subject_hash -issuer_hash -serial --nocert -in {tmpf.name}")
+
+
+def main():
+    with open(sys.argv[1]) as chain:
+        if not chain.readlines():
+            sys.exit(0)
+    with open(sys.argv[1]) as chain:
+        for line in chain.readlines():
+            if not line.strip() or line.startswith("#"):
+                continue
+            if "---BEGIN" in line:
+                lines = []
+            lines.append(line)
+            if "---END" in line:
+                print(decode(lines))
+
+
+if __name__ == "__main__":
+    main()

terraform/tools/cert-update

@@ -0,0 +1,76 @@
+#! /usr/bin/env python3
+
+# This script downloads and cleans the SSL cert needed for Docker builds
+# to transit the STScI packet inspection firewall on AWS.   Originally
+# shellscript,  it was easier to switch to Python than deal with variations
+# in "sed" behavior between OS-X and Linux.
+
+import sys
+import subprocess
+
+
+def run(cmd, cwd=".", timeout=10):
+    """Run subprocess `cmd` in dir `cwd` failing if not completed within `timeout` seconds
+    of if `cmd` returns a non-zero exit status.
+
+    Returns both stdout+stderr from `cmd`.  (untested, verify manually if in doubt)
+    """
+    result = subprocess.run(
+        cmd.split(),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        check=True,
+        cwd=cwd,
+        timeout=timeout,
+    )  # maybe succeeds
+    return result.stdout
+
+
+def get_cert(s3_path, local_path):
+    run(f"aws s3 cp {s3_path} {local_path}")
+    print(f"Copied {s3_path} to {local_path}.")
+    # run(f"echo >{local_path}")
+    # print(f"Nullified cert at {local_path}.")
+
+
+def clean_cert(fpath):
+    print(f"Conditioning cert file {fpath}")
+    cleaned = []
+    with open(fpath) as cfile:
+        for line in cfile.readlines():
+            line = line.strip()
+            if line.startswith("#") or not line:
+                continue
+            cleaned.append(line)
+    with open(fpath, "w+") as cfile:
+        cfile.write("\n".join(cleaned))
+
+
+def check_cert(local_cert_path):
+    print("Checking cert file.")
+    run(f"cert-list {local_cert_path}")
+
+
+def main(s3_cert_path, local_cert_path):
+    try:
+        get_cert(s3_cert_path, local_cert_path)
+    except Exception as exc:  # noqa: E722
+        print(f"Failed to obtain updated STSCICA.crt from {s3_cert_path}:", repr(exc), str(exc))
+        return 1
+    try:
+        clean_cert(local_cert_path)
+    except Exception as exc:  # noqa: E722
+        print(f"Conditioning cert {local_cert_path} failed:", repr(exc), str(exc))
+        return 1
+    try:
+        check_cert(local_cert_path)
+    except Exception as exc:  # noqa: E722
+        print(f"Conditioned cert {local_cert_path} was not valid:", repr(exc), str(exc))
+        return 1
+    print("Cert update complete.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main("s3://stsci-ami-installs/STSCICA.crt", "STSCICA.crt"))

terraform/tools/hst_admin_role_shim.sh

@@ -0,0 +1,44 @@
+#/bin/bash
+# This script assumes the hst_reprocessing_admin_role, runs a given command using that role, and then switches back to original role
+
+# Set region
+export AWS_DEFAULT_REGION="us-east-1"
+
+# Role to assume
+ACCOUNT_ID=`aws sts get-caller-identity --output=text | awk '{ print $1 }'`
+HST_ADMIN_ARN="arn:aws:iam::${ACCOUNT_ID}:role/hst_reprocessing_admin_role"
+
+# Grab parameters
+COMMAND_TO_RUN=$*
+
+# Save current AWS credentials
+CURRENT_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+CURRENT_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+CURRENT_AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN
+
+#Assume role, run command, and switch back
+printf "\n Assuming role..."
+if CREDENTIALS=`aws sts assume-role --role-arn $HST_ADMIN_ARN --role-session-name temp_admin_session --duration-seconds 3599` ; then
+
+        export AWS_ACCESS_KEY_ID=`echo ${CREDENTIALS} | python -c "import sys, json, os; temp=json.load(sys.stdin)['Credentials']['AccessKeyId'];print(temp)"`
+        export AWS_SECRET_ACCESS_KEY=`echo ${CREDENTIALS} | python -c "import sys, json, os; temp=json.load(sys.stdin)['Credentials']['SecretAccessKey'];print(temp)"`
+        export AWS_SESSION_TOKEN=`echo ${CREDENTIALS} | python -c "import sys, json, os; temp=json.load(sys.stdin)['Credentials']['SessionToken'];print(temp)"`
+
+        printf "\n Role assumed:"
+        aws sts get-caller-identity
+
+        printf "\n Running command\n"
+        $COMMAND_TO_RUN
+
+        printf "\n Switching back to original role"
+        export AWS_ACCESS_KEY_ID=$CURRENT_AWS_ACCESS_KEY_ID
+        export AWS_SECRET_ACCESS_KEY=$CURRENT_AWS_SECRET_ACCESS_KEY
+        export AWS_SESSION_TOKEN=$CURRENT_AWS_SESSION_TOKEN
+
+        printf "\n Role switched back:"
+        aws sts get-caller-identity
+
+else
+        printf "\n==========================\n\n - Error assuming role. Aborting execution\n\n"
+        exit 1
+fi