Merge pull request #199 from spacetelescope/release/v0.4.41

Release/v0.4.41

changelog.md

@@ -1,2 +1,6 @@
-- default base docker image set to CALDP_20230613_CAL_final
-- default crds update to hst_1077.pmap
+- Created the calcloud-ami-rotation CodeBuild project to take over running
+  the biweekly AMI rotation script previously run by the
+  calcloud-env-AmiRotation Lambda
+- Replaced the deprecated sklearn==0.0 package with scikit-learn==1.0.2
+- default base docker image set to CALDP_cosandpin_CAL_rc1
+- default crds update to hst_1089.pmap

iac/codebuild/Dockerfile

@@ -0,0 +1,63 @@
+# Copyright 2020-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Amazon Software License (the "License"). You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#    http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+FROM public.ecr.aws/amazonlinux/amazonlinux:2 
+
+ENV AWS_DEFAULT_REGION="us-east-1"
+ARG aws_env
+ARG CALCLOUD_VER
+
+# Install git, SSH, and other utilities
+RUN set -ex \
+    && yum update -y --security \
+    && yum install -y  gcc libpng-devel libjpeg-devel unzip yum-utils \ 
+    && yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo \
+    && yum install terraform-1.0.11 -y \
+    && yum install git -y  \
+    && yum install tar wget python3 which -y 
+
+RUN mkdir -p /etc/ssl/certs && \
+    mkdir -p /etc/pki/ca-trust/source/anchors
+
+COPY tls-ca-bundle.pem /etc/pki/ca-trust/source/anchors/stsci-tls-ca-bundle.pem
+
+COPY tls-ca-bundle.pem /etc/ssl/certs/stsci-tls-ca-bundle.pem
+
+RUN update-ca-trust
+
+# These should only happen once
+ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/stsci-tls-ca-bundle.pem
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/stsci-tls-ca-bundle.pem
+ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/stsci-tls-ca-bundle.pem
+
+#Update Certificates
+#RUN yum update ca-certificates -y
+
+RUN curl -L https://rpm.nodesource.com/setup_16.x | bash - && \
+    yum install -y nodejs
+
+ENV NODE_16_VERSION="16.15.1"
+
+RUN npm install n -g
+
+RUN npm config set registry http://registry.npmjs.org/ && \
+    npm install -g [email protected]
+
+RUN pip3 install awscli
+
+COPY calcloud_checkout.sh /root/
+
+RUN chmod +x /root/calcloud_checkout.sh && /root/calcloud_checkout.sh
+
+COPY log_listener.py /root/
+
+ENTRYPOINT [ "/bin/bash", "-l", "-c" ]
+

iac/codebuild/calcloud_checkout.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+calcloud_ver_ssm=$(aws ssm get-parameter --name /tf/env/awsysver-$aws_env --output text | cut -f 7)
+calcloud_ver=${CALCLOUD_VER:-$calcloud_ver_ssm}
+
+# calcloud checkout, need right tag
+mkdir -p /opt/calcloud/ami_rotate && cd /opt/calcloud/ami_rotate
+git clone https://github.com/spacetelescope/calcloud.git
+cd calcloud
+git remote set-url origin DISABLED --push
+git fetch
+git fetch --all --tags && git checkout tags/$calcloud_ver && cd ..
+git_exit_status=$?
+if [[ $git_exit_status -ne 0 ]]; then
+    # try without the v
+    cd calcloud && git fetch --all --tags && git checkout tags/$calcloud_ver && cd ..
+    git_exit_status=$?
+fi
+if [[ $git_exit_status -ne 0 ]]; then
+    echo "could not checkout $calcloud_ver; exiting"
+    exit 1
+fi

iac/codebuild/copy-cert

@@ -0,0 +1,48 @@
+#! /usr/bin/env python3
+
+# This script copies the most recent SSL cert from the CI-node 
+# (must be run on the CI-node with latest AMI)
+# The cert is needed for Docker builds to transit the STScI packet inspection firewall on AWS.   
+
+import sys
+import subprocess
+import os
+
+
+def run(cmd, cwd=".", timeout=100):
+    """Run subprocess `cmd` in dir `cwd` failing if not completed within `timeout` seconds
+    of if `cmd` returns a non-zero exit status.
+
+    Returns both stdout+stderr from `cmd`.  (untested, verify manually if in doubt)
+    """
+    print(cmd)
+    result = subprocess.run(
+        cmd.split(),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        check=True,
+        cwd=cwd,
+        timeout=timeout,
+    )  # maybe succeeds
+    return result.stdout
+
+
+def get_cert(cert_path, local_path):
+    run(f"cp {cert_path} {local_path}")  
+    #os.system(f"cp {cert_path} {local_path}")  # Using os module temporarily
+    print(f"Copied {cert_path} to {local_path}.")
+
+
+def main(cert_path, local_cert_path):
+    try:
+        get_cert(cert_path, local_cert_path)
+    except:  
+        print("Skipping update of tls-ca-bundle.pem needed to build on science platforms.")
+        return 0
+    print("Cert update complete.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "tls-ca-bundle.pem"))

iac/codebuild/log_listener.py

@@ -0,0 +1,63 @@
+import boto3
+import time
+import sys
+from datetime import datetime
+
+client = boto3.client('logs')
+
+log_group = sys.argv[1]
+log_stream = sys.argv[2]
+
+pushed_lines = []
+
+while True:
+    response = client.describe_log_streams(
+        logGroupName=log_group,
+        logStreamNamePrefix=log_stream
+    )
+    try:
+        nextToken = response['logStreams'][0]['uploadSequenceToken']
+    except KeyError:
+        nextToken = None
+    with open("/var/log/user-data.log", 'r') as f:
+        lines = f.readlines()
+        new_lines = []
+        for line in lines:
+            if line in pushed_lines:
+                continue
+            timestamp = line.split(" ")[0].strip()
+            try:
+                dt = datetime.strptime(timestamp, "%Y-%m-%dT%H.%M.%S%z")
+                dt_ts = int(dt.timestamp())*1000 #milliseconds
+                if nextToken is None:
+                    response = client.put_log_events(
+                        logGroupName = log_group,
+                        logStreamName = log_stream,
+                        logEvents = [
+                            {
+                                'timestamp': dt_ts,
+                                'message': line
+                            }
+                        ]
+                    )
+                    nextToken = response['nextSequenceToken']
+                else:
+                    response = client.put_log_events(
+                        logGroupName = log_group,
+                        logStreamName = log_stream,
+                        logEvents = [
+                            {
+                                'timestamp': dt_ts,
+                                'message': line
+                            }
+                        ],
+                        sequenceToken=nextToken
+                    )
+                    nextToken = response['nextSequenceToken']
+            except Exception as e:
+                # print(e)
+                continue
+
+            pushed_lines.append(line)
+            time.sleep(0.21) #AWS throttles at 5 calls/second
+    time.sleep(2)