Merge pull request #202 from spacetelescope/release/v0.4.42

Release/v0.4.42

.github/workflows/checks.yml

@@ -7,10 +7,10 @@ jobs:
     name: run flake8
     runs-on: ubuntu-20.04
     steps:
-      - name: set up python 3.7
+      - name: set up python 3.11
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.11
 
       - name: checkout code
         uses: actions/checkout@v2
@@ -25,10 +25,10 @@ jobs:
     name: run black
     runs-on: ubuntu-20.04
     steps:
-      - name: set up python 3.7
+      - name: set up python 3.11
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.11
 
       - name: checkout code
         uses: actions/checkout@v2
@@ -43,10 +43,10 @@ jobs:
     name: run bandit
     runs-on: ubuntu-20.04
     steps:
-      - name: set up python 3.7
+      - name: set up python 3.11
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.11
 
       - name: checkout code
         uses: actions/checkout@v2

.github/workflows/pytest.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - name: set up python 3.7.12
+      - name: set up python 3.11
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7.12
+          python-version: 3.11
 
       - name: checkout code
         uses: actions/checkout@v2
@@ -45,10 +45,10 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - name: set up python 3.7.12
+      - name: set up python 3.11
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7.12
+          python-version: 3.11
 
       - name: checkout code
         uses: actions/checkout@v2

calcloud/model_ingest.py

@@ -224,7 +224,7 @@ def convert_target_data(self):
             clock += sum(x * int(t) for x, t in zip([1, 60, 3600], clocktime))
             print(clock)
         for memstr in self.target_data["memory"]:
-            kb += np.float(memstr)
+            kb += float(memstr)
             print(kb)
         targets["wallclock"] = clock + 1
         targets["memory"] = kb / (10**6)

changelog.md

@@ -1,6 +1,3 @@
-- Created the calcloud-ami-rotation CodeBuild project to take over running
-  the biweekly AMI rotation script previously run by the
-  calcloud-env-AmiRotation Lambda
-- Replaced the deprecated sklearn==0.0 package with scikit-learn==1.0.2
-- default base docker image set to CALDP_cosandpin_CAL_rc1
-- default crds update to hst_1089.pmap
+- default base docker image set to CALDP_20231201_CAL_final
+- default crds update to hst_1123.pmap
+- updated lambda runtime from python 3.7 to python 3.11

iac/codebuild/Dockerfile

@@ -38,13 +38,13 @@ ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/stsci-tls-ca-bundle.pem
 ENV CURL_CA_BUNDLE=/etc/ssl/certs/stsci-tls-ca-bundle.pem
 ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/stsci-tls-ca-bundle.pem
 
-#Update Certificates
-#RUN yum update ca-certificates -y
+#RUN curl -L https://rpm.nodesource.com/setup_16.x | bash - && \
+#    yum install -y nodejs
 
-RUN curl -L https://rpm.nodesource.com/setup_16.x | bash - && \
-    yum install -y nodejs
+#ENV NODE_16_VERSION="16.15.1"
 
-ENV NODE_16_VERSION="16.15.1"
+RUN yum install https://rpm.nodesource.com/pub_16.x/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y
+RUN yum install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1
 
 RUN npm install n -g
 

lambda/JobPredict/Dockerfile

@@ -1,5 +1,5 @@
 #FROM public.ecr.aws/lambda/python:3.7
-FROM amazon/aws-lambda-python:3.7
+FROM amazon/aws-lambda-python:3.11
 COPY requirements.txt predict_handler.py ./
 # SSL/TLS cert setup for STScI AWS firewalling
 USER root
@@ -15,6 +15,7 @@ RUN mv /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-bundle.crt.org && ln -s /e
 #log4j vuln patch
 RUN yum update -y java-1.8.0-openjdk
 
-RUN python3.7 -m pip install --upgrade pip && python3.7 -m pip install -r requirements.txt && python3.7 -m pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.6.2-cp37-cp37m-manylinux2010_x86_64.whl
+#RUN python3.7 -m pip install --upgrade pip && python3.7 -m pip install -r requirements.txt && python3.7 -m pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.6.2-cp37-cp37m-manylinux2010_x86_64.whl
+RUN python3.11 -m pip install --upgrade pip && python3.11 -m pip install -r requirements.txt && python3.11 -m pip install tensorflow-cpu==2.12.1
 ADD  models ./models/
 CMD ["predict_handler.lambda_handler"]

lambda/JobPredict/requirements.txt

@@ -1,3 +1,3 @@
-scikit-learn==1.0.2
-boto3==1.17.25
-protobuf==3.20.1
+scikit-learn==1.3.0
+boto3==1.28.56
+protobuf==4.24.3

modeling/Dockerfile

@@ -25,12 +25,12 @@ unzip \
 zlib1g-dev 
 
 RUN DEBIAN_FRONTEND=noninteractive \
-&& curl -o Python-3.9.5.tgz https://www.python.org/ftp/python/3.9.5/Python-3.9.5.tgz \
-&& tar -xf Python-3.9.5.tgz \
-&& cd Python-3.9.5 \
+&& curl -o Python-3.11.5.tgz https://www.python.org/ftp/python/3.11.5/Python-3.11.5.tgz \
+&& tar -xf Python-3.11.5.tgz \
+&& cd Python-3.11.5 \
 && ./configure --enable-optimizations \
 && make altinstall \
-&& update-alternatives --install /usr/local/bin/python python /usr/local/bin/python3.9 10 
+&& update-alternatives --install /usr/local/bin/python python /usr/local/bin/python3.11 10 
 
 RUN DEBIAN_FRONTEND=noninteractive \
 && cd ../ && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \

modeling/requirements.txt

@@ -1,5 +1,5 @@
-tensorflow==2.6.2
-scikit-learn==1.0.2
-boto3==1.17.79
-pandas==1.2.4
-protobuf==3.20.1
+tensorflow==2.12.1
+scikit-learn==1.3.0
+boto3==1.28.56
+pandas==2.1.0
+protobuf==4.24.3

notes/lambda.function.configuration.json

@@ -1,7 +1,7 @@
 {
     "FunctionName": "calcloud-hst-trigger",
     "FunctionArn": "arn:aws:lambda:us-east-1:162808325377:function:calcloud-hst-trigger",
-    "Runtime": "python3.7",
+    "Runtime": "python3.11",
     "Role": "arn:aws:iam::162808325377:role/calcloud-hst-trigger-lambda-role",
     "Handler": "lambda_function.lambda_handler",
     "CodeSize": 1435,

notes/lambda.function.json

@@ -2,7 +2,7 @@
     "Configuration": {
         "FunctionName": "calcloud-hst-trigger",
         "FunctionArn": "arn:aws:lambda:us-east-1:162808325377:function:calcloud-hst-trigger",
-        "Runtime": "python3.7",
+        "Runtime": "python3.11",
         "Role": "arn:aws:iam::162808325377:role/calcloud-hst-trigger-lambda-role",
         "Handler": "lambda_function.lambda_handler",
         "CodeSize": 1435,

setup.cfg

@@ -13,7 +13,7 @@ github_project = spacetelescope/calcloud
 
 [options]
 packages = find:
-python_requires = ~=3.7.0
+python_requires = ~=3.11.0
 setup_requires =
     setuptools >=41.0.1
 install_requires =

terraform/ami-rotation.tf

@@ -57,13 +57,13 @@ resource "aws_launch_template" "ami_rotation" {
 
 module "calcloud_env_amiRotation" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-env-AmiRotation${local.environment}"
   description   = "spawns an ec2 bi-weekly which rotates the ami for batch"
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "ami_rotation.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 60
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days

terraform/batch.tf

@@ -8,27 +8,27 @@ terraform {
   required_providers {
     aws = {
       source = "hashicorp/aws"
-      version = "~> 3.65.0"
+      version = "~> 5.17.0"
     }
     hashicorp-null = {
       source = "hashicorp/null"
-      version = "~> 3.1.0"
+      version = "~> 3.2.1"
     }
     hashicorp-external = {
       source = "hashicorp/external"
-      version = "~> 2.1.0"
+      version = "~> 2.3.1"
     }
     hashicorp-local = {
       source = "hashicorp/local"
-      version = "~> 2.1.0"
+      version = "~> 2.4.0"
     }
     hashicorp-random = {
       source = "hashicorp/random"
-      version = "~> 3.1.0"
+      version = "~> 3.5.1"
     }
     docker = {
       source = "kreuzwerker/docker"
-      version = "~> 2.15.0"
+      version = "~> 3.0.2"
     }
   }
 }

terraform/deploy_vars.sh

@@ -1,7 +1,7 @@
 #! /bin/bash -xu
-export CALCLOUD_VER="v0.4.41"
-export CALDP_VER="v0.2.23"
-export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20230810_CAL_final"
+export CALCLOUD_VER="v0.4.42"
+export CALDP_VER="v0.2.24"
+export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20231201_CAL_final"
 export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} `
 
 export COMMON_IMAGE_TAG="CALCLOUD_${CALCLOUD_VER}-CALDP_${CALDP_VER}-BASE_${BASE_IMAGE_TAG}"

terraform/lambda_batch_events.tf

@@ -1,12 +1,12 @@
 module "calcloud_lambda_batchEvents" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-job-events${local.environment}"
   description   = "listens for Batch failure events from cloudWatch event rule"
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "batch_event_handler.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 900
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days

terraform/lambda_blackboard.tf

@@ -1,12 +1,12 @@
 module "calcloud_lambda_blackboard" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-job-blackboard${local.environment}"
   description   = "scrapes the Batch console for job metadata and posts to S3 bucket for on-premise poller"
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "scrape_batch.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 300
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days

terraform/lambda_broadcast.tf

@@ -1,12 +1,12 @@
 module "calcloud_lambda_broadcast" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-broadcast${local.environment}"
   description   = "Broadcasts the specified message type across a list of job_ids or ippppssoots."
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "broadcast_handler.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 300
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days

terraform/lambda_job_clean.tf

@@ -1,12 +1,12 @@
 module "calcloud_lambda_cleanJob" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-job-clean${local.environment}"
   description   = "accepts messages from s3 event and cleans either individual jobs by dataset, or all active jobs"
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "clean_handler.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 900
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days

terraform/lambda_job_delete.tf

@@ -1,12 +1,12 @@
 module "calcloud_lambda_deleteJob" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-job-delete${local.environment}"
   description   = "accepts messages from s3 event and deletes either individual jobs by dataset, or all active jobs"
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "delete_handler.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 900
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days

terraform/lambda_job_predict.tf

@@ -1,6 +1,6 @@
 module "lambda_function_container_image" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
   function_name = "calcloud-job-predict${local.environment}"
   description   = "pretrained neural networks for predicting job resource requirements (memory bin and max execution time)"
 

terraform/lambda_job_rescue.tf

@@ -1,12 +1,12 @@
 module "calcloud_lambda_rescueJob" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-job-rescue${local.environment}"
   description   = "Rescues the specified dataset (must be in error state) by deleting all outputs and messages and re-placing."
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "rescue_handler.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 900
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days

terraform/lambda_job_submit.tf

@@ -1,13 +1,13 @@
 module "calcloud_lambda_submit" {
   source = "terraform-aws-modules/lambda/aws"
   # https://github.com/hashicorp/terraform/issues/17211
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-job-submit${local.environment}"
   description   = "looks for placed-dataset messages and submits jobs to Batch"
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "s3_trigger_handler.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 15*60   # see also SUBMIT_TIMEOUT below;  this is the AWS timeout, calcloud error handling may not occur
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days

terraform/lambda_model_ingest.tf

@@ -16,13 +16,13 @@ resource "aws_dynamodb_table" "calcloud_model_db" {
 
 module "lambda_model_ingest" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-model-ingest${local.environment}"
   lambda_role = nonsensitive(data.aws_ssm_parameter.model_ingest_role.value)
   description   = "looks for processed-ipppssoot.trigger messages, scrapes and uploads completed job data to DynamoDB"
   handler       = "lambda_scrape.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 180
   memory_size   = 256

terraform/lambda_refresh_cache.tf

@@ -1,12 +1,12 @@
 module "calcloud_lambda_refresh_cache_submit" {
   source = "terraform-aws-modules/lambda/aws"
-  version = "~> 2.26.0"
+  version = "~> 6.0.0"
 
   function_name = "calcloud-fileshare-refresh_cache_submit${local.environment}"
   description   = "submits refresh cache operations"
   # the path is relative to the path inside the lambda env, not in the local filesystem.
   handler       = "refresh_cache_submit.lambda_handler"
-  runtime       = "python3.7"
+  runtime       = "python3.11"
   publish       = false
   timeout       = 900
   cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days