diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 311cd33a..80c6b00e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -7,10 +7,10 @@ jobs: name: run flake8 runs-on: ubuntu-20.04 steps: - - name: set up python 3.7 + - name: set up python 3.11 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.11 - name: checkout code uses: actions/checkout@v2 @@ -25,10 +25,10 @@ jobs: name: run black runs-on: ubuntu-20.04 steps: - - name: set up python 3.7 + - name: set up python 3.11 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.11 - name: checkout code uses: actions/checkout@v2 @@ -43,10 +43,10 @@ jobs: name: run bandit runs-on: ubuntu-20.04 steps: - - name: set up python 3.7 + - name: set up python 3.11 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.11 - name: checkout code uses: actions/checkout@v2 diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 9d2c4ef5..570be108 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -8,10 +8,10 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: set up python 3.7.12 + - name: set up python 3.11 uses: actions/setup-python@v2 with: - python-version: 3.7.12 + python-version: 3.11 - name: checkout code uses: actions/checkout@v2 @@ -45,10 +45,10 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: set up python 3.7.12 + - name: set up python 3.11 uses: actions/setup-python@v2 with: - python-version: 3.7.12 + python-version: 3.11 - name: checkout code uses: actions/checkout@v2 diff --git a/calcloud/model_ingest.py b/calcloud/model_ingest.py index 994debdf..b54eed46 100644 --- a/calcloud/model_ingest.py +++ b/calcloud/model_ingest.py @@ -224,7 +224,7 @@ def convert_target_data(self): clock += sum(x * int(t) for x, t in zip([1, 60, 3600], clocktime)) print(clock) for memstr in self.target_data["memory"]: - kb += np.float(memstr) + kb += float(memstr) print(kb) targets["wallclock"] = clock + 1 targets["memory"] = kb / (10**6) diff --git a/iac/codebuild/Dockerfile b/iac/codebuild/Dockerfile index c2c2a6d3..03458f25 100644 --- a/iac/codebuild/Dockerfile +++ b/iac/codebuild/Dockerfile @@ -38,13 +38,13 @@ ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/stsci-tls-ca-bundle.pem ENV CURL_CA_BUNDLE=/etc/ssl/certs/stsci-tls-ca-bundle.pem ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/stsci-tls-ca-bundle.pem -#Update Certificates -#RUN yum update ca-certificates -y +#RUN curl -L https://rpm.nodesource.com/setup_16.x | bash - && \ +# yum install -y nodejs -RUN curl -L https://rpm.nodesource.com/setup_16.x | bash - && \ - yum install -y nodejs +#ENV NODE_16_VERSION="16.15.1" -ENV NODE_16_VERSION="16.15.1" +RUN yum install https://rpm.nodesource.com/pub_16.x/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y +RUN yum install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1 RUN npm install n -g diff --git a/lambda/JobPredict/Dockerfile b/lambda/JobPredict/Dockerfile index 993da9a7..61548c92 100644 --- a/lambda/JobPredict/Dockerfile +++ b/lambda/JobPredict/Dockerfile @@ -1,5 +1,5 @@ #FROM public.ecr.aws/lambda/python:3.7 -FROM amazon/aws-lambda-python:3.7 +FROM amazon/aws-lambda-python:3.11 COPY requirements.txt predict_handler.py ./ # SSL/TLS cert setup for STScI AWS firewalling USER root @@ -15,6 +15,7 @@ RUN mv /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-bundle.crt.org && ln -s /e #log4j vuln patch RUN yum update -y java-1.8.0-openjdk -RUN python3.7 -m pip install --upgrade pip && python3.7 -m pip install -r requirements.txt && python3.7 -m pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.6.2-cp37-cp37m-manylinux2010_x86_64.whl +#RUN python3.7 -m pip install --upgrade pip && python3.7 -m pip install -r requirements.txt && python3.7 -m pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.6.2-cp37-cp37m-manylinux2010_x86_64.whl +RUN python3.11 -m pip install --upgrade pip && python3.11 -m pip install -r requirements.txt && python3.11 -m pip install tensorflow-cpu==2.12.1 ADD models ./models/ CMD ["predict_handler.lambda_handler"] diff --git a/lambda/JobPredict/requirements.txt b/lambda/JobPredict/requirements.txt index 2cfbc6eb..b95fbebc 100644 --- a/lambda/JobPredict/requirements.txt +++ b/lambda/JobPredict/requirements.txt @@ -1,3 +1,3 @@ -scikit-learn==1.0.2 -boto3==1.17.25 -protobuf==3.20.1 +scikit-learn==1.3.0 +boto3==1.28.56 +protobuf==4.24.3 diff --git a/modeling/Dockerfile b/modeling/Dockerfile index bea38ad2..b60d638d 100644 --- a/modeling/Dockerfile +++ b/modeling/Dockerfile @@ -25,12 +25,12 @@ unzip \ zlib1g-dev RUN DEBIAN_FRONTEND=noninteractive \ -&& curl -o Python-3.9.5.tgz https://www.python.org/ftp/python/3.9.5/Python-3.9.5.tgz \ -&& tar -xf Python-3.9.5.tgz \ -&& cd Python-3.9.5 \ +&& curl -o Python-3.11.5.tgz https://www.python.org/ftp/python/3.11.5/Python-3.11.5.tgz \ +&& tar -xf Python-3.11.5.tgz \ +&& cd Python-3.11.5 \ && ./configure --enable-optimizations \ && make altinstall \ -&& update-alternatives --install /usr/local/bin/python python /usr/local/bin/python3.9 10 +&& update-alternatives --install /usr/local/bin/python python /usr/local/bin/python3.11 10 RUN DEBIAN_FRONTEND=noninteractive \ && cd ../ && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ diff --git a/modeling/requirements.txt b/modeling/requirements.txt index 0cfc22c5..953b8c15 100644 --- a/modeling/requirements.txt +++ b/modeling/requirements.txt @@ -1,5 +1,5 @@ -tensorflow==2.6.2 -scikit-learn==1.0.2 -boto3==1.17.79 -pandas==1.2.4 -protobuf==3.20.1 \ No newline at end of file +tensorflow==2.12.1 +scikit-learn==1.3.0 +boto3==1.28.56 +pandas==2.1.0 +protobuf==4.24.3 \ No newline at end of file diff --git a/notes/lambda.function.configuration.json b/notes/lambda.function.configuration.json index 89e85c93..33782f8d 100644 --- a/notes/lambda.function.configuration.json +++ b/notes/lambda.function.configuration.json @@ -1,7 +1,7 @@ { "FunctionName": "calcloud-hst-trigger", "FunctionArn": "arn:aws:lambda:us-east-1:162808325377:function:calcloud-hst-trigger", - "Runtime": "python3.7", + "Runtime": "python3.11", "Role": "arn:aws:iam::162808325377:role/calcloud-hst-trigger-lambda-role", "Handler": "lambda_function.lambda_handler", "CodeSize": 1435, diff --git a/notes/lambda.function.json b/notes/lambda.function.json index 6a2c5a9c..30922b25 100644 --- a/notes/lambda.function.json +++ b/notes/lambda.function.json @@ -2,7 +2,7 @@ "Configuration": { "FunctionName": "calcloud-hst-trigger", "FunctionArn": "arn:aws:lambda:us-east-1:162808325377:function:calcloud-hst-trigger", - "Runtime": "python3.7", + "Runtime": "python3.11", "Role": "arn:aws:iam::162808325377:role/calcloud-hst-trigger-lambda-role", "Handler": "lambda_function.lambda_handler", "CodeSize": 1435, diff --git a/setup.cfg b/setup.cfg index 8011f68b..16dfb611 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ github_project = spacetelescope/calcloud [options] packages = find: -python_requires = ~=3.7.0 +python_requires = ~=3.11.0 setup_requires = setuptools >=41.0.1 install_requires = diff --git a/terraform/ami-rotation.tf b/terraform/ami-rotation.tf index adb157d7..cff24335 100644 --- a/terraform/ami-rotation.tf +++ b/terraform/ami-rotation.tf @@ -57,13 +57,13 @@ resource "aws_launch_template" "ami_rotation" { module "calcloud_env_amiRotation" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-env-AmiRotation${local.environment}" description = "spawns an ec2 bi-weekly which rotates the ami for batch" # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "ami_rotation.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 60 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/batch.tf b/terraform/batch.tf index 4ba207f4..8701777a 100644 --- a/terraform/batch.tf +++ b/terraform/batch.tf @@ -8,27 +8,27 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 3.65.0" + version = "~> 5.17.0" } hashicorp-null = { source = "hashicorp/null" - version = "~> 3.1.0" + version = "~> 3.2.1" } hashicorp-external = { source = "hashicorp/external" - version = "~> 2.1.0" + version = "~> 2.3.1" } hashicorp-local = { source = "hashicorp/local" - version = "~> 2.1.0" + version = "~> 2.4.0" } hashicorp-random = { source = "hashicorp/random" - version = "~> 3.1.0" + version = "~> 3.5.1" } docker = { source = "kreuzwerker/docker" - version = "~> 2.15.0" + version = "~> 3.0.2" } } } diff --git a/terraform/lambda_batch_events.tf b/terraform/lambda_batch_events.tf index 12cf8e6e..f09e08ac 100644 --- a/terraform/lambda_batch_events.tf +++ b/terraform/lambda_batch_events.tf @@ -1,12 +1,12 @@ module "calcloud_lambda_batchEvents" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-job-events${local.environment}" description = "listens for Batch failure events from cloudWatch event rule" # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "batch_event_handler.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 900 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/lambda_blackboard.tf b/terraform/lambda_blackboard.tf index 471cdb38..445d5660 100644 --- a/terraform/lambda_blackboard.tf +++ b/terraform/lambda_blackboard.tf @@ -1,12 +1,12 @@ module "calcloud_lambda_blackboard" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-job-blackboard${local.environment}" description = "scrapes the Batch console for job metadata and posts to S3 bucket for on-premise poller" # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "scrape_batch.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 300 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/lambda_broadcast.tf b/terraform/lambda_broadcast.tf index 2e1c72b7..abb6f65f 100644 --- a/terraform/lambda_broadcast.tf +++ b/terraform/lambda_broadcast.tf @@ -1,12 +1,12 @@ module "calcloud_lambda_broadcast" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-broadcast${local.environment}" description = "Broadcasts the specified message type across a list of job_ids or ippppssoots." # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "broadcast_handler.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 300 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/lambda_job_clean.tf b/terraform/lambda_job_clean.tf index 245a07ca..14e071b4 100644 --- a/terraform/lambda_job_clean.tf +++ b/terraform/lambda_job_clean.tf @@ -1,12 +1,12 @@ module "calcloud_lambda_cleanJob" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-job-clean${local.environment}" description = "accepts messages from s3 event and cleans either individual jobs by dataset, or all active jobs" # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "clean_handler.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 900 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/lambda_job_delete.tf b/terraform/lambda_job_delete.tf index 438f9a32..0b6154fd 100644 --- a/terraform/lambda_job_delete.tf +++ b/terraform/lambda_job_delete.tf @@ -1,12 +1,12 @@ module "calcloud_lambda_deleteJob" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-job-delete${local.environment}" description = "accepts messages from s3 event and deletes either individual jobs by dataset, or all active jobs" # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "delete_handler.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 900 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/lambda_job_predict.tf b/terraform/lambda_job_predict.tf index 35c910d3..fecb87ab 100644 --- a/terraform/lambda_job_predict.tf +++ b/terraform/lambda_job_predict.tf @@ -1,6 +1,6 @@ module "lambda_function_container_image" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-job-predict${local.environment}" description = "pretrained neural networks for predicting job resource requirements (memory bin and max execution time)" diff --git a/terraform/lambda_job_rescue.tf b/terraform/lambda_job_rescue.tf index 99f8dc13..f39b37d6 100644 --- a/terraform/lambda_job_rescue.tf +++ b/terraform/lambda_job_rescue.tf @@ -1,12 +1,12 @@ module "calcloud_lambda_rescueJob" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-job-rescue${local.environment}" description = "Rescues the specified dataset (must be in error state) by deleting all outputs and messages and re-placing." # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "rescue_handler.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 900 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/lambda_job_submit.tf b/terraform/lambda_job_submit.tf index e42221ce..20437ed9 100644 --- a/terraform/lambda_job_submit.tf +++ b/terraform/lambda_job_submit.tf @@ -1,13 +1,13 @@ module "calcloud_lambda_submit" { source = "terraform-aws-modules/lambda/aws" # https://github.com/hashicorp/terraform/issues/17211 - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-job-submit${local.environment}" description = "looks for placed-dataset messages and submits jobs to Batch" # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "s3_trigger_handler.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 15*60 # see also SUBMIT_TIMEOUT below; this is the AWS timeout, calcloud error handling may not occur cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/lambda_model_ingest.tf b/terraform/lambda_model_ingest.tf index 1298151d..55a1c85d 100644 --- a/terraform/lambda_model_ingest.tf +++ b/terraform/lambda_model_ingest.tf @@ -16,13 +16,13 @@ resource "aws_dynamodb_table" "calcloud_model_db" { module "lambda_model_ingest" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-model-ingest${local.environment}" lambda_role = nonsensitive(data.aws_ssm_parameter.model_ingest_role.value) description = "looks for processed-ipppssoot.trigger messages, scrapes and uploads completed job data to DynamoDB" handler = "lambda_scrape.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 180 memory_size = 256 diff --git a/terraform/lambda_refresh_cache.tf b/terraform/lambda_refresh_cache.tf index 941f3e74..be4074b1 100644 --- a/terraform/lambda_refresh_cache.tf +++ b/terraform/lambda_refresh_cache.tf @@ -1,12 +1,12 @@ module "calcloud_lambda_refresh_cache_submit" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-fileshare-refresh_cache_submit${local.environment}" description = "submits refresh cache operations" # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "refresh_cache_submit.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 900 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/terraform/lambda_refresh_cache_logging.tf b/terraform/lambda_refresh_cache_logging.tf index e4ca844d..6ce5f953 100644 --- a/terraform/lambda_refresh_cache_logging.tf +++ b/terraform/lambda_refresh_cache_logging.tf @@ -1,12 +1,12 @@ module "calcloud_lambda_refresh_cache_logs" { source = "terraform-aws-modules/lambda/aws" - version = "~> 2.26.0" + version = "~> 6.0.0" function_name = "calcloud-fileshare-refresh_cache_logs${local.environment}" description = "listens for refresh cache operations and logs them" # the path is relative to the path inside the lambda env, not in the local filesystem. handler = "refresh_cache_logs.lambda_handler" - runtime = "python3.7" + runtime = "python3.11" publish = false timeout = 900 cloudwatch_logs_retention_in_days = local.lambda_log_retention_in_days diff --git a/tests/conftest.py b/tests/conftest.py index a42d6b47..b477e617 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -234,7 +234,7 @@ def create_mock_lambda(lambda_client, iam_client, name=os.environ["JOBPREDICTLAM response = lambda_client.create_function( FunctionName=name, - Runtime="python3.8", + Runtime="python3.11", Role=iam_arn, Handler="lambda_function.lambda_handler", Code={"ZipFile": job_predict_zip(memBin=memBin, clockTime=clockTime)}, diff --git a/tox.ini b/tox.ini index e332ca38..2de441be 100644 --- a/tox.ini +++ b/tox.ini @@ -2,21 +2,21 @@ envlist = black, flake8, bandit #, coverage, pytest [testenv:black] -basepython = python3.7 +basepython = python3.11 extras = dev whitelist_externals = black commands= black --check calcloud tests lambda ami_rotation scripts [testenv:flake8] -basepython = python3.7 +basepython = python3.11 extras = dev whitelist_externals = flake8 commands = flake8 --count --ignore E501,W503,E203 --max-line-length 88 calcloud tests lambda ami_rotation [testenv:bandit] -basepython = python3.7 +basepython = python3.11 extras = dev whitelist_externals = bandit commands =