From fbb2fd8cb2d0dacc375edf98e07ff4ce4f8ff218 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Tue, 17 Oct 2023 11:12:03 -0700 Subject: [PATCH] WIP implement S3 bucket for raw PeMS data --- .pre-commit-config.yaml | 9 +++ terraform/.gitignore | 1 + .../environments/dev/.terraform.lock.hcl | 27 +++++++ .../dev/caltrans-pems-dev.tfbackend | 4 ++ terraform/environments/dev/main.tf | 47 ++++++++++++ terraform/modules/s3-lake/iam.tf | 16 +++++ terraform/modules/s3-lake/main.tf | 18 +++++ terraform/modules/s3-lake/s3.tf | 72 +++++++++++++++++++ terraform/modules/s3-lake/variables.tf | 10 +++ 9 files changed, 204 insertions(+) create mode 100644 terraform/.gitignore create mode 100644 terraform/environments/dev/.terraform.lock.hcl create mode 100644 terraform/environments/dev/caltrans-pems-dev.tfbackend create mode 100644 terraform/environments/dev/main.tf create mode 100644 terraform/modules/s3-lake/iam.tf create mode 100644 terraform/modules/s3-lake/main.tf create mode 100644 terraform/modules/s3-lake/s3.tf create mode 100644 terraform/modules/s3-lake/variables.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40ed227d..e0ff8dce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,6 +20,15 @@ repos: hooks: - id: yamllint args: [] + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.77.1 + hooks: + - id: terraform_fmt + - id: terraform_validate + # Exclude modules to work around + # https://github.com/hashicorp/terraform/issues/28490 + exclude: "terraform/[^/]+/modules/[^/]+/[^/]+$" + - id: terraform_tflint - repo: local hooks: - name: Dbt deps diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 00000000..3fa8c86b --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1 @@ +.terraform diff --git a/terraform/environments/dev/.terraform.lock.hcl b/terraform/environments/dev/.terraform.lock.hcl new file mode 100644 index 00000000..d15c61cb --- /dev/null +++ b/terraform/environments/dev/.terraform.lock.hcl @@ -0,0 +1,27 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.56.0" + constraints = "4.56.0" + hashes = [ + "h1:CnpBvf3mH16Kcez47OsjmIeGkY2PUVihKRwbkyOvo48=", + "h1:koDunHl/LUmCAKy3VFie6MakXN7ng93v8HBRpKI8He8=", + "h1:v6DE95Ll2mxE96IGUsT/h6WQTU1d2cfHydWah1FgT20=", + "zh:1d2b7693a102da015a86b9235b554272b9280597011216c3ddd1a6dc95ad8dab", + "zh:28c3e8ebaa077f65c4ac5fd051c95887070293fcff0386dfc2e4b7e248a0aefa", + "zh:2a620bc4a87be06e7acac1bc15e966dba45df643bf6c3efb811e74e6c2122b03", + "zh:30d3ac148fa0634e7ba1de66e1af1328481c92cd774adcfc0e27f828103b17e0", + "zh:3d3eebf916f25e11b12dd3c692f8fe1e4c4e9a0c414af9d0d881ddebd28dcd39", + "zh:3f4600f2881c02fcc69080df68747c9a0b9b11cb002117fd918b7800f2ac402b", + "zh:7156fb12c3b4f2964f7e78cee97f31d95b43045467f90749d2ed545725c36baa", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a5bbc84fd37d468c7b016009776b6d2a287bbb746af81aba786cdf8eb5fce4a1", + "zh:d5322bcd4e11caddbbfaa1198893824d4b4d28f504517a3a87902cf86d75bd87", + "zh:d766eb9f86a40060d63e12ef674d7c9c47ec4e47ade487f1f49af8c89b441711", + "zh:df23f592b99f6617f09e449009bbb49068a69fc926b15ca29e30b068c9c67365", + "zh:e7b0acee2d98549731547259b539f598e18db07c0c202d3a34b922beff711054", + "zh:ec317f79fdcce934c39458ea312862e7f7ec48cafb8bcc9b5a00d9b78b629d81", + "zh:f78ec7a771867d96dfee96bf74523341ba42feeb64ce2f108b5bf2e7ebef0fef", + ] +} diff --git a/terraform/environments/dev/caltrans-pems-dev.tfbackend b/terraform/environments/dev/caltrans-pems-dev.tfbackend new file mode 100644 index 00000000..51557fc2 --- /dev/null +++ b/terraform/environments/dev/caltrans-pems-dev.tfbackend @@ -0,0 +1,4 @@ +bucket = "dse-infra-dev-terraform-state" +dynamodb_table = "dse-infra-dev-terraform-state-lock" +key = "caltrans-pems-dev.tfstate" +region = "us-west-1" diff --git a/terraform/environments/dev/main.tf b/terraform/environments/dev/main.tf new file mode 100644 index 00000000..1cf29b51 --- /dev/null +++ b/terraform/environments/dev/main.tf @@ -0,0 +1,47 @@ +################################## +# Terraform Setup # +################################## + +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "4.56.0" + } + } + + backend "s3" { + } +} + +locals { + owner = "caltrans" + environment = "dev" + project = "pems" + region = "us-west-2" +} + +provider "aws" { + region = local.region + + default_tags { + tags = { + Owner = local.owner + Project = local.project + Environment = local.environment + } + } +} + +############################ +# Infrastructure # +############################ + +module "s3_lake" { + source = "../../modules/s3-lake" + + prefix = "${local.owner}-${local.project}-${local.environment}" + region = local.region +} diff --git a/terraform/modules/s3-lake/iam.tf b/terraform/modules/s3-lake/iam.tf new file mode 100644 index 00000000..4c274ef9 --- /dev/null +++ b/terraform/modules/s3-lake/iam.tf @@ -0,0 +1,16 @@ +################################## +# IAM Service Users # +################################## + +# NOTE: in general, policies and roles are defined close to the resources +# they support. + +# Airflow service user for writing to S3 +resource "aws_iam_user" "airflow_s3_writer" { + name = "${var.prefix}-airflow-s3-writer" +} + +resource "aws_iam_user_policy_attachment" "airflow_s3_writer_policy_attachment" { + user = aws_iam_user.airflow_s3_writer.name + policy_arn = aws_iam_policy.pems_raw_write.arn +} diff --git a/terraform/modules/s3-lake/main.tf b/terraform/modules/s3-lake/main.tf new file mode 100644 index 00000000..784b2dd5 --- /dev/null +++ b/terraform/modules/s3-lake/main.tf @@ -0,0 +1,18 @@ +################################## +# Terraform Setup # +################################## + +terraform { + # Note: when a package is added or updated, we have to update the lockfile in a + # platform-independent way, cf. https://github.com/hashicorp/terraform/issues/28041 + # To update the lockfile run: + # + # terraform providers lock -platform=linux_amd64 -platform=darwin_amd64 + required_providers { + aws = { + source = "hashicorp/aws" + version = "4.56.0" + } + } + required_version = ">= 1.0" +} diff --git a/terraform/modules/s3-lake/s3.tf b/terraform/modules/s3-lake/s3.tf new file mode 100644 index 00000000..522fb5f8 --- /dev/null +++ b/terraform/modules/s3-lake/s3.tf @@ -0,0 +1,72 @@ +################################## +# Caltrans PeMS Infrastructure # +################################## + +# PeMS raw +resource "aws_s3_bucket" "pems_raw" { + bucket = "${var.prefix}-${var.region}-pems-raw" +} + +# Versioning +resource "aws_s3_bucket_versioning" "pems_raw" { + bucket = aws_s3_bucket.pems_raw.bucket + versioning_configuration { + status = "Enabled" + } +} + +# Write access +data "aws_iam_policy_document" "pems_raw_write" { + statement { + actions = [ + "s3:ListBucket" + ] + resources = [aws_s3_bucket.pems_raw.arn] + } + statement { + actions = [ + "s3:PutObject", + ] + resources = ["${aws_s3_bucket.pems_raw.arn}/*"] + } +} + +resource "aws_iam_policy" "pems_raw_write" { + name = "${var.prefix}-${var.region}-pems-raw-write" + description = "Policy allowing write for s3 pems raw bucket" + policy = data.aws_iam_policy_document.pems_raw_write.json +} + + +# Public read access +data "aws_iam_policy_document" "pems_raw_read" { + statement { + principals { + type = "AWS" + identifiers = ["*"] + } + actions = [ + "s3:GetObject", + "s3:ListBucket", + ] + + resources = [ + aws_s3_bucket.pems_raw.arn, + "${aws_s3_bucket.pems_raw.arn}/*", + ] + } +} + +resource "aws_s3_bucket_policy" "pems_raw_read" { + bucket = aws_s3_bucket.pems_raw.id + policy = data.aws_iam_policy_document.pems_raw_read.json +} + +resource "aws_s3_bucket_public_access_block" "pems_raw" { + bucket = aws_s3_bucket.pems_raw.id + + block_public_acls = false + block_public_policy = false + ignore_public_acls = false + restrict_public_buckets = false +} diff --git a/terraform/modules/s3-lake/variables.tf b/terraform/modules/s3-lake/variables.tf new file mode 100644 index 00000000..e7284503 --- /dev/null +++ b/terraform/modules/s3-lake/variables.tf @@ -0,0 +1,10 @@ +variable "prefix" { + description = "Prefix for resource names" + type = string +} + +variable "region" { + description = "Region for AWS resources" + type = string + default = "us-west-2" +}