WIP implement S3 bucket for raw PeMS data

.pre-commit-config.yaml

@@ -20,6 +20,15 @@ repos:
     hooks:
       - id: yamllint
         args: []
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.77.1
+    hooks:
+      - id: terraform_fmt
+      - id: terraform_validate
+        # Exclude modules to work around
+        # https://github.com/hashicorp/terraform/issues/28490
+        exclude: "terraform/[^/]+/modules/[^/]+/[^/]+$"
+      - id: terraform_tflint
   - repo: local
     hooks:
       - name: Dbt deps

terraform/.gitignore

@@ -0,0 +1 @@
+.terraform

terraform/environments/dev/caltrans-pems-dev.tfbackend

@@ -0,0 +1,4 @@
+bucket = "dse-infra-dev-terraform-state"
+dynamodb_table = "dse-infra-dev-terraform-state-lock"
+key = "caltrans-pems-dev.tfstate"
+region = "us-west-1"

terraform/environments/dev/main.tf

@@ -0,0 +1,47 @@
+##################################
+#        Terraform Setup         #
+##################################
+
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "4.56.0"
+    }
+  }
+
+  backend "s3" {
+  }
+}
+
+locals {
+  owner       = "caltrans"
+  environment = "dev"
+  project     = "pems"
+  region      = "us-west-2"
+}
+
+provider "aws" {
+  region = local.region
+
+  default_tags {
+    tags = {
+      Owner       = local.owner
+      Project     = local.project
+      Environment = local.environment
+    }
+  }
+}
+
+############################
+#      Infrastructure      #
+############################
+
+module "s3_lake" {
+  source = "../../modules/s3-lake"
+
+  prefix = "${local.owner}-${local.project}-${local.environment}"
+  region = local.region
+}

terraform/modules/s3-lake/iam.tf

@@ -0,0 +1,16 @@
+##################################
+#        IAM Service Users       #
+##################################
+
+# NOTE: in general, policies and roles are defined close to the resources
+# they support.
+
+# Airflow service user for writing to S3
+resource "aws_iam_user" "airflow_s3_writer" {
+  name = "${var.prefix}-airflow-s3-writer"
+}
+
+resource "aws_iam_user_policy_attachment" "airflow_s3_writer_policy_attachment" {
+  user       = aws_iam_user.airflow_s3_writer.name
+  policy_arn = aws_iam_policy.pems_raw_write.arn
+}

terraform/modules/s3-lake/main.tf

@@ -0,0 +1,18 @@
+##################################
+#        Terraform Setup         #
+##################################
+
+terraform {
+  # Note: when a package is added or updated, we have to update the lockfile in a
+  # platform-independent way, cf. https://github.com/hashicorp/terraform/issues/28041
+  # To update the lockfile run:
+  #
+  # terraform providers lock -platform=linux_amd64 -platform=darwin_amd64
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "4.56.0"
+    }
+  }
+  required_version = ">= 1.0"
+}

terraform/modules/s3-lake/s3.tf

@@ -0,0 +1,72 @@
+##################################
+#  Caltrans PeMS Infrastructure  #
+##################################
+
+# PeMS raw
+resource "aws_s3_bucket" "pems_raw" {
+  bucket = "${var.prefix}-${var.region}-pems-raw"
+}
+
+# Versioning
+resource "aws_s3_bucket_versioning" "pems_raw" {
+  bucket = aws_s3_bucket.pems_raw.bucket
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+# Write access
+data "aws_iam_policy_document" "pems_raw_write" {
+  statement {
+    actions = [
+      "s3:ListBucket"
+    ]
+    resources = [aws_s3_bucket.pems_raw.arn]
+  }
+  statement {
+    actions = [
+      "s3:PutObject",
+    ]
+    resources = ["${aws_s3_bucket.pems_raw.arn}/*"]
+  }
+}
+
+resource "aws_iam_policy" "pems_raw_write" {
+  name        = "${var.prefix}-${var.region}-pems-raw-write"
+  description = "Policy allowing write for s3 pems raw bucket"
+  policy      = data.aws_iam_policy_document.pems_raw_write.json
+}
+
+
+# Public read access
+data "aws_iam_policy_document" "pems_raw_read" {
+  statement {
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+    actions = [
+      "s3:GetObject",
+      "s3:ListBucket",
+    ]
+
+    resources = [
+      aws_s3_bucket.pems_raw.arn,
+      "${aws_s3_bucket.pems_raw.arn}/*",
+    ]
+  }
+}
+
+resource "aws_s3_bucket_policy" "pems_raw_read" {
+  bucket = aws_s3_bucket.pems_raw.id
+  policy = data.aws_iam_policy_document.pems_raw_read.json
+}
+
+resource "aws_s3_bucket_public_access_block" "pems_raw" {
+  bucket = aws_s3_bucket.pems_raw.id
+
+  block_public_acls       = false
+  block_public_policy     = false
+  ignore_public_acls      = false
+  restrict_public_buckets = false
+}

terraform/modules/s3-lake/variables.tf

@@ -0,0 +1,10 @@
+variable "prefix" {
+  description = "Prefix for resource names"
+  type        = string
+}
+
+variable "region" {
+  description = "Region for AWS resources"
+  type        = string
+  default     = "us-west-2"
+}