cloudbuild.yaml

# Copyright 2021 Google LLC

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     https://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

steps:
  - name: 'gcr.io/$PROJECT_ID/airflow-dags-builder:latest'
    id: Generate Files
    entrypoint: python
    args:
      - scripts/dag_generator.py

  - name: 'gcr.io/$PROJECT_ID/airflow-dags-builder:latest'
    id: Dag validation test
    entrypoint: python
    env:
      - AIRFLOW_HOME=/workspace/airflow
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__CORE__DAGS_FOLDER=/workspace/datasets
    args:
      - -m
      - unittest
      - tests/dag_integrity_test.py

  - name: gcr.io/cloud-builders/gcloud
    id: Get composer GCS bucket location
    entrypoint: sh
    args:
      - '-c'
      - |
        gcloud beta composer environments describe composer-environment \
        --location us-central1 --format='value(config.dagGcsPrefix)' | sed -e 's/gs:\/\/*//' -e 's/\/dags*//' > .generated/composer_bucket.txt

  - id: 'Validate Terraform'
    name: 'hashicorp/terraform:1.0.10'
    entrypoint: 'sh'
    env:
      - 'TF_IN_AUTOMATION=1'
    args:
      - '-c'
      - |
        export TF_VAR_composer_bucket=$(cat .generated/composer_bucket.txt)
        terraform init \
        -backend-config="bucket=$PROJECT_ID-pipeline" \
        -backend-config="prefix=tfstate" \
        -input=false \
        .generated && \
        terraform validate .generated && \
        terraform plan .generated || exit 1