databricks.yml

# This is a Databricks asset bundle definition for marvelous-databricks-course-benitomartin.
# The Databricks extension requires databricks.yml configuration file.
# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
# https://docs.databricks.com/en/dev-tools/bundles/settings.html

bundle:
  name: marvelous-databricks-course-benitomartin

include:
  - bundle_monitoring.yml

# These are the default artifact settings if not otherwise overridden in
# the following "targets" top-level mapping.
artifacts:
  default:
    type: whl
    build: uv build
    path: .

# These are for any custom variables for use throughout the bundle.
variables:
  root_path:
    description: root_path for the bundle files
    default: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}/files
  git_sha:
    description: git_sha
    default: 1aa0037267af83a7685bea002a332739563fa3c9
  schedule_pause_status:
    description: schedule pause status
    default: UNPAUSED

# These are the default job and pipeline settings if not otherwise overridden in
# the following "targets" top-level mapping.
# Recommended: only jobs and pipelines
resources:
  jobs:
    credit-default:
      name: credit-default-workflow-test-demo
      schedule:
        quartz_cron_expression: "0 0 6 ? * MON"
        timezone_id: "Europe/Amsterdam"
        pause_status: ${var.schedule_pause_status}
      tags:
        project_name: "credit-default"
      # job_clusters:
      #   - job_cluster_key: "credit-default-cluster"
      #     new_cluster:
      #       spark_version: "15.4.x-scala2.12"
      #       data_security_mode: "SINGLE_USER"
      #       node_type_id: "i3.xlarge"
      #       driver_node_type_id: "i3.xlarge"
      #       autoscale:
      #         min_workers: 1
      #         max_workers: 1

      tasks:

        - task_key: "preprocessing"
          # job_cluster_key: "credit-default-cluster"
          existing_cluster_id: 1109-205408-dob4qyuc
          spark_python_task:
            python_file: "workflows/preprocess.py"
            parameters:
              - "--root_path"
              - ${var.root_path}
          libraries:
           - whl: ./dist/*.whl

        - task_key: if_refreshed
          condition_task:
            op: "EQUAL_TO"
            left: "{{tasks.preprocessing.values.refreshed}}"
            right: "1"
          depends_on:
            - task_key: "preprocessing"

        - task_key: "train_model"
          depends_on:
            - task_key: "if_refreshed"
              outcome: "true"
          # job_cluster_key: "credit-default-cluster"
          existing_cluster_id: 1109-205408-dob4qyuc
          spark_python_task:
            python_file: "workflows/train_model.py"
            parameters:
              - "--root_path"
              - ${var.root_path}
              - "--git_sha"
              - ${var.git_sha}
              - "--job_run_id"
              - "{{job.id}}"
          libraries:
            - whl: ./dist/*.whl

        - task_key: "evaluate_model"
          depends_on:
            - task_key: "train_model"
          # job_cluster_key: "credit-default-cluster"
          existing_cluster_id: 1109-205408-dob4qyuc
          spark_python_task:
            python_file: "workflows/evaluate_model.py"
            parameters:
              - "--root_path"
              - ${var.root_path}
              - "--new_model_uri"
              - "{{tasks.train_model.values.new_model_uri}}"
              - "--job_run_id"
              - "{{job.id}}"
              - "--git_sha"
              - ${var.git_sha}
          libraries:
            - whl: ./dist/*.whl

        - task_key: model_update
          condition_task:
            op: "EQUAL_TO"
            left: "{{tasks.evaluate_model.values.model_update}}"
            right: "1"
          depends_on:
            - task_key: "evaluate_model"
        - task_key: "deploy_model"
          depends_on:
            - task_key: "model_update"
              outcome: "true"
          # job_cluster_key: "credit-default-cluster"
          existing_cluster_id: 1109-205408-dob4qyuc
          spark_python_task:
            python_file: "workflows/deploy_model.py"
            parameters:
              - "--root_path"
              - ${var.root_path}
          libraries:
            - whl: ./dist/*.whl


# Two targets
targets:

  dev_west: # AWS West Region workspace
    mode: development
    default: true
    workspace:
      host: https://dbc-46617658-3f2b.cloud.databricks.com

  prod_west:
    mode: production
    default: false
    workspace:
      host: https://dbc-46617658-3f2b.cloud.databricks.com

  # dev_std: # Students workspace
  #   mode: development
  #   default: true
  #   workspace:
  #     host: https://dbc-643c4c2b-d6c9.cloud.databricks.com

  # prod_std:
  #   mode: production
  #   default: false
  #   workspace:
  #     host: https://dbc-643c4c2b-d6c9.cloud.databricks.com


#   dev: # AWS Central Region workspace
#     mode: development
#     default: true
#     workspace:
#       host: https://dbc-df5087bc-8b50.cloud.databricks.com

  # prod:
  #   mode: production
  #   default: false
  #   workspace:
  #     host: https://dbc-df5087bc-8b50.cloud.databricks.com