update perf dag to use postgres to allow latest dbt-core

.github/workflows/test.yml

@@ -299,7 +299,18 @@ jobs:
         python-version: ["3.11"]
         airflow-version: ["2.7"]
         num-models: [1, 10, 50, 100]
-
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
       - uses: actions/checkout@v3
         with:
@@ -336,7 +347,12 @@ jobs:
           AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
           PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
           MODEL_COUNT: ${{ matrix.num-models }}
-
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+          POSTGRES_SCHEMA: public
+          POSTGRES_PORT: 5432
     env:
       AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
       AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:[email protected]:5432/postgres

dev/dags/dbt/perf/profiles.yml

@@ -1,11 +1,12 @@
-simple:
+default:
   target: dev
   outputs:
     dev:
-      type: sqlite
-      threads: 1
-      database: "database"
-      schema: "main"
-      schemas_and_paths:
-        main: "{{ env_var('DBT_SQLITE_PATH') }}/imdb.db"
-      schema_directory: "{{ env_var('DBT_SQLITE_PATH') }}"
+      type: postgres
+      host: "{{ env_var('POSTGRES_HOST') }}"
+      user: "{{ env_var('POSTGRES_USER') }}"
+      password: "{{ env_var('POSTGRES_PASSWORD') }}"
+      port: "{{ env_var('POSTGRES_PORT') | int }}"
+      dbname: "{{ env_var('POSTGRES_DB') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA') }}"
+      threads: 4

dev/dags/performance_dag.py

@@ -1,28 +1,31 @@
 """
-A DAG that uses Cosmos to render a dbt project for performance testing.
+An airflow DAG that uses Cosmos to render a dbt project for performance testing.
 """
 
-import airflow
 from datetime import datetime
 import os
 from pathlib import Path
 
 from cosmos import DbtDag, ProjectConfig, ProfileConfig, RenderConfig
+from cosmos.profiles import PostgresUserPasswordProfileMapping
+
 
 DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt"
 DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH))
-DBT_SQLITE_PATH = str(DEFAULT_DBT_ROOT_PATH / "data")
+
 
 profile_config = ProfileConfig(
-    profile_name="simple",
+    profile_name="default",
     target_name="dev",
-    profiles_yml_filepath=(DBT_ROOT_PATH / "simple/profiles.yml"),
+    profile_mapping=PostgresUserPasswordProfileMapping(
+        conn_id="airflow_db",
+        profile_args={"schema": "public"},
+    ),
 )
 
 cosmos_perf_dag = DbtDag(
     project_config=ProjectConfig(
         DBT_ROOT_PATH / "perf",
-        env_vars={"DBT_SQLITE_PATH": DBT_SQLITE_PATH},
     ),
     profile_config=profile_config,
     render_config=RenderConfig(

scripts/test/performance-setup.sh

@@ -1,4 +1,4 @@
-pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \
+pip uninstall -y dbt-core dbt-sqlite dbt-postgres openlineage-airflow openlineage-integration-common; \
 rm -rf airflow.*; \
 airflow db init; \
-pip install 'dbt-core==1.4' 'dbt-sqlite<=1.4' 'dbt-databricks<=1.4' 'dbt-postgres<=1.4'
+pip install 'dbt-postgres'