From 3c0727e9b7195b05e55557bfa0a56f390808eb3a Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 6 Dec 2023 15:07:50 +0530 Subject: [PATCH 1/3] feat(ci): split no cypress test suite (#9387) --- .github/workflows/docker-unified.yml | 12 ++++++------ smoke-test/smoke.sh | 8 +++++--- smoke-test/test_e2e.py | 2 ++ smoke-test/tests/privileges/test_privileges.py | 2 ++ smoke-test/tests/timeline/timeline_test.py | 2 ++ 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 8bb82a0a0608cb..fef23f9efa85f1 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -696,7 +696,12 @@ jobs: strategy: fail-fast: false matrix: - test_strategy: ["no_cypress", "cypress_suite1", "cypress_rest"] + test_strategy: [ + "no_cypress_suite0", + "no_cypress_suite1", + "cypress_suite1", + "cypress_rest" + ] needs: [ setup, @@ -792,11 +797,6 @@ jobs: ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml" run: | ./smoke-test/run-quickstart.sh - - name: sleep 60s - run: | - # we are doing this because gms takes time to get ready - # and we don't have a better readiness check when bootstrap is done - sleep 60s - name: Disk Check run: df -h . && docker images - name: Disable ES Disk Threshold diff --git a/smoke-test/smoke.sh b/smoke-test/smoke.sh index 3236a0e5c3f0c0..db0389be1f4897 100755 --- a/smoke-test/smoke.sh +++ b/smoke-test/smoke.sh @@ -24,12 +24,14 @@ source venv/bin/activate source ./set-cypress-creds.sh -# no_cypress, cypress_suite1, cypress_rest +# no_cypress_suite0, no_cypress_suite1, cypress_suite1, cypress_rest if [[ -z "${TEST_STRATEGY}" ]]; then pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml else - if [ "$TEST_STRATEGY" == "no_cypress" ]; then - pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress' + if [ "$TEST_STRATEGY" == "no_cypress_suite0" ]; then + pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress' -m 'not no_cypress_suite1' + elif [ "$TEST_STRATEGY" == "no_cypress_suite1" ]; then + pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -m 'no_cypress_suite1' else pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_cypress_${TEST_STRATEGY}.xml tests/cypress/integration_test.py fi diff --git a/smoke-test/test_e2e.py b/smoke-test/test_e2e.py index 4a0a122b796702..abb4841314c4af 100644 --- a/smoke-test/test_e2e.py +++ b/smoke-test/test_e2e.py @@ -8,6 +8,8 @@ import tenacity from datahub.ingestion.run.pipeline import Pipeline +pytestmark = pytest.mark.no_cypress_suite1 + from tests.utils import ( get_frontend_url, get_gms_url, diff --git a/smoke-test/tests/privileges/test_privileges.py b/smoke-test/tests/privileges/test_privileges.py index d0f00734ae9f37..aa54a50b04e7f8 100644 --- a/smoke-test/tests/privileges/test_privileges.py +++ b/smoke-test/tests/privileges/test_privileges.py @@ -5,6 +5,8 @@ get_frontend_url, get_admin_credentials,get_sleep_info) from tests.privileges.utils import * +pytestmark = pytest.mark.no_cypress_suite1 + sleep_sec, sleep_times = get_sleep_info() @pytest.fixture(scope="session") diff --git a/smoke-test/tests/timeline/timeline_test.py b/smoke-test/tests/timeline/timeline_test.py index 4705343c1a2baf..c075d981487dbd 100644 --- a/smoke-test/tests/timeline/timeline_test.py +++ b/smoke-test/tests/timeline/timeline_test.py @@ -1,4 +1,5 @@ import json +import pytest from time import sleep from datahub.cli import timeline_cli @@ -7,6 +8,7 @@ from tests.utils import (get_datahub_graph, ingest_file_via_rest, wait_for_writes_to_sync) +pytestmark = pytest.mark.no_cypress_suite1 def test_all(): platform = "urn:li:dataPlatform:kafka" From a9c5c3903c3af88bd1aaf45b1a131f04d3ef57c1 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 6 Dec 2023 15:58:20 +0530 Subject: [PATCH 2/3] fix(ingest/redshift): too many values unpack (#9394) --- .../src/datahub/ingestion/source/redshift/lineage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py index 05011b2d7a7694..abed8505f168bf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py @@ -381,7 +381,8 @@ def _get_upstream_lineages( qualified_table_name = dataset_urn.DatasetUrn.create_from_string( source.urn ).get_entity_id()[1] - db, schema, table = qualified_table_name.split(".") + # -3 because platform instance is optional and that can cause the split to have more than 3 elements + db, schema, table = qualified_table_name.split(".")[-3:] if db == raw_db_name: db = alias_db_name path = f"{db}.{schema}.{table}" From 2eee3332ead690178edf6a5e8f8f551fa0065163 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Wed, 6 Dec 2023 12:18:24 +0100 Subject: [PATCH 3/3] fix(ingest/redshift): Fix psycopg2 removal from Redshift Source (#9395) --- .../src/datahub/ingestion/source/redshift/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py index 51ad8a050adc24..540adbf4bfd15a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py @@ -10,7 +10,7 @@ from datahub.configuration.source_common import DatasetLineageProviderConfigBase from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.ingestion.source.data_lake_common.path_spec import PathSpec -from datahub.ingestion.source.sql.postgres import BasePostgresConfig +from datahub.ingestion.source.sql.sql_config import BasicSQLAlchemyConfig from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulLineageConfigMixin, StatefulProfilingConfigMixin, @@ -64,7 +64,7 @@ class RedshiftUsageConfig(BaseUsageConfig, StatefulUsageConfigMixin): class RedshiftConfig( - BasePostgresConfig, + BasicSQLAlchemyConfig, DatasetLineageProviderConfigBase, S3DatasetLineageProviderConfigBase, RedshiftUsageConfig,