From 0dda590aa89ff860a300ca705b62e33c559718ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Thu, 21 Nov 2024 12:45:39 +0000 Subject: [PATCH] [DOP-21634] Make DBReader and DBWriter table name requirements less strict --- .github/workflows/test-mssql.yml | 7 +++++++ docs/changelog/0.12.3.rst | 7 +++++++ docs/changelog/index.rst | 1 + .../dialect_mixins/support_name_with_schema_only.py | 2 +- .../test_db_reader_unit/test_clickhouse_reader_unit.py | 5 ++--- .../test_db_reader_unit/test_greenplum_reader_unit.py | 5 ++--- .../test_db/test_db_reader_unit/test_hive_reader_unit.py | 5 ++--- .../test_db/test_db_reader_unit/test_mssql_reader_unit.py | 5 ++--- .../test_db/test_db_reader_unit/test_mysql_reader_unit.py | 5 ++--- .../test_db/test_db_reader_unit/test_oracle_reader_unit.py | 5 ++--- .../test_db_reader_unit/test_postgres_reader_unit.py | 5 ++--- .../test_db_reader_unit/test_teradata_reader_unit.py | 5 ++--- .../test_db_writer_unit/test_clickhouse_writer_unit.py | 5 ++--- .../test_db_writer_unit/test_greenplum_writer_unit.py | 5 ++--- .../test_db/test_db_writer_unit/test_hive_writer_unit.py | 5 ++--- .../test_db/test_db_writer_unit/test_mssql_writer_unit.py | 5 ++--- .../test_db/test_db_writer_unit/test_mysql_writer_unit.py | 5 ++--- .../test_db/test_db_writer_unit/test_oracle_writer_unit.py | 5 ++--- .../test_db_writer_unit/test_postgres_writer_unit.py | 5 ++--- .../test_db_writer_unit/test_teradata_writer_unit.py | 5 ++--- 20 files changed, 48 insertions(+), 49 deletions(-) create mode 100644 docs/changelog/0.12.3.rst diff --git a/.github/workflows/test-mssql.yml b/.github/workflows/test-mssql.yml index cb7252f91..2980ece05 100644 --- a/.github/workflows/test-mssql.yml +++ b/.github/workflows/test-mssql.yml @@ -45,6 +45,13 @@ jobs: with: python-version: ${{ inputs.python-version }} + # https://github.com/pymssql/pymssql/issues/372#issuecomment-742386160 + - name: Add missing sqlfont.h file + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends freetds-dev libkrb5-dev gcc + - name: Cache Ivy uses: actions/cache@v4 if: inputs.with-cache diff --git a/docs/changelog/0.12.3.rst b/docs/changelog/0.12.3.rst new file mode 100644 index 000000000..778270280 --- /dev/null +++ b/docs/changelog/0.12.3.rst @@ -0,0 +1,7 @@ +0.12.3 (2024-11-21) +=================== + +Bug Fixes +--------- + +- Allow passing table names in format ``schema."table.with.dots"`` to ``DBReader(name=...)`` and ``DBWriter(name=...)``. diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst index 8e0370bb9..bb23a9875 100644 --- a/docs/changelog/index.rst +++ b/docs/changelog/index.rst @@ -3,6 +3,7 @@ :caption: Changelog DRAFT + 0.12.3 0.12.2 0.12.1 0.12.0 diff --git a/onetl/connection/db_connection/dialect_mixins/support_name_with_schema_only.py b/onetl/connection/db_connection/dialect_mixins/support_name_with_schema_only.py index 0e66b9800..db369eb1f 100644 --- a/onetl/connection/db_connection/dialect_mixins/support_name_with_schema_only.py +++ b/onetl/connection/db_connection/dialect_mixins/support_name_with_schema_only.py @@ -5,7 +5,7 @@ class SupportNameWithSchemaOnly: def validate_name(self, value: str) -> str: - if value.count(".") != 1: + if "." not in value: raise ValueError( f"Name should be passed in `schema.name` format, got '{value}'", ) diff --git a/tests/tests_unit/test_db/test_db_reader_unit/test_clickhouse_reader_unit.py b/tests/tests_unit/test_db/test_db_reader_unit/test_clickhouse_reader_unit.py index 598e9d1b5..5df85fed3 100644 --- a/tests/tests_unit/test_db/test_db_reader_unit/test_clickhouse_reader_unit.py +++ b/tests/tests_unit/test_db/test_db_reader_unit/test_clickhouse_reader_unit.py @@ -35,14 +35,13 @@ def test_clickhouse_reader_snapshot_error_pass_df_schema(spark_mock): ) -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_clickhouse_reader_wrong_table_name(spark_mock, table): +def test_clickhouse_reader_wrong_table_name(spark_mock): clickhouse = Clickhouse(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBReader( connection=clickhouse, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_reader_unit/test_greenplum_reader_unit.py b/tests/tests_unit/test_db/test_db_reader_unit/test_greenplum_reader_unit.py index 4f4820d4d..7ad9a1a60 100644 --- a/tests/tests_unit/test_db/test_db_reader_unit/test_greenplum_reader_unit.py +++ b/tests/tests_unit/test_db/test_db_reader_unit/test_greenplum_reader_unit.py @@ -36,14 +36,13 @@ def test_greenplum_reader_snapshot_error_pass_df_schema(spark_mock): ) -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_greenplum_reader_wrong_table_name(spark_mock, table): +def test_greenplum_reader_wrong_table_name(spark_mock): greenplum = Greenplum(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBReader( connection=greenplum, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_reader_unit/test_hive_reader_unit.py b/tests/tests_unit/test_db/test_db_reader_unit/test_hive_reader_unit.py index acb467e3b..400fe2ace 100644 --- a/tests/tests_unit/test_db/test_db_reader_unit/test_hive_reader_unit.py +++ b/tests/tests_unit/test_db/test_db_reader_unit/test_hive_reader_unit.py @@ -36,14 +36,13 @@ def test_hive_reader_snapshot_error_pass_df_schema(spark_mock): ) -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_hive_reader_wrong_table_name(spark_mock, table): +def test_hive_reader_wrong_table_name(spark_mock): hive = Hive(cluster="rnd-dwh", spark=spark_mock) with pytest.raises(ValueError): DBReader( connection=hive, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_reader_unit/test_mssql_reader_unit.py b/tests/tests_unit/test_db/test_db_reader_unit/test_mssql_reader_unit.py index ea9953e68..291fff179 100644 --- a/tests/tests_unit/test_db/test_db_reader_unit/test_mssql_reader_unit.py +++ b/tests/tests_unit/test_db/test_db_reader_unit/test_mssql_reader_unit.py @@ -36,14 +36,13 @@ def test_mssql_reader_snapshot_error_pass_df_schema(spark_mock): ) -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_mssql_reader_wrong_table_name(spark_mock, table): +def test_mssql_reader_wrong_table_name(spark_mock): mssql = MSSQL(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBReader( connection=mssql, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_reader_unit/test_mysql_reader_unit.py b/tests/tests_unit/test_db/test_db_reader_unit/test_mysql_reader_unit.py index 8976393e3..41aeb8d77 100644 --- a/tests/tests_unit/test_db/test_db_reader_unit/test_mysql_reader_unit.py +++ b/tests/tests_unit/test_db/test_db_reader_unit/test_mysql_reader_unit.py @@ -36,14 +36,13 @@ def test_mysql_reader_snapshot_error_pass_df_schema(spark_mock): ) -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_mysql_reader_wrong_table_name(spark_mock, table): +def test_mysql_reader_wrong_table_name(spark_mock): mysql = MySQL(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBReader( connection=mysql, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_reader_unit/test_oracle_reader_unit.py b/tests/tests_unit/test_db/test_db_reader_unit/test_oracle_reader_unit.py index 444bc596f..60f40f6e1 100644 --- a/tests/tests_unit/test_db/test_db_reader_unit/test_oracle_reader_unit.py +++ b/tests/tests_unit/test_db/test_db_reader_unit/test_oracle_reader_unit.py @@ -36,13 +36,12 @@ def test_oracle_reader_error_df_schema(spark_mock): ) -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_oracle_reader_wrong_table_name(spark_mock, table): +def test_oracle_reader_wrong_table_name(spark_mock): oracle = Oracle(host="some_host", user="user", sid="sid", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBReader( connection=oracle, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_reader_unit/test_postgres_reader_unit.py b/tests/tests_unit/test_db/test_db_reader_unit/test_postgres_reader_unit.py index 03c646811..e973675fa 100644 --- a/tests/tests_unit/test_db/test_db_reader_unit/test_postgres_reader_unit.py +++ b/tests/tests_unit/test_db/test_db_reader_unit/test_postgres_reader_unit.py @@ -36,14 +36,13 @@ def test_postgres_reader_snapshot_error_pass_df_schema(spark_mock): ) -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_postgres_reader_wrong_table_name(spark_mock, table): +def test_postgres_reader_wrong_table_name(spark_mock): postgres = Postgres(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBReader( connection=postgres, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_reader_unit/test_teradata_reader_unit.py b/tests/tests_unit/test_db/test_db_reader_unit/test_teradata_reader_unit.py index c61abca72..f68ded1ae 100644 --- a/tests/tests_unit/test_db/test_db_reader_unit/test_teradata_reader_unit.py +++ b/tests/tests_unit/test_db/test_db_reader_unit/test_teradata_reader_unit.py @@ -36,14 +36,13 @@ def test_teradata_reader_snapshot_error_pass_df_schema(spark_mock): ) -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_teradata_reader_wrong_table_name(spark_mock, table): +def test_teradata_reader_wrong_table_name(spark_mock): teradata = Teradata(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBReader( connection=teradata, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_writer_unit/test_clickhouse_writer_unit.py b/tests/tests_unit/test_db/test_db_writer_unit/test_clickhouse_writer_unit.py index a631c95b7..801138e30 100644 --- a/tests/tests_unit/test_db/test_db_writer_unit/test_clickhouse_writer_unit.py +++ b/tests/tests_unit/test_db/test_db_writer_unit/test_clickhouse_writer_unit.py @@ -6,12 +6,11 @@ pytestmark = pytest.mark.clickhouse -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_clickhouse_writer_wrong_table_name(spark_mock, table): +def test_clickhouse_writer_wrong_table_name(spark_mock): clickhouse = Clickhouse(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBWriter( connection=clickhouse, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_writer_unit/test_greenplum_writer_unit.py b/tests/tests_unit/test_db/test_db_writer_unit/test_greenplum_writer_unit.py index 784d43580..0d79b2fc2 100644 --- a/tests/tests_unit/test_db/test_db_writer_unit/test_greenplum_writer_unit.py +++ b/tests/tests_unit/test_db/test_db_writer_unit/test_greenplum_writer_unit.py @@ -6,12 +6,11 @@ pytestmark = pytest.mark.greenplum -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_greenplum_writer_wrong_table_name(spark_mock, table): +def test_greenplum_writer_wrong_table_name(spark_mock): greenplum = Greenplum(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBWriter( connection=greenplum, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_writer_unit/test_hive_writer_unit.py b/tests/tests_unit/test_db/test_db_writer_unit/test_hive_writer_unit.py index c6c6ddc70..d96cd5210 100644 --- a/tests/tests_unit/test_db/test_db_writer_unit/test_hive_writer_unit.py +++ b/tests/tests_unit/test_db/test_db_writer_unit/test_hive_writer_unit.py @@ -6,12 +6,11 @@ pytestmark = pytest.mark.hive -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_hive_writer_wrong_table_name(spark_mock, table): +def test_hive_writer_wrong_table_name(spark_mock): hive = Hive(cluster="rnd-dwh", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBWriter( connection=hive, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_writer_unit/test_mssql_writer_unit.py b/tests/tests_unit/test_db/test_db_writer_unit/test_mssql_writer_unit.py index 0690e6a45..3bb30b655 100644 --- a/tests/tests_unit/test_db/test_db_writer_unit/test_mssql_writer_unit.py +++ b/tests/tests_unit/test_db/test_db_writer_unit/test_mssql_writer_unit.py @@ -6,12 +6,11 @@ pytestmark = pytest.mark.mssql -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_mssql_writer_wrong_table_name(spark_mock, table): +def test_mssql_writer_wrong_table_name(spark_mock): mssql = MSSQL(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBWriter( connection=mssql, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_writer_unit/test_mysql_writer_unit.py b/tests/tests_unit/test_db/test_db_writer_unit/test_mysql_writer_unit.py index fc00e96c3..c8b84c6a5 100644 --- a/tests/tests_unit/test_db/test_db_writer_unit/test_mysql_writer_unit.py +++ b/tests/tests_unit/test_db/test_db_writer_unit/test_mysql_writer_unit.py @@ -6,12 +6,11 @@ pytestmark = pytest.mark.mysql -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_mysql_writer_wrong_table_name(spark_mock, table): +def test_mysql_writer_wrong_table_name(spark_mock): mysql = MySQL(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBWriter( connection=mysql, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_writer_unit/test_oracle_writer_unit.py b/tests/tests_unit/test_db/test_db_writer_unit/test_oracle_writer_unit.py index ae53e4515..6ab40bcb7 100644 --- a/tests/tests_unit/test_db/test_db_writer_unit/test_oracle_writer_unit.py +++ b/tests/tests_unit/test_db/test_db_writer_unit/test_oracle_writer_unit.py @@ -6,12 +6,11 @@ pytestmark = pytest.mark.oracle -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_oracle_writer_wrong_table_name(spark_mock, table): +def test_oracle_writer_wrong_table_name(spark_mock): oracle = Oracle(host="some_host", user="user", sid="sid", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBWriter( connection=oracle, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_writer_unit/test_postgres_writer_unit.py b/tests/tests_unit/test_db/test_db_writer_unit/test_postgres_writer_unit.py index a2794466f..cecb6864d 100644 --- a/tests/tests_unit/test_db/test_db_writer_unit/test_postgres_writer_unit.py +++ b/tests/tests_unit/test_db/test_db_writer_unit/test_postgres_writer_unit.py @@ -6,12 +6,11 @@ pytestmark = pytest.mark.postgres -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_postgres_writer_wrong_table_name(spark_mock, table): +def test_postgres_writer_wrong_table_name(spark_mock): postgres = Postgres(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBWriter( connection=postgres, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" ) diff --git a/tests/tests_unit/test_db/test_db_writer_unit/test_teradata_writer_unit.py b/tests/tests_unit/test_db/test_db_writer_unit/test_teradata_writer_unit.py index 76bc48358..57bf81eac 100644 --- a/tests/tests_unit/test_db/test_db_writer_unit/test_teradata_writer_unit.py +++ b/tests/tests_unit/test_db/test_db_writer_unit/test_teradata_writer_unit.py @@ -6,12 +6,11 @@ pytestmark = pytest.mark.teradata -@pytest.mark.parametrize("table", ["table", "table.table.table"]) -def test_teradata_writer_wrong_table_name(spark_mock, table): +def test_teradata_writer_wrong_table_name(spark_mock): teradata = Teradata(host="some_host", user="user", database="database", password="passwd", spark=spark_mock) with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"): DBWriter( connection=teradata, - table=table, # Required format: table="schema.table" + table="table", # Required format: table="schema.table" )