Skip to content

Commit

Permalink
[DOP-21634] Make DBReader and DBWriter table name requirements less s…
Browse files Browse the repository at this point in the history
…trict
  • Loading branch information
dolfinus committed Nov 21, 2024
1 parent 4b1cde7 commit f03d936
Show file tree
Hide file tree
Showing 17 changed files with 33 additions and 49 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

class SupportNameWithSchemaOnly:
def validate_name(self, value: str) -> str:
if value.count(".") != 1:
if "." not in value:
raise ValueError(
f"Name should be passed in `schema.name` format, got '{value}'",
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,13 @@ def test_clickhouse_reader_snapshot_error_pass_df_schema(spark_mock):
)


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_clickhouse_reader_wrong_table_name(spark_mock, table):
def test_clickhouse_reader_wrong_table_name(spark_mock):
clickhouse = Clickhouse(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBReader(
connection=clickhouse,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,13 @@ def test_greenplum_reader_snapshot_error_pass_df_schema(spark_mock):
)


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_greenplum_reader_wrong_table_name(spark_mock, table):
def test_greenplum_reader_wrong_table_name(spark_mock):
greenplum = Greenplum(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBReader(
connection=greenplum,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,13 @@ def test_hive_reader_snapshot_error_pass_df_schema(spark_mock):
)


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_hive_reader_wrong_table_name(spark_mock, table):
def test_hive_reader_wrong_table_name(spark_mock):
hive = Hive(cluster="rnd-dwh", spark=spark_mock)

with pytest.raises(ValueError):
DBReader(
connection=hive,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,13 @@ def test_mssql_reader_snapshot_error_pass_df_schema(spark_mock):
)


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_mssql_reader_wrong_table_name(spark_mock, table):
def test_mssql_reader_wrong_table_name(spark_mock):
mssql = MSSQL(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBReader(
connection=mssql,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,13 @@ def test_mysql_reader_snapshot_error_pass_df_schema(spark_mock):
)


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_mysql_reader_wrong_table_name(spark_mock, table):
def test_mysql_reader_wrong_table_name(spark_mock):
mysql = MySQL(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBReader(
connection=mysql,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,12 @@ def test_oracle_reader_error_df_schema(spark_mock):
)


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_oracle_reader_wrong_table_name(spark_mock, table):
def test_oracle_reader_wrong_table_name(spark_mock):
oracle = Oracle(host="some_host", user="user", sid="sid", password="passwd", spark=spark_mock)
with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBReader(
connection=oracle,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,13 @@ def test_postgres_reader_snapshot_error_pass_df_schema(spark_mock):
)


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_postgres_reader_wrong_table_name(spark_mock, table):
def test_postgres_reader_wrong_table_name(spark_mock):
postgres = Postgres(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBReader(
connection=postgres,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,13 @@ def test_teradata_reader_snapshot_error_pass_df_schema(spark_mock):
)


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_teradata_reader_wrong_table_name(spark_mock, table):
def test_teradata_reader_wrong_table_name(spark_mock):
teradata = Teradata(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBReader(
connection=teradata,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
pytestmark = pytest.mark.clickhouse


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_clickhouse_writer_wrong_table_name(spark_mock, table):
def test_clickhouse_writer_wrong_table_name(spark_mock):
clickhouse = Clickhouse(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBWriter(
connection=clickhouse,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
pytestmark = pytest.mark.greenplum


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_greenplum_writer_wrong_table_name(spark_mock, table):
def test_greenplum_writer_wrong_table_name(spark_mock):
greenplum = Greenplum(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBWriter(
connection=greenplum,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
pytestmark = pytest.mark.hive


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_hive_writer_wrong_table_name(spark_mock, table):
def test_hive_writer_wrong_table_name(spark_mock):
hive = Hive(cluster="rnd-dwh", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBWriter(
connection=hive,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
pytestmark = pytest.mark.mssql


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_mssql_writer_wrong_table_name(spark_mock, table):
def test_mssql_writer_wrong_table_name(spark_mock):
mssql = MSSQL(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBWriter(
connection=mssql,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
pytestmark = pytest.mark.mysql


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_mysql_writer_wrong_table_name(spark_mock, table):
def test_mysql_writer_wrong_table_name(spark_mock):
mysql = MySQL(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBWriter(
connection=mysql,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
pytestmark = pytest.mark.oracle


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_oracle_writer_wrong_table_name(spark_mock, table):
def test_oracle_writer_wrong_table_name(spark_mock):
oracle = Oracle(host="some_host", user="user", sid="sid", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBWriter(
connection=oracle,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
pytestmark = pytest.mark.postgres


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_postgres_writer_wrong_table_name(spark_mock, table):
def test_postgres_writer_wrong_table_name(spark_mock):
postgres = Postgres(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBWriter(
connection=postgres,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
pytestmark = pytest.mark.teradata


@pytest.mark.parametrize("table", ["table", "table.table.table"])
def test_teradata_writer_wrong_table_name(spark_mock, table):
def test_teradata_writer_wrong_table_name(spark_mock):
teradata = Teradata(host="some_host", user="user", database="database", password="passwd", spark=spark_mock)

with pytest.raises(ValueError, match="Name should be passed in `schema.name` format"):
DBWriter(
connection=teradata,
table=table, # Required format: table="schema.table"
table="table", # Required format: table="schema.table"
)

0 comments on commit f03d936

Please sign in to comment.