Merge pull request #288 from dyvenia/dev

Release 0.3.2

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
+## [0.3.2] - 2022-02-17
+### Fixed
+- fixed an issue with schema info within `CheckColumnOrder` class. 
 ## [0.3.1] - 2022-02-17
 ### Changed
 -`ADLSToAzureSQL` - added `remove_tab`  parameter to remove uncessery tab separators from data. 

tests/integration/tasks/test_azure_sql.py

@@ -84,7 +84,7 @@ def test_check_column_order_append_same_col_number(caplog):
 
     check_column_order = CheckColumnOrder()
     with caplog.at_level(logging.WARNING):
-        check_column_order.run(table=TABLE, if_exists="append", df=df)
+        check_column_order.run(table=TABLE, schema=SCHEMA, if_exists="append", df=df)
 
     assert (
         "Detected column order difference between the CSV file and the table. Reordering..."

tests/test_viadot.py

@@ -2,4 +2,4 @@
 
 
 def test_version():
-    assert __version__ == "0.3.1"
+    assert __version__ == "0.3.2"

viadot/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.1"
+__version__ = "0.3.2"

viadot/flows/adls_to_azure_sql.py

@@ -232,6 +232,7 @@ def gen_flow(self) -> Flow:
 
         df_reorder = check_column_order_task.bind(
             table=self.table,
+            schema=self.schema,
             df=df,
             if_exists=self.if_exists,
             credentials_secret=self.sqldb_credentials_secret,

viadot/tasks/azure_sql.py

@@ -289,6 +289,7 @@ class CheckColumnOrder(Task):
     def __init__(
         self,
         table: str = None,
+        schema: str = None,
         if_exists: Literal["fail", "replace", "append", "delete"] = "replace",
         df: pd.DataFrame = None,
         credentials_secret: str = None,
@@ -317,6 +318,7 @@ def df_change_order(
     def run(
         self,
         table: str = None,
+        schema: str = None,
         if_exists: Literal["fail", "replace", "append", "delete"] = "replace",
         df: pd.DataFrame = None,
         credentials_secret: str = None,
@@ -327,6 +329,7 @@ def run(
 
         Args:
             table (str, optional): SQL table name without schema. Defaults to None.
+            schema (str, optional): SQL schema name. Defaults to None.
             if_exists (Literal, optional): What to do if the table exists. Defaults to "replace".
             df (pd.DataFrame, optional): Data Frame. Defaults to None.
             credentials_secret (str, optional): The name of the Azure Key Vault secret containing a dictionary
@@ -337,7 +340,7 @@ def run(
         azure_sql = AzureSQL(credentials=credentials)
 
         if if_exists not in ["replace", "fail"]:
-            query = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '{table}'"
+            query = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{schema}' AND TABLE_NAME = '{table}'"
             result = azure_sql.run(query=query)
             sql_column_list = [table for row in result for table in row]
             df_column_list = list(df.columns)
@@ -347,7 +350,6 @@ def run(
                     "Detected column order difference between the CSV file and the table. Reordering..."
                 )
                 df = self.df_change_order(df=df, sql_column_list=sql_column_list)
-                print(df)
             else:
                 return df
         else: