Fix tests before release

tests/integration/flows/test_bigquery_to_adls.py

@@ -101,13 +101,9 @@ def test_bigquery_to_adls_validate_df_fail(mocked_data):
         adls_sp_credentials_secret=ADLS_CREDENTIAL_SECRET,
         validate_df_dict={"column_list_to_match": ["type", "country", "test"]},
     )
-    try:
-        result = flow_bigquery.run()
-    except ValidationError:
-        pass
 
-    os.remove("test_bigquery_to_adls_validate_df_fail.parquet")
-    os.remove("test_bigquery_to_adls_validate_df_fail.json")
+    result = flow_bigquery.run()
+    assert result.is_failed()
 
 
 @mock.patch(
@@ -138,7 +134,5 @@ def test_bigquery_to_adls_validate_df_success(mocked_data):
     os.remove("test_bigquery_to_adls_validate_df_success.parquet")
     os.remove("test_bigquery_to_adls_validate_df_success.json")
 
-    rm = AzureDataLakeRemove(
-        path=ADLS_DIR_PATH + ADLS_FILE_NAME, vault_name="azuwevelcrkeyv001s"
-    )
+    rm = AzureDataLakeRemove(path=ADLS_DIR_PATH + ADLS_FILE_NAME)
     rm.run(sp_credentials_secret=ADLS_CREDENTIAL_SECRET)

tests/integration/flows/test_mysql_to_adls.py

@@ -18,7 +18,6 @@ def test_adls_gen1_to_azure_sql_new_mock(TEST_PARQUET_FILE_PATH):
             query=query,
             file_path=TEST_PARQUET_FILE_PATH,
             to_path=f"raw/examples/{TEST_PARQUET_FILE_PATH}",
-            sp_credentials_secret="App-Azure-CR-DatalakeGen2-AIA-DEV",
             overwrite_adls=True,
         )
         flow.run()
@@ -32,7 +31,6 @@ def test_validate_df(TEST_PARQUET_FILE_PATH):
             country_short="DE",
             query=query,
             file_path=TEST_PARQUET_FILE_PATH,
-            sp_credentials_secret="App-Azure-CR-DatalakeGen2-AIA",
             to_path=f"raw/examples/{TEST_PARQUET_FILE_PATH}",
             validate_df_dict={"column_size": {"sales_org": 3}},
         )

tests/integration/flows/test_salesforce_to_adls.py

@@ -30,7 +30,6 @@ def test_salesforce_to_adls():
     os.remove("test_salesforce_to_adls_run_flow.json")
     rm = AzureDataLakeRemove(
         path=ADLS_DIR_PATH + ADLS_FILE_NAME,
-        vault_name="azuwevelcrkeyv001s",
     )
     rm.run(sp_credentials_secret=credentials_secret)
 
@@ -56,6 +55,5 @@ def test_salesforce_to_adls_validate_success():
     os.remove("test_salesforce_to_adls_run_flow.json")
     rm = AzureDataLakeRemove(
         path=ADLS_DIR_PATH + ADLS_FILE_NAME,
-        vault_name="azuwevelcrkeyv001s",
     )
     rm.run(sp_credentials_secret=credentials_secret)

tests/integration/flows/test_vidclub_to_adls.py

@@ -47,7 +47,6 @@ def test_vidclub_validate_df_task_success(caplog):
         to_date="2023-10-25",
         adls_dir_path="raw/tests",
         adls_file_name="test.parquet",
-        adls_sp_credentials_secret="App-Azure-CR-DatalakeGen2-AIA",
         overwrite_adls=True,
         validate_df_dict={
             "column_size": {"submissionID": 5},
@@ -78,15 +77,12 @@ def test_vidclub_validate_df_task_fail(caplog):
         to_date="2023-10-25",
         adls_dir_path="raw/tests",
         adls_file_name="test.parquet",
-        adls_sp_credentials_secret="App-Azure-CR-DatalakeGen2-AIA",
         overwrite_adls=True,
         validate_df_dict={
             "column_size": {"submissionID": 5},
-            "column_unique_values": ["regionID"],
+            "column_unique_values": ["id"],
         },
     )
 
-    try:
-        flow.run()
-    except ValidationError:
-        pass
+    result = flow.run()
+    assert result.is_failed()

tests/integration/tasks/test_vid_club.py

@@ -56,7 +56,7 @@ def test_drop_columns(var_dictionary):
     Args:
         var_dictionary: Dictionary with example arguments for run method.
     """
-    cols_to_drop = ["regionID", "submissionDate"]
+    cols_to_drop = ["__v", "status"]
     vc_to_df = VidClubToDF(credentials=CREDENTIALS)
 
     output_with_dropped = vc_to_df.run(