Skip to content

Commit

Permalink
refactor(ingestion): remove company domain for security reason (datah…
Browse files Browse the repository at this point in the history 
…ub-project#10839)
  • Loading branch information
@shubhamjagtap639
shubhamjagtap639 authored Jul 9, 2024
1 parent 8722878 commit b6c7fe8
Show file tree
Hide file tree
Showing 20 changed files with 86 additions and 76 deletions.
4 changes: 2 additions & 2 deletions metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down Expand Up @@ -1709,7 +1709,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_cll.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -483,7 +483,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
4 changes: 2 additions & 2 deletions metadata-ingestion/tests/integration/powerbi/golden_test_container.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -780,7 +780,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down Expand Up @@ -2213,7 +2213,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
4 changes: 2 additions & 2 deletions metadata-ingestion/tests/integration/powerbi/golden_test_report.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down Expand Up @@ -1493,7 +1493,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
"name": "job-history",
"source": [
{
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
}
],
"datasourceUsages": [
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/test_m_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
'let\n Source = Sql.Database("AUPRDWHDB", "COMMOPSDB", [Query="Select#(lf)*,#(lf)concat((UPPER(REPLACE(SALES_SPECIALIST,\'-\',\'\'))),#(lf)LEFT(CAST(INVOICE_DATE AS DATE),4)+LEFT(RIGHT(CAST(INVOICE_DATE AS DATE),5),2)) AS AGENT_KEY,#(lf)CASE#(lf) WHEN CLASS = \'Software\' and (NOT(PRODUCT in (\'ADV\', \'Adv\') and left(ACCOUNT_ID,2)=\'10\') #(lf) or V_ENTERPRISE_INVOICED_REVENUE.TYPE = \'Manual Adjustment\') THEN INVOICE_AMOUNT#(lf) WHEN V_ENTERPRISE_INVOICED_REVENUE.TYPE IN (\'Recurring\',\'0\') THEN INVOICE_AMOUNT#(lf) ELSE 0#(lf)END as SOFTWARE_INV#(lf)#(lf)from V_ENTERPRISE_INVOICED_REVENUE", CommandTimeout=#duration(0, 1, 30, 0)]),\n #"Added Conditional Column" = Table.AddColumn(Source, "Services", each if [CLASS] = "Services" then [INVOICE_AMOUNT] else 0),\n #"Added Custom" = Table.AddColumn(#"Added Conditional Column", "Advanced New Sites", each if [PRODUCT] = "ADV"\nor [PRODUCT] = "Adv"\nthen [NEW_SITE]\nelse 0)\nin\n #"Added Custom"',
'let\n Source = Snowflake.Databases("ghh48144.snowflakefakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]),\n Source2 = PostgreSQL.Database("localhost", "mics"),\n public_order_date = Source2{[Schema="public",Item="order_date"]}[Data],\n GSL_TEST_DB_Database = Source{[Name="GSL_TEST_DB",Kind="Database"]}[Data],\n PUBLIC_Schema = GSL_TEST_DB_Database{[Name="PUBLIC",Kind="Schema"]}[Data],\n SALES_ANALYST_VIEW_View = PUBLIC_Schema{[Name="SALES_ANALYST_VIEW",Kind="View"]}[Data],\n two_source_table = Table.Combine({public_order_date, SALES_ANALYST_VIEW_View})\n in\n two_source_table',
'let\n Source = PostgreSQL.Database("localhost" , "mics" ),\n public_order_date = Source{[Schema="public",Item="order_date"]}[Data] \n in \n public_order_date',
'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
'let\n Source = Sql.Database("localhost", "library"),\n dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]\n in dbo_book_issue',
'let\n Source = Snowflake.Databases("ghh48144.snowflakefakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]),\n GSL_TEST_DB_Database = Source{[Name="GSL_TEST_DB",Kind="Database"]}[Data],\n PUBLIC_Schema = GSL_TEST_DB_Database{[Name="PUBLIC",Kind="Schema"]}[Data],\n SALES_FORECAST_Table = PUBLIC_Schema{[Name="SALES_FORECAST",Kind="Table"]}[Data],\n SALES_ANALYST_Table = PUBLIC_Schema{[Name="SALES_ANALYST",Kind="Table"]}[Data],\n RESULT = Table.Combine({SALES_FORECAST_Table, SALES_ANALYST_Table})\n\nin\n RESULT',
'let\n Source = GoogleBigQuery.Database(),\n #"seraphic-music-344307" = Source{[Name="seraphic-music-344307"]}[Data],\n school_dataset_Schema = #"seraphic-music-344307"{[Name="school_dataset",Kind="Schema"]}[Data],\n first_Table = school_dataset_Schema{[Name="first",Kind="Table"]}[Data]\nin\n first_Table',
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/powerbi/test_powerbi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
"name": "job-history",
"source": [
{
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
}
],
"datasourceUsages": [
Expand Down
25 changes: 15 additions & 10 deletions metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,19 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
}
}
},
"systemMetadata": {
"lastObserved": 1707327546106,
"lastObserved": 1720073671135,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
Expand Down Expand Up @@ -319,18 +320,19 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
}
}
},
"systemMetadata": {
"lastObserved": 1707327546129,
"lastObserved": 1720073671192,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
Expand Down Expand Up @@ -469,18 +471,19 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
}
}
},
"systemMetadata": {
"lastObserved": 1707327546135,
"lastObserved": 1720073671209,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
Expand Down Expand Up @@ -1217,18 +1220,19 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
}
}
},
"systemMetadata": {
"lastObserved": 1707327546161,
"lastObserved": 1720073671300,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
Expand Down Expand Up @@ -1414,18 +1418,19 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
}
}
},
"systemMetadata": {
"lastObserved": 1707327546168,
"lastObserved": 1720073671321,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
Expand Down
Loading

0 comments on commit b6c7fe8

Please sign in to comment.