From 90fe4e56d7cb48c11689668b1879c215988be7b0 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 19 Sep 2024 08:19:52 +0300 Subject: [PATCH 1/3] DLOL --- .../directlake/_generate_shared_expression.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/sempy_labs/directlake/_generate_shared_expression.py b/src/sempy_labs/directlake/_generate_shared_expression.py index bdc05959..a8daa4f4 100644 --- a/src/sempy_labs/directlake/_generate_shared_expression.py +++ b/src/sempy_labs/directlake/_generate_shared_expression.py @@ -13,6 +13,7 @@ def generate_shared_expression( item_name: Optional[str] = None, item_type: Optional[str] = "Lakehouse", workspace: Optional[str] = None, + direct_lake_over_onelake: Optional[bool] = False, ) -> str: """ Dynamically generates the M expression used by a Direct Lake model for a given lakehouse/warehouse. @@ -28,6 +29,8 @@ def generate_shared_expression( The Fabric workspace used by the item. Defaults to None which resolves to the workspace of the attached lakehouse or if no lakehouse attached, resolves to the workspace of the notebook. + direct_lake_over_onelake : bool, defualt=False + Generates an expression required for a Direct Lake over OneLake semantic mode. Only available for lakehouses, not warehouses. Returns ------- @@ -44,6 +47,9 @@ def generate_shared_expression( f"{icons.red_dot} Invalid item type. Valid options: {item_types}." ) + if item_type == 'Warehouse' and direct_lake_over_onelake: + raise ValueError(f"{icons.red_dot} Direct Lake over OneLake is only supported for lakehouses, not warehouses.") + if item_name is None: item_id = fabric.get_lakehouse_id() item_name = resolve_lakehouse_name(item_id, workspace) @@ -58,14 +64,15 @@ def generate_shared_expression( if response.status_code != 200: raise FabricHTTPException(response) + prop = response.json().get("properties") + if item_type == "Lakehouse": - prop = response.json()["properties"]["sqlEndpointProperties"] - sqlEPCS = prop["connectionString"] - sqlepid = prop["id"] - provStatus = prop["provisioningStatus"] + sqlprop = prop.get("sqlEndpointProperties") + sqlEPCS = sqlprop.get("connectionString") + sqlepid = sqlprop.get("id") + provStatus = sqlprop.get("provisioningStatus") elif item_type == "Warehouse": - prop = response.json()["properties"] - sqlEPCS = prop["connectionString"] + sqlEPCS = prop.get("connectionString") sqlepid = item_id provStatus = None @@ -74,12 +81,14 @@ def generate_shared_expression( f"{icons.red_dot} The SQL Endpoint for the '{item_name}' lakehouse within the '{workspace}' workspace has not yet been provisioned. Please wait until it has been provisioned." ) - sh = ( - 'let\n\tdatabase = Sql.Database("' - + sqlEPCS - + '", "' - + sqlepid - + '")\nin\n\tdatabase' - ) + start_expr = 'let\n\tdatabase = ' + end_expr = '\nin\n\tdatabase' + if not direct_lake_over_onelake: + mid_expr = f'Sql.Database("{sqlEPCS}", "{sqlepid}")' + else: + url = prop.get('oneLakeTablesPath').rstrip('/Tables') + mid_expr = f'AzureStorage.DataLake(\\"{url}\\")"' + + sh = f"{start_expr}{mid_expr}{end_expr}" return sh From dc705e986b7c218fd988539089cc51cfb61d9712 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 23 Sep 2024 20:48:15 +0200 Subject: [PATCH 2/3] undo merge --- .../directlake/_generate_shared_expression.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/sempy_labs/directlake/_generate_shared_expression.py b/src/sempy_labs/directlake/_generate_shared_expression.py index bdc05959..a8daa4f4 100644 --- a/src/sempy_labs/directlake/_generate_shared_expression.py +++ b/src/sempy_labs/directlake/_generate_shared_expression.py @@ -13,6 +13,7 @@ def generate_shared_expression( item_name: Optional[str] = None, item_type: Optional[str] = "Lakehouse", workspace: Optional[str] = None, + direct_lake_over_onelake: Optional[bool] = False, ) -> str: """ Dynamically generates the M expression used by a Direct Lake model for a given lakehouse/warehouse. @@ -28,6 +29,8 @@ def generate_shared_expression( The Fabric workspace used by the item. Defaults to None which resolves to the workspace of the attached lakehouse or if no lakehouse attached, resolves to the workspace of the notebook. + direct_lake_over_onelake : bool, defualt=False + Generates an expression required for a Direct Lake over OneLake semantic mode. Only available for lakehouses, not warehouses. Returns ------- @@ -44,6 +47,9 @@ def generate_shared_expression( f"{icons.red_dot} Invalid item type. Valid options: {item_types}." ) + if item_type == 'Warehouse' and direct_lake_over_onelake: + raise ValueError(f"{icons.red_dot} Direct Lake over OneLake is only supported for lakehouses, not warehouses.") + if item_name is None: item_id = fabric.get_lakehouse_id() item_name = resolve_lakehouse_name(item_id, workspace) @@ -58,14 +64,15 @@ def generate_shared_expression( if response.status_code != 200: raise FabricHTTPException(response) + prop = response.json().get("properties") + if item_type == "Lakehouse": - prop = response.json()["properties"]["sqlEndpointProperties"] - sqlEPCS = prop["connectionString"] - sqlepid = prop["id"] - provStatus = prop["provisioningStatus"] + sqlprop = prop.get("sqlEndpointProperties") + sqlEPCS = sqlprop.get("connectionString") + sqlepid = sqlprop.get("id") + provStatus = sqlprop.get("provisioningStatus") elif item_type == "Warehouse": - prop = response.json()["properties"] - sqlEPCS = prop["connectionString"] + sqlEPCS = prop.get("connectionString") sqlepid = item_id provStatus = None @@ -74,12 +81,14 @@ def generate_shared_expression( f"{icons.red_dot} The SQL Endpoint for the '{item_name}' lakehouse within the '{workspace}' workspace has not yet been provisioned. Please wait until it has been provisioned." ) - sh = ( - 'let\n\tdatabase = Sql.Database("' - + sqlEPCS - + '", "' - + sqlepid - + '")\nin\n\tdatabase' - ) + start_expr = 'let\n\tdatabase = ' + end_expr = '\nin\n\tdatabase' + if not direct_lake_over_onelake: + mid_expr = f'Sql.Database("{sqlEPCS}", "{sqlepid}")' + else: + url = prop.get('oneLakeTablesPath').rstrip('/Tables') + mid_expr = f'AzureStorage.DataLake(\\"{url}\\")"' + + sh = f"{start_expr}{mid_expr}{end_expr}" return sh From 7f8a7505d136e7b2e6a1a056a63e27a365226684 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 23 Sep 2024 20:50:15 +0200 Subject: [PATCH 3/3] undo commit --- src/sempy_labs/_spark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sempy_labs/_spark.py b/src/sempy_labs/_spark.py index 39297ab4..fec4ed50 100644 --- a/src/sempy_labs/_spark.py +++ b/src/sempy_labs/_spark.py @@ -363,10 +363,10 @@ def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame: "High Concurrency Enabled", "Customize Compute Enabled", ] - int_cols = ["Max Node Count", "Max Executors"] + # int_cols = ["Max Node Count", "Max Executors"] df[bool_cols] = df[bool_cols].astype(bool) - df[int_cols] = df[int_cols].astype(int) + # df[int_cols] = df[int_cols].astype(int) return df