diff --git a/README.md b/README.md index 8701e90..35084f1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Semantic Link Labs [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs) -[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.8.9&style=flat)](https://readthedocs.org/projects/semantic-link-labs/) +[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.8.10&style=flat)](https://readthedocs.org/projects/semantic-link-labs/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs) @@ -116,6 +116,7 @@ An even better way to ensure the semantic-link-labs library is available in your 2. Select your newly created environment within the 'Environment' drop down in the navigation bar at the top of the notebook ## Version History +* [0.8.10](https://github.com/microsoft/semantic-link-labs/releases/tag/0.8.10) (December 16, 2024) * [0.8.9](https://github.com/microsoft/semantic-link-labs/releases/tag/0.8.9) (December 4, 2024) * [0.8.8](https://github.com/microsoft/semantic-link-labs/releases/tag/0.8.8) (November 28, 2024) * [0.8.7](https://github.com/microsoft/semantic-link-labs/releases/tag/0.8.7) (November 27, 2024) diff --git a/docs/source/conf.py b/docs/source/conf.py index 86640c0..cbee739 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,7 +13,7 @@ project = 'semantic-link-labs' copyright = '2024, Microsoft and community' author = 'Microsoft and community' -release = '0.8.9' +release = '0.8.10' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/pyproject.toml b/pyproject.toml index 9b921e0..63e97bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name="semantic-link-labs" authors = [ { name = "Microsoft Corporation" }, ] -version="0.8.9" +version="0.8.10" description="Semantic Link Labs for Microsoft Fabric" readme="README.md" requires-python=">=3.10,<3.12" diff --git a/src/sempy_labs/__init__.py b/src/sempy_labs/__init__.py index e0bf0de..a2b99f4 100644 --- a/src/sempy_labs/__init__.py +++ b/src/sempy_labs/__init__.py @@ -8,6 +8,7 @@ create_vnet_gateway, update_vnet_gateway, update_on_premises_gateway, + bind_semantic_model_to_gateway, ) from sempy_labs._authentication import ( @@ -205,6 +206,8 @@ list_lakehouses, list_sql_endpoints, update_item, + list_server_properties, + list_semantic_model_errors, ) from sempy_labs._helper_functions import ( convert_to_friendly_case, @@ -230,6 +233,7 @@ get_capacity_id, get_capacity_name, resolve_capacity_name, + get_tenant_id, ) from sempy_labs._model_bpa_bulk import ( run_model_bpa_bulk, @@ -458,4 +462,8 @@ "update_vnet_gateway", "update_on_premises_gateway", "get_semantic_model_definition", + "get_tenant_id", + "list_server_properties", + "bind_semantic_model_to_gateway", + "list_semantic_model_errors", ] diff --git a/src/sempy_labs/_authentication.py b/src/sempy_labs/_authentication.py index 340eecf..c1b3731 100644 --- a/src/sempy_labs/_authentication.py +++ b/src/sempy_labs/_authentication.py @@ -91,11 +91,13 @@ def from_azure_key_vault( return cls(credential) - def __call__(self, audience: Literal["pbi", "storage"] = "pbi") -> str: + def __call__( + self, audience: Literal["pbi", "storage", "azure", "graph"] = "pbi" + ) -> str: """ Parameters ---------- - audience : Literal["pbi", "storage"] = "pbi") -> str + audience : Literal["pbi", "storage", "azure", "graph"] = "pbi") -> str Literal if it's for PBI/Fabric API call or OneLake/Storage Account call. """ if audience == "pbi": @@ -104,5 +106,32 @@ def __call__(self, audience: Literal["pbi", "storage"] = "pbi") -> str: ).token elif audience == "storage": return self.credential.get_token("https://storage.azure.com/.default").token + elif audience == "azure": + return self.credential.get_token( + "https://management.azure.com/.default" + ).token + elif audience == "graph": + return self.credential.get_token( + "https://graph.microsoft.com/.default" + ).token else: raise NotImplementedError + + +def _get_headers( + token_provider: str, audience: Literal["pbi", "storage", "azure", "graph"] = "azure" +): + """ + Generates headers for an API request. + """ + + token = token_provider(audience=audience) + + headers = {"Authorization": f"Bearer {token}"} + + if audience == "graph": + headers["ConsistencyLevel"] = "eventual" + else: + headers["Content-Type"] = "application/json" + + return headers diff --git a/src/sempy_labs/_gateways.py b/src/sempy_labs/_gateways.py index c8b9583..03ebaa6 100644 --- a/src/sempy_labs/_gateways.py +++ b/src/sempy_labs/_gateways.py @@ -6,6 +6,8 @@ pagination, _is_valid_uuid, resolve_capacity_id, + resolve_workspace_name_and_id, + resolve_dataset_name_and_id, ) from uuid import UUID import sempy_labs._icons as icons @@ -437,3 +439,47 @@ def update_vnet_gateway( raise FabricHTTPException(response) print(f"{icons.green_dot} The '{gateway}' has been updated accordingly.") + + +def bind_semantic_model_to_gateway( + dataset: str | UUID, gateway: str | UUID, workspace: Optional[str | UUID] = None +): + """ + Binds the specified dataset from the specified workspace to the specified gateway. + + This is a wrapper function for the following API: `Datasets - Bind To Gateway In Group `_. + + Parameters + ---------- + dataset : str | UUID + The name or ID of the semantic model. + gateway : str | UUID + The name or ID of the gateway. + workspace : str | UUID, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace) + (dataset_name, dataset_id) = resolve_dataset_name_and_id( + dataset, workspace=workspace_id + ) + + gateway_id = _resolve_gateway_id(gateway) + payload = { + "gatewayObjectId": gateway_id, + } + + client = fabric.FabricRestClient() + response = client.post( + f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/Default.BindToGateway", + json=payload, + ) + + if response.status_code != 200: + raise FabricHTTPException(response) + + print( + f"{icons.green_dot} The '{dataset_name}' semantic model within the '{workspace_name}' workspace has been binded to the '{gateway_id}' gateway." + ) diff --git a/src/sempy_labs/_git.py b/src/sempy_labs/_git.py index 959df93..4e58078 100644 --- a/src/sempy_labs/_git.py +++ b/src/sempy_labs/_git.py @@ -277,43 +277,48 @@ def commit_to_git( workspace, workspace_id = resolve_workspace_name_and_id(workspace) gs = get_git_status(workspace=workspace) - workspace_head = gs["Workspace Head"].iloc[0] + if not gs.empty: + workspace_head = gs["Workspace Head"].iloc[0] - if item_ids is None: - commit_mode = "All" - else: - commit_mode = "Selective" + if item_ids is None: + commit_mode = "All" + else: + commit_mode = "Selective" - if isinstance(item_ids, str): - item_ids = [item_ids] + if isinstance(item_ids, str): + item_ids = [item_ids] - request_body = { - "mode": commit_mode, - "workspaceHead": workspace_head, - "comment": comment, - } + request_body = { + "mode": commit_mode, + "workspaceHead": workspace_head, + "comment": comment, + } - if item_ids is not None: - request_body["items"] = [{"objectId": item_id} for item_id in item_ids] + if item_ids is not None: + request_body["items"] = [{"objectId": item_id} for item_id in item_ids] - client = fabric.FabricRestClient() - response = client.post( - f"/v1/workspaces/{workspace_id}/git/commitToGit", - json=request_body, - ) + client = fabric.FabricRestClient() + response = client.post( + f"/v1/workspaces/{workspace_id}/git/commitToGit", + json=request_body, + ) - if response.status_code not in [200, 202]: - raise FabricHTTPException(response) + if response.status_code not in [200, 202]: + raise FabricHTTPException(response) - lro(client, response) + lro(client=client, response=response, return_status_code=True) - if commit_mode == "All": - print( - f"{icons.green_dot} All items within the '{workspace}' workspace have been committed to Git." - ) + if commit_mode == "All": + print( + f"{icons.green_dot} All items within the '{workspace}' workspace have been committed to Git." + ) + else: + print( + f"{icons.green_dot} The {item_ids} items within the '{workspace}' workspace have been committed to Git." + ) else: print( - f"{icons.green_dot} The {item_ids} items ithin the '{workspace}' workspace have been committed to Git." + f"{icons.info} Git already up to date: no modified items found within the '{workspace}' workspace." ) diff --git a/src/sempy_labs/_helper_functions.py b/src/sempy_labs/_helper_functions.py index 13edf39..2ff51e5 100644 --- a/src/sempy_labs/_helper_functions.py +++ b/src/sempy_labs/_helper_functions.py @@ -780,13 +780,19 @@ def get_capacity_id(workspace: Optional[str] = None) -> UUID: The capacity Id. """ - workspace = fabric.resolve_workspace_name(workspace) - filter_condition = urllib.parse.quote(workspace) - dfW = fabric.list_workspaces(filter=f"name eq '{filter_condition}'") - if len(dfW) == 0: - raise ValueError(f"{icons.red_dot} The '{workspace}' does not exist'.") + if workspace is None: + capacity_id = _get_x_id(name="trident.capacity.id") + else: + + workspace = fabric.resolve_workspace_name(workspace) + filter_condition = urllib.parse.quote(workspace) + dfW = fabric.list_workspaces(filter=f"name eq '{filter_condition}'") + if len(dfW) == 0: + raise ValueError(f"{icons.red_dot} The '{workspace}' does not exist'.") + + capacity_id = dfW["Capacity Id"].iloc[0] - return dfW["Capacity Id"].iloc[0] + return capacity_id def get_capacity_name(workspace: Optional[str] = None) -> str: @@ -1371,3 +1377,15 @@ def _is_valid_uuid( return True except ValueError: return False + + +def _get_fabric_context_setting(name: str): + + from synapse.ml.internal_utils.session_utils import get_fabric_context + + return get_fabric_context().get(name) + + +def get_tenant_id(): + + _get_fabric_context_setting(name="trident.tenant.id") diff --git a/src/sempy_labs/_list_functions.py b/src/sempy_labs/_list_functions.py index f4157cc..60103a3 100644 --- a/src/sempy_labs/_list_functions.py +++ b/src/sempy_labs/_list_functions.py @@ -1575,3 +1575,148 @@ def list_semantic_model_object_report_usage( final_df.reset_index(drop=True, inplace=True) return final_df + + +def list_server_properties(workspace: Optional[str | UUID] = None) -> pd.DataFrame: + """ + Lists the `properties `_ of the Analysis Services instance. + + Parameters + ---------- + workspace : str, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + pandas.DataFrame + A pandas dataframe showing a list of the server properties. + """ + + tom_server = fabric.create_tom_server(readonly=True, workspace=workspace) + + rows = [ + { + "Name": sp.Name, + "Value": sp.Value, + "Default Value": sp.DefaultValue, + "Is Read Only": sp.IsReadOnly, + "Requires Restart": sp.RequiresRestart, + "Units": sp.Units, + "Category": sp.Category, + } + for sp in tom_server.ServerProperties + ] + + tom_server.Dispose() + df = pd.DataFrame(rows) + + bool_cols = ["Is Read Only", "Requires Restart"] + df[bool_cols] = df[bool_cols].astype(bool) + + return df + + +def list_semantic_model_errors( + dataset: str | UUID, workspace: Optional[str | UUID] +) -> pd.DataFrame: + """ + Shows a list of a semantic model's errors and their error messages (if they exist). + + Parameters + ---------- + dataset : str | UUID + Name or ID of the semantic model. + workspace : str | UUID, default=None + The Fabric workspace name or ID. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + pandas.DataFrame + A pandas dataframe showing a list of the errors and error messages for a given semantic model. + """ + + from sempy_labs.tom import connect_semantic_model + + (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace) + (dataset_name, dataset_id) = resolve_dataset_name_and_id( + dataset, workspace=workspace_id + ) + + error_rows = [] + + with connect_semantic_model( + dataset=dataset_id, workspace=workspace_id, readonly=True + ) as tom: + # Define mappings of TOM objects to object types and attributes + error_checks = [ + ("Column", tom.all_columns, lambda o: o.ErrorMessage), + ("Partition", tom.all_partitions, lambda o: o.ErrorMessage), + ( + "Partition - Data Coverage Expression", + tom.all_partitions, + lambda o: ( + o.DataCoverageDefinition.ErrorMessage + if o.DataCoverageDefinition + else "" + ), + ), + ("Row Level Security", tom.all_rls, lambda o: o.ErrorMessage), + ("Calculation Item", tom.all_calculation_items, lambda o: o.ErrorMessage), + ("Measure", tom.all_measures, lambda o: o.ErrorMessage), + ( + "Measure - Detail Rows Expression", + tom.all_measures, + lambda o: ( + o.DetailRowsDefinition.ErrorMessage + if o.DetailRowsDefinition + else "" + ), + ), + ( + "Measure - Format String Expression", + tom.all_measures, + lambda o: ( + o.FormatStringDefinition.ErrorMessage + if o.FormatStringDefinition + else "" + ), + ), + ( + "Calculation Group - Multiple or Empty Selection Expression", + tom.all_calculation_groups, + lambda o: ( + o.CalculationGroup.MultipleOrEmptySelectionExpression.ErrorMessage + if o.CalculationGroup.MultipleOrEmptySelectionExpression + else "" + ), + ), + ( + "Calculation Group - No Selection Expression", + tom.all_calculation_groups, + lambda o: ( + o.CalculationGroup.NoSelectionExpression.ErrorMessage + if o.CalculationGroup.NoSelectionExpression + else "" + ), + ), + ] + + # Iterate over all error checks + for object_type, getter, error_extractor in error_checks: + for obj in getter(): + error_message = error_extractor(obj) + if error_message: # Only add rows if there's an error message + error_rows.append( + { + "Object Type": object_type, + "Table Name": obj.Parent.Name, + "Object Name": obj.Name, + "Error Message": error_message, + } + ) + + return pd.DataFrame(error_rows) diff --git a/src/sempy_labs/_notebooks.py b/src/sempy_labs/_notebooks.py index b9d2422..afcec7d 100644 --- a/src/sempy_labs/_notebooks.py +++ b/src/sempy_labs/_notebooks.py @@ -10,6 +10,42 @@ _decode_b64, ) from sempy.fabric.exceptions import FabricHTTPException +import os + +_notebook_prefix = "notebook-content." + + +def _get_notebook_definition_base( + notebook_name: str, workspace: Optional[str] = None +) -> pd.DataFrame: + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + item_id = fabric.resolve_item_id( + item_name=notebook_name, type="Notebook", workspace=workspace + ) + client = fabric.FabricRestClient() + response = client.post( + f"v1/workspaces/{workspace_id}/notebooks/{item_id}/getDefinition", + ) + + result = lro(client, response).json() + + return pd.json_normalize(result["definition"]["parts"]) + + +def _get_notebook_type(notebook_name: str, workspace: Optional[str] = None) -> str: + + df_items = _get_notebook_definition_base( + notebook_name=notebook_name, workspace=workspace + ) + + file_path = df_items[df_items["path"].str.startswith(_notebook_prefix)][ + "path" + ].iloc[0] + + _, file_extension = os.path.splitext(file_path) + + return file_extension[1:] def get_notebook_definition( @@ -38,18 +74,10 @@ def get_notebook_definition( The notebook definition. """ - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - item_id = fabric.resolve_item_id( - item_name=notebook_name, type="Notebook", workspace=workspace + df_items = _get_notebook_definition_base( + notebook_name=notebook_name, workspace=workspace ) - client = fabric.FabricRestClient() - response = client.post( - f"v1/workspaces/{workspace_id}/notebooks/{item_id}/getDefinition", - ) - - result = lro(client, response).json() - df_items = pd.json_normalize(result["definition"]["parts"]) - df_items_filt = df_items[df_items["path"] == "notebook-content.py"] + df_items_filt = df_items[df_items["path"].str.startswith(_notebook_prefix)] payload = df_items_filt["payload"].iloc[0] if decode: @@ -115,9 +143,10 @@ def import_notebook_from_web( description=description, ) elif len(dfI_filt) > 0 and overwrite: - update_notebook_definition( - name=notebook_name, notebook_content=response.content, workspace=workspace - ) + print(f"{icons.info} Overwrite of notebooks is currently not supported.") + # update_notebook_definition( + # name=notebook_name, notebook_content=response.content, workspace=workspace + # ) else: raise ValueError( f"{icons.red_dot} The '{notebook_name}' already exists within the '{workspace}' workspace and 'overwrite' is set to False." @@ -127,6 +156,7 @@ def import_notebook_from_web( def create_notebook( name: str, notebook_content: str, + type: str = "py", description: Optional[str] = None, workspace: Optional[str] = None, ): @@ -139,6 +169,8 @@ def create_notebook( The name of the notebook to be created. notebook_content : str The Jupyter notebook content (not in Base64 format). + type : str, default="py" + The notebook type. description : str, default=None The description of the notebook. Defaults to None which does not place a description. @@ -158,7 +190,7 @@ def create_notebook( "format": "ipynb", "parts": [ { - "path": "notebook-content.py", + "path": f"{_notebook_prefix}.{type}", "payload": notebook_payload, "payloadType": "InlineBase64", } @@ -202,13 +234,13 @@ def update_notebook_definition( item_name=name, type="Notebook", workspace=workspace ) + type = _get_notebook_type(notebook_name=name, workspace=workspace_id) + request_body = { - "displayName": name, "definition": { - "format": "ipynb", "parts": [ { - "path": "notebook-content.py", + "path": f"{_notebook_prefix}.{type}", "payload": notebook_payload, "payloadType": "InlineBase64", } diff --git a/src/sempy_labs/admin/_scanner.py b/src/sempy_labs/admin/_scanner.py index 84f56f0..a20720a 100644 --- a/src/sempy_labs/admin/_scanner.py +++ b/src/sempy_labs/admin/_scanner.py @@ -16,12 +16,12 @@ def scan_workspaces( workspace: Optional[str | List[str] | UUID | List[UUID]] = None, ) -> dict: """ - Get the inventory and details of the tenant. + Gets the scan result for the specified scan. This is a wrapper function for the following APIs: - `Admin - WorkspaceInfo PostWorkspaceInfo `_. - `Admin - WorkspaceInfo GetScanStatus `_. - `Admin - WorkspaceInfo GetScanResult `_. + `Admin - WorkspaceInfo PostWorkspaceInfo `_. + `Admin - WorkspaceInfo GetScanStatus `_. + `Admin - WorkspaceInfo GetScanResult `_. Parameters ----------