From 40699e0caad177a246681a2d5c6632504ae51f73 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 12 Sep 2024 12:54:26 +0300 Subject: [PATCH 1/4] added functions for environments --- src/sempy_labs/__init__.py | 9 +- src/sempy_labs/_environments.py | 126 ++++++++++++++++++++++++++++ src/sempy_labs/_helper_functions.py | 1 + src/sempy_labs/_list_functions.py | 8 +- src/sempy_labs/_model_bpa.py | 2 +- src/sempy_labs/tom/_model.py | 6 +- 6 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 src/sempy_labs/_environments.py diff --git a/src/sempy_labs/__init__.py b/src/sempy_labs/__init__.py index 4c212519..e4e3e96b 100644 --- a/src/sempy_labs/__init__.py +++ b/src/sempy_labs/__init__.py @@ -1,4 +1,9 @@ -from sempy_labs._sql import( +from sempy_labs._environments import ( + create_environment, + delete_environment, +) + +from sempy_labs._sql import ( ConnectWarehouse, ) @@ -262,4 +267,6 @@ "update_from_git", "connect_workspace_to_git", "disconnect_workspace_from_git", + "create_environment", + "delete_environment", ] diff --git a/src/sempy_labs/_environments.py b/src/sempy_labs/_environments.py new file mode 100644 index 00000000..9190bdf6 --- /dev/null +++ b/src/sempy_labs/_environments.py @@ -0,0 +1,126 @@ +import sempy.fabric as fabric +import pandas as pd +import sempy_labs._icons as icons +from typing import Optional +from sempy_labs._helper_functions import ( + resolve_workspace_name_and_id, + lro, + pagination, +) +from sempy.fabric.exceptions import FabricHTTPException + + +def create_environment( + environment: str, description: Optional[str] = None, workspace: Optional[str] = None +): + """ + Creates a Fabric environment. + + Parameters + ---------- + environment: str + Name of the environment. + description : str, default=None + A description of the environment. + workspace : str, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + request_body = {"displayName": environment} + + if description: + request_body["description"] = description + + client = fabric.FabricRestClient() + response = client.post( + f"/v1/workspaces/{workspace_id}/environments", json=request_body + ) + + lro(client, response, status_codes=[201, 202]) + + print( + f"{icons.green_dot} The '{environment}' environment has been created within the '{workspace}' workspace." + ) + + +def list_environments(workspace: Optional[str] = None) -> pd.DataFrame: + """ + Shows the environments within a workspace. + + Parameters + ---------- + workspace : str, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + pandas.DataFrame + A pandas dataframe showing the environments within a workspace. + """ + + df = pd.DataFrame( + columns=["Environment Name", "Environment Id", "Description"] + ) + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + client = fabric.FabricRestClient() + response = client.get(f"/v1/workspaces/{workspace_id}/environments") + if response.status_code != 200: + raise FabricHTTPException(response) + + responses = pagination(client, response) + + for r in responses: + for v in r.get("value", []): + new_data = { + "Environment Name": v.get("displayName"), + "Environment Id": v.get("id"), + "Description": v.get("description"), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + + return df + + +def delete_environment( + environment: str, workspace: Optional[str] = None +): + """ + Deletes a Fabric environment. + + Parameters + ---------- + environment: str + Name of the environment. + workspace : str, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + dfE = list_environments(workspace=workspace) + dfE_filt = dfE[dfE['Environment Name'] == environment] + if len(dfE_filt) == 0: + raise ValueError(f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace.") + environment_id = dfE_filt['Environment Id'].iloc[0] + + client = fabric.FabricRestClient() + response = client.delete( + f"/v1/workspaces/{workspace_id}/environments/{environment_id}" + ) + + if response.status_code != 200: + raise FabricHTTPException(response) + + print( + f"{icons.green_dot} The '{environment}' environment within the '{workspace}' workspace has been deleted." + ) diff --git a/src/sempy_labs/_helper_functions.py b/src/sempy_labs/_helper_functions.py index ec6b71e0..8864e4e7 100644 --- a/src/sempy_labs/_helper_functions.py +++ b/src/sempy_labs/_helper_functions.py @@ -883,6 +883,7 @@ def get_token( ) -> AccessToken: from notebookutils import mssparkutils + token = mssparkutils.credentials.getToken(scopes) access_token = AccessToken(token, 0) diff --git a/src/sempy_labs/_list_functions.py b/src/sempy_labs/_list_functions.py index 152c92fe..55dee0e0 100644 --- a/src/sempy_labs/_list_functions.py +++ b/src/sempy_labs/_list_functions.py @@ -2019,7 +2019,9 @@ def assign_workspace_to_capacity(capacity_name: str, workspace: Optional[str] = dfC_filt = dfC[dfC["Display Name"] == capacity_name] if len(dfC_filt) == 0: - raise ValueError(f"{icons.red_dot} The '{capacity_name}' capacity does not exist.") + raise ValueError( + f"{icons.red_dot} The '{capacity_name}' capacity does not exist." + ) capacity_id = dfC_filt["Id"].iloc[0] @@ -2054,9 +2056,7 @@ def unassign_workspace_from_capacity(workspace: Optional[str] = None): (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) client = fabric.FabricRestClient() - response = client.post( - f"/v1/workspaces/{workspace_id}/unassignFromCapacity" - ) + response = client.post(f"/v1/workspaces/{workspace_id}/unassignFromCapacity") if response.status_code not in [200, 202]: raise FabricHTTPException(response) diff --git a/src/sempy_labs/_model_bpa.py b/src/sempy_labs/_model_bpa.py index 388fc2e3..8e4b9e32 100644 --- a/src/sempy_labs/_model_bpa.py +++ b/src/sempy_labs/_model_bpa.py @@ -63,7 +63,7 @@ def run_model_bpa( pandas.DataFrame A pandas dataframe in HTML format showing semantic model objects which violated the best practice analyzer rules. """ - + import polib if "extend" in kwargs: diff --git a/src/sempy_labs/tom/_model.py b/src/sempy_labs/tom/_model.py index 4531aacc..b1b330f1 100644 --- a/src/sempy_labs/tom/_model.py +++ b/src/sempy_labs/tom/_model.py @@ -871,7 +871,11 @@ def add_calculation_group( self.model.Tables.Add(tbl) def add_expression( - self, name: str, expression: str, description: Optional[str] = None, source_lineage_tag: Optional[str] = None, + self, + name: str, + expression: str, + description: Optional[str] = None, + source_lineage_tag: Optional[str] = None, ): """ Adds an `expression `_ to a semantic model. From 7e653b08159f52e40fc6c3f3ee1a53b21253e35a Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 12 Sep 2024 13:10:06 +0300 Subject: [PATCH 2/4] reorganized files --- src/sempy_labs/__init__.py | 37 +- src/sempy_labs/_environments.py | 16 +- src/sempy_labs/_list_functions.py | 886 +----------------------------- src/sempy_labs/_notebooks.py | 140 +++++ src/sempy_labs/_spark.py | 465 ++++++++++++++++ src/sempy_labs/_workspaces.py | 302 ++++++++++ 6 files changed, 937 insertions(+), 909 deletions(-) create mode 100644 src/sempy_labs/_notebooks.py create mode 100644 src/sempy_labs/_spark.py create mode 100644 src/sempy_labs/_workspaces.py diff --git a/src/sempy_labs/__init__.py b/src/sempy_labs/__init__.py index e4e3e96b..781be4c3 100644 --- a/src/sempy_labs/__init__.py +++ b/src/sempy_labs/__init__.py @@ -3,6 +3,28 @@ delete_environment, ) +from sempy_labs._spark import ( + get_spark_settings, + update_spark_settings, + list_custom_pools, + create_custom_pool, + delete_custom_pool, + update_custom_pool, +) + +from sempy_labs._workspaces import ( + list_workspace_users, + update_workspace_user, + add_user_to_workspace, + delete_user_from_workspace, + assign_workspace_to_capacity, + unassign_workspace_from_capacity, + list_workspace_role_assignments, +) +from sempy_labs._notebooks import ( + get_notebook_definition, + import_notebook_from_web, +) from sempy_labs._sql import ( ConnectWarehouse, ) @@ -55,7 +77,6 @@ ) from sempy_labs._list_functions import ( list_reports_using_semantic_model, - delete_custom_pool, list_semantic_model_objects, list_shortcuts, get_object_level_security, @@ -77,22 +98,8 @@ # list_sqlendpoints, # list_tables, list_warehouses, - list_workspace_role_assignments, create_warehouse, update_item, - list_custom_pools, - create_custom_pool, - update_custom_pool, - assign_workspace_to_capacity, - unassign_workspace_from_capacity, - get_spark_settings, - update_spark_settings, - add_user_to_workspace, - delete_user_from_workspace, - update_workspace_user, - list_workspace_users, - get_notebook_definition, - import_notebook_from_web, ) from sempy_labs._helper_functions import ( diff --git a/src/sempy_labs/_environments.py b/src/sempy_labs/_environments.py index 9190bdf6..0d0564b5 100644 --- a/src/sempy_labs/_environments.py +++ b/src/sempy_labs/_environments.py @@ -64,9 +64,7 @@ def list_environments(workspace: Optional[str] = None) -> pd.DataFrame: A pandas dataframe showing the environments within a workspace. """ - df = pd.DataFrame( - columns=["Environment Name", "Environment Id", "Description"] - ) + df = pd.DataFrame(columns=["Environment Name", "Environment Id", "Description"]) (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) @@ -89,9 +87,7 @@ def list_environments(workspace: Optional[str] = None) -> pd.DataFrame: return df -def delete_environment( - environment: str, workspace: Optional[str] = None -): +def delete_environment(environment: str, workspace: Optional[str] = None): """ Deletes a Fabric environment. @@ -108,10 +104,12 @@ def delete_environment( (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) dfE = list_environments(workspace=workspace) - dfE_filt = dfE[dfE['Environment Name'] == environment] + dfE_filt = dfE[dfE["Environment Name"] == environment] if len(dfE_filt) == 0: - raise ValueError(f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace.") - environment_id = dfE_filt['Environment Id'].iloc[0] + raise ValueError( + f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace." + ) + environment_id = dfE_filt["Environment Id"].iloc[0] client = fabric.FabricRestClient() response = client.delete( diff --git a/src/sempy_labs/_list_functions.py b/src/sempy_labs/_list_functions.py index 55dee0e0..664efe35 100644 --- a/src/sempy_labs/_list_functions.py +++ b/src/sempy_labs/_list_functions.py @@ -4,15 +4,11 @@ create_relationship_name, resolve_lakehouse_id, resolve_dataset_id, - _decode_b64, pagination, lro, resolve_item_type, ) import pandas as pd -import base64 -import requests -from pyspark.sql import SparkSession from typing import Optional import sempy_labs._icons as icons from sempy.fabric.exceptions import FabricHTTPException @@ -505,6 +501,7 @@ def list_columns( from sempy_labs.directlake._get_directlake_lakehouse import ( get_direct_lake_lakehouse, ) + from pyspark.sql import SparkSession workspace = fabric.resolve_workspace_name(workspace) @@ -1393,48 +1390,6 @@ def list_kpis(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame: return df -def list_workspace_role_assignments(workspace: Optional[str] = None) -> pd.DataFrame: - """ - Shows the members of a given workspace. - - Parameters - ---------- - workspace : str, default=None - The Fabric workspace name. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - Returns - ------- - pandas.DataFrame - A pandas dataframe showing the members of a given workspace and their roles. - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - df = pd.DataFrame(columns=["User Name", "User Email", "Role Name", "Type"]) - - client = fabric.FabricRestClient() - response = client.get(f"/v1/workspaces/{workspace_id}/roleAssignments") - if response.status_code != 200: - raise FabricHTTPException(response) - - responses = pagination(client, response) - - for r in responses: - for i in r.get("value", []): - principal = i.get("principal", {}) - new_data = { - "User Name": principal.get("displayName"), - "Role Name": i.get("role"), - "Type": principal.get("type"), - "User Email": principal.get("userDetails", {}).get("userPrincipalName"), - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - - return df - - def list_semantic_model_objects( dataset: str, workspace: Optional[str] = None ) -> pd.DataFrame: @@ -1709,717 +1664,6 @@ def list_shortcuts( return df -def list_custom_pools(workspace: Optional[str] = None) -> pd.DataFrame: - """ - Lists all `custom pools `_ within a workspace. - - Parameters - ---------- - workspace : str, default=None - The name of the Fabric workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - Returns - ------- - pandas.DataFrame - A pandas dataframe showing all the custom pools within the Fabric workspace. - """ - - # https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/list-workspace-custom-pools - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - df = pd.DataFrame( - columns=[ - "Custom Pool ID", - "Custom Pool Name", - "Type", - "Node Family", - "Node Size", - "Auto Scale Enabled", - "Auto Scale Min Node Count", - "Auto Scale Max Node Count", - "Dynamic Executor Allocation Enabled", - "Dynamic Executor Allocation Min Executors", - "Dynamic Executor Allocation Max Executors", - ] - ) - - client = fabric.FabricRestClient() - response = client.get(f"/v1/workspaces/{workspace_id}/spark/pools") - if response.status_code != 200: - raise FabricHTTPException(response) - - for i in response.json()["value"]: - - aScale = i.get("autoScale", {}) - d = i.get("dynamicExecutorAllocation", {}) - - new_data = { - "Custom Pool ID": i.get("id"), - "Custom Pool Name": i.get("name"), - "Type": i.get("type"), - "Node Family": i.get("nodeFamily"), - "Node Size": i.get("nodeSize"), - "Auto Scale Enabled": aScale.get("enabled"), - "Auto Scale Min Node Count": aScale.get("minNodeCount"), - "Auto Scale Max Node Count": aScale.get("maxNodeCount"), - "Dynamic Executor Allocation Enabled": d.get("enabled"), - "Dynamic Executor Allocation Min Executors": d.get("minExecutors"), - "Dynamic Executor Allocation Max Executors": d.get("maxExecutors"), - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - - bool_cols = ["Auto Scale Enabled", "Dynamic Executor Allocation Enabled"] - int_cols = [ - "Auto Scale Min Node Count", - "Auto Scale Max Node Count", - "Dynamic Executor Allocation Enabled", - "Dynamic Executor Allocation Min Executors", - "Dynamic Executor Allocation Max Executors", - ] - - df[bool_cols] = df[bool_cols].astype(bool) - df[int_cols] = df[int_cols].astype(int) - - return df - - -def create_custom_pool( - pool_name: str, - node_size: str, - min_node_count: int, - max_node_count: int, - min_executors: int, - max_executors: int, - node_family: Optional[str] = "MemoryOptimized", - auto_scale_enabled: Optional[bool] = True, - dynamic_executor_allocation_enabled: Optional[bool] = True, - workspace: Optional[str] = None, -): - """ - Creates a `custom pool `_ within a workspace. - - Parameters - ---------- - pool_name : str - The custom pool name. - node_size : str - The `node size `_. - min_node_count : int - The `minimum node count `_. - max_node_count : int - The `maximum node count `_. - min_executors : int - The `minimum executors `_. - max_executors : int - The `maximum executors `_. - node_family : str, default='MemoryOptimized' - The `node family `_. - auto_scale_enabled : bool, default=True - The status of `auto scale `_. - dynamic_executor_allocation_enabled : bool, default=True - The status of the `dynamic executor allocation `_. - workspace : str, default=None - The name of the Fabric workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - """ - - # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - request_body = { - "name": pool_name, - "nodeFamily": node_family, - "nodeSize": node_size, - "autoScale": { - "enabled": auto_scale_enabled, - "minNodeCount": min_node_count, - "maxNodeCount": max_node_count, - }, - "dynamicExecutorAllocation": { - "enabled": dynamic_executor_allocation_enabled, - "minExecutors": min_executors, - "maxExecutors": max_executors, - }, - } - - client = fabric.FabricRestClient() - response = client.post( - f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body - ) - - if response.status_code != 201: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The '{pool_name}' spark pool has been created within the '{workspace}' workspace." - ) - - -def update_custom_pool( - pool_name: str, - node_size: Optional[str] = None, - min_node_count: Optional[int] = None, - max_node_count: Optional[int] = None, - min_executors: Optional[int] = None, - max_executors: Optional[int] = None, - node_family: Optional[str] = None, - auto_scale_enabled: Optional[bool] = None, - dynamic_executor_allocation_enabled: Optional[bool] = None, - workspace: Optional[str] = None, -): - """ - Updates the properties of a `custom pool `_ within a workspace. - - Parameters - ---------- - pool_name : str - The custom pool name. - node_size : str, default=None - The `node size `_. - Defaults to None which keeps the existing property setting. - min_node_count : int, default=None - The `minimum node count `_. - Defaults to None which keeps the existing property setting. - max_node_count : int, default=None - The `maximum node count `_. - Defaults to None which keeps the existing property setting. - min_executors : int, default=None - The `minimum executors `_. - Defaults to None which keeps the existing property setting. - max_executors : int, default=None - The `maximum executors `_. - Defaults to None which keeps the existing property setting. - node_family : str, default=None - The `node family `_. - Defaults to None which keeps the existing property setting. - auto_scale_enabled : bool, default=None - The status of `auto scale `_. - Defaults to None which keeps the existing property setting. - dynamic_executor_allocation_enabled : bool, default=None - The status of the `dynamic executor allocation `_. - Defaults to None which keeps the existing property setting. - workspace : str, default=None - The name of the Fabric workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - """ - - # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/update-workspace-custom-pool?tabs=HTTP - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - df = list_custom_pools(workspace=workspace) - df_pool = df[df["Custom Pool Name"] == pool_name] - - if len(df_pool) == 0: - raise ValueError( - f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}'. Please choose a valid custom pool." - ) - - if node_family is None: - node_family = df_pool["Node Family"].iloc[0] - if node_size is None: - node_size = df_pool["Node Size"].iloc[0] - if auto_scale_enabled is None: - auto_scale_enabled = bool(df_pool["Auto Scale Enabled"].iloc[0]) - if min_node_count is None: - min_node_count = int(df_pool["Min Node Count"].iloc[0]) - if max_node_count is None: - max_node_count = int(df_pool["Max Node Count"].iloc[0]) - if dynamic_executor_allocation_enabled is None: - dynamic_executor_allocation_enabled = bool( - df_pool["Dynami Executor Allocation Enabled"].iloc[0] - ) - if min_executors is None: - min_executors = int(df_pool["Min Executors"].iloc[0]) - if max_executors is None: - max_executors = int(df_pool["Max Executors"].iloc[0]) - - request_body = { - "name": pool_name, - "nodeFamily": node_family, - "nodeSize": node_size, - "autoScale": { - "enabled": auto_scale_enabled, - "minNodeCount": min_node_count, - "maxNodeCount": max_node_count, - }, - "dynamicExecutorAllocation": { - "enabled": dynamic_executor_allocation_enabled, - "minExecutors": min_executors, - "maxExecutors": max_executors, - }, - } - - client = fabric.FabricRestClient() - response = client.post( - f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body - ) - - if response.status_code != 200: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The '{pool_name}' spark pool within the '{workspace}' workspace has been updated." - ) - - -def delete_custom_pool(pool_name: str, workspace: Optional[str] = None): - """ - Deletes a `custom pool `_ within a workspace. - - Parameters - ---------- - pool_name : str - The custom pool name. - workspace : str, default=None - The name of the Fabric workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - dfL = list_custom_pools(workspace=workspace) - dfL_filt = dfL[dfL["Custom Pool Name"] == pool_name] - - if len(dfL_filt) == 0: - raise ValueError( - f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}' workspace." - ) - poolId = dfL_filt["Custom Pool ID"].iloc[0] - - client = fabric.FabricRestClient() - response = client.delete(f"/v1/workspaces/{workspace_id}/spark/pools/{poolId}") - - if response.status_code != 200: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The '{pool_name}' spark pool has been deleted from the '{workspace}' workspace." - ) - - -def assign_workspace_to_capacity(capacity_name: str, workspace: Optional[str] = None): - """ - Assigns a workspace to a capacity. - - Parameters - ---------- - capacity_name : str - The name of the capacity. - workspace : str, default=None - The name of the Fabric workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - dfC = fabric.list_capacities() - dfC_filt = dfC[dfC["Display Name"] == capacity_name] - - if len(dfC_filt) == 0: - raise ValueError( - f"{icons.red_dot} The '{capacity_name}' capacity does not exist." - ) - - capacity_id = dfC_filt["Id"].iloc[0] - - request_body = {"capacityId": capacity_id} - - client = fabric.FabricRestClient() - response = client.post( - f"/v1/workspaces/{workspace_id}/assignToCapacity", - json=request_body, - ) - - if response.status_code not in [200, 202]: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The '{workspace}' workspace has been assigned to the '{capacity_name}' capacity." - ) - - -def unassign_workspace_from_capacity(workspace: Optional[str] = None): - """ - Unassigns a workspace from its assigned capacity. - - Parameters - ---------- - workspace : str, default=None - The name of the Fabric workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - """ - - # https://learn.microsoft.com/en-us/rest/api/fabric/core/workspaces/unassign-from-capacity?tabs=HTTP - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - client = fabric.FabricRestClient() - response = client.post(f"/v1/workspaces/{workspace_id}/unassignFromCapacity") - - if response.status_code not in [200, 202]: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The '{workspace}' workspace has been unassigned from its capacity." - ) - - -def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame: - """ - Shows the spark settings for a workspace. - - Parameters - ---------- - workspace : str, default=None - The name of the Fabric workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - Returns - ------- - pandas.DataFrame - A pandas dataframe showing the spark settings for a workspace. - """ - - # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/get-spark-settings?tabs=HTTP - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - df = pd.DataFrame( - columns=[ - "Automatic Log Enabled", - "High Concurrency Enabled", - "Customize Compute Enabled", - "Default Pool Name", - "Default Pool Type", - "Max Node Count", - "Max Executors", - "Environment Name", - "Runtime Version", - ] - ) - - client = fabric.FabricRestClient() - response = client.get(f"/v1/workspaces/{workspace_id}/spark/settings") - if response.status_code != 200: - raise FabricHTTPException(response) - - i = response.json() - p = i.get("pool") - dp = i.get("pool", {}).get("defaultPool", {}) - sp = i.get("pool", {}).get("starterPool", {}) - e = i.get("environment", {}) - - new_data = { - "Automatic Log Enabled": i.get("automaticLog").get("enabled"), - "High Concurrency Enabled": i.get("highConcurrency").get( - "notebookInteractiveRunEnabled" - ), - "Customize Compute Enabled": p.get("customizeComputeEnabled"), - "Default Pool Name": dp.get("name"), - "Default Pool Type": dp.get("type"), - "Max Node Count": sp.get("maxNodeCount"), - "Max Node Executors": sp.get("maxExecutors"), - "Environment Name": e.get("name"), - "Runtime Version": e.get("runtimeVersion"), - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - - bool_cols = [ - "Automatic Log Enabled", - "High Concurrency Enabled", - "Customize Compute Enabled", - ] - int_cols = ["Max Node Count", "Max Executors"] - - df[bool_cols] = df[bool_cols].astype(bool) - df[int_cols] = df[int_cols].astype(int) - - return df - - -def update_spark_settings( - automatic_log_enabled: Optional[bool] = None, - high_concurrency_enabled: Optional[bool] = None, - customize_compute_enabled: Optional[bool] = None, - default_pool_name: Optional[str] = None, - max_node_count: Optional[int] = None, - max_executors: Optional[int] = None, - environment_name: Optional[str] = None, - runtime_version: Optional[str] = None, - workspace: Optional[str] = None, -): - """ - Updates the spark settings for a workspace. - - Parameters - ---------- - automatic_log_enabled : bool, default=None - The status of the `automatic log `_. - Defaults to None which keeps the existing property setting. - high_concurrency_enabled : bool, default=None - The status of the `high concurrency `_ for notebook interactive run. - Defaults to None which keeps the existing property setting. - customize_compute_enabled : bool, default=None - `Customize compute `_ configurations for items. - Defaults to None which keeps the existing property setting. - default_pool_name : str, default=None - `Default pool `_ for workspace. - Defaults to None which keeps the existing property setting. - max_node_count : int, default=None - The `maximum node count `_. - Defaults to None which keeps the existing property setting. - max_executors : int, default=None - The `maximum executors `_. - Defaults to None which keeps the existing property setting. - environment_name : str, default=None - The name of the `default environment `_. Empty string indicated there is no workspace default environment - Defaults to None which keeps the existing property setting. - runtime_version : str, default=None - The `runtime version `_. - Defaults to None which keeps the existing property setting. - workspace : str, default=None - The name of the Fabric workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - Returns - ------- - """ - - # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - dfS = get_spark_settings(workspace=workspace) - - if automatic_log_enabled is None: - automatic_log_enabled = bool(dfS["Automatic Log Enabled"].iloc[0]) - if high_concurrency_enabled is None: - high_concurrency_enabled = bool(dfS["High Concurrency Enabled"].iloc[0]) - if customize_compute_enabled is None: - customize_compute_enabled = bool(dfS["Customize Compute Enabled"].iloc[0]) - if default_pool_name is None: - default_pool_name = dfS["Default Pool Name"].iloc[0] - if max_node_count is None: - max_node_count = int(dfS["Max Node Count"].iloc[0]) - if max_executors is None: - max_executors = int(dfS["Max Executors"].iloc[0]) - if environment_name is None: - environment_name = dfS["Environment Name"].iloc[0] - if runtime_version is None: - runtime_version = dfS["Runtime Version"].iloc[0] - - request_body = { - "automaticLog": {"enabled": automatic_log_enabled}, - "highConcurrency": {"notebookInteractiveRunEnabled": high_concurrency_enabled}, - "pool": { - "customizeComputeEnabled": customize_compute_enabled, - "defaultPool": {"name": default_pool_name, "type": "Workspace"}, - "starterPool": { - "maxNodeCount": max_node_count, - "maxExecutors": max_executors, - }, - }, - "environment": {"name": environment_name, "runtimeVersion": runtime_version}, - } - - client = fabric.FabricRestClient() - response = client.patch( - f"/v1/workspaces/{workspace_id}/spark/settings", json=request_body - ) - - if response.status_code != 200: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The spark settings within the '{workspace}' workspace have been updated accordingly." - ) - - -def add_user_to_workspace( - email_address: str, - role_name: str, - principal_type: Optional[str] = "User", - workspace: Optional[str] = None, -): - """ - Adds a user to a workspace. - - Parameters - ---------- - email_address : str - The email address of the user. - role_name : str - The `role `_ of the user within the workspace. - principal_type : str, default='User' - The `principal type `_. - workspace : str, default=None - The name of the workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - role_names = ["Admin", "Member", "Viewer", "Contributor"] - role_name = role_name.capitalize() - if role_name not in role_names: - raise ValueError( - f"{icons.red_dot} Invalid role. The 'role_name' parameter must be one of the following: {role_names}." - ) - plural = "n" if role_name == "Admin" else "" - principal_types = ["App", "Group", "None", "User"] - principal_type = principal_type.capitalize() - if principal_type not in principal_types: - raise ValueError( - f"{icons.red_dot} Invalid princpal type. Valid options: {principal_types}." - ) - - client = fabric.PowerBIRestClient() - - request_body = { - "emailAddress": email_address, - "groupUserAccessRight": role_name, - "principalType": principal_type, - "identifier": email_address, - } - - response = client.post( - f"/v1.0/myorg/groups/{workspace_id}/users", json=request_body - ) - - if response.status_code != 200: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The '{email_address}' user has been added as a{plural} '{role_name}' within the '{workspace}' workspace." - ) - - -def delete_user_from_workspace(email_address: str, workspace: Optional[str] = None): - """ - Removes a user from a workspace. - - Parameters - ---------- - email_address : str - The email address of the user. - workspace : str, default=None - The name of the workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - Returns - ------- - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - client = fabric.PowerBIRestClient() - response = client.delete(f"/v1.0/myorg/groups/{workspace_id}/users/{email_address}") - - if response.status_code != 200: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The '{email_address}' user has been removed from accessing the '{workspace}' workspace." - ) - - -def update_workspace_user( - email_address: str, - role_name: str, - principal_type: Optional[str] = "User", - workspace: Optional[str] = None, -): - """ - Updates a user's role within a workspace. - - Parameters - ---------- - email_address : str - The email address of the user. - role_name : str - The `role `_ of the user within the workspace. - principal_type : str, default='User' - The `principal type `_. - workspace : str, default=None - The name of the workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - role_names = ["Admin", "Member", "Viewer", "Contributor"] - role_name = role_name.capitalize() - if role_name not in role_names: - raise ValueError( - f"{icons.red_dot} Invalid role. The 'role_name' parameter must be one of the following: {role_names}." - ) - principal_types = ["App", "Group", "None", "User"] - principal_type = principal_type.capitalize() - if principal_type not in principal_types: - raise ValueError( - f"{icons.red_dot} Invalid princpal type. Valid options: {principal_types}." - ) - - request_body = { - "emailAddress": email_address, - "groupUserAccessRight": role_name, - "principalType": principal_type, - "identifier": email_address, - } - - client = fabric.PowerBIRestClient() - response = client.put(f"/v1.0/myorg/groups/{workspace_id}/users", json=request_body) - - if response.status_code != 200: - raise FabricHTTPException(response) - print( - f"{icons.green_dot} The '{email_address}' user has been updated to a '{role_name}' within the '{workspace}' workspace." - ) - - -def list_workspace_users(workspace: Optional[str] = None) -> pd.DataFrame: - """ - A list of all the users of a workspace and their roles. - - Parameters - ---------- - workspace : str, default=None - The name of the workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - Returns - ------- - pandas.DataFrame - A pandas dataframe the users of a workspace and their properties. - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - df = pd.DataFrame(columns=["User Name", "Email Address", "Role", "Type", "User ID"]) - client = fabric.FabricRestClient() - response = client.get(f"/v1/workspaces/{workspace_id}/roleAssignments") - if response.status_code != 200: - raise FabricHTTPException(response) - - responses = pagination(client, response) - - for r in responses: - for v in r.get("value", []): - p = v.get("principal", {}) - new_data = { - "User Name": p.get("displayName"), - "User ID": p.get("id"), - "Type": p.get("type"), - "Role": v.get("role"), - "Email Address": p.get("userDetails", {}).get("userPrincipalName"), - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - - return df - - def list_capacities() -> pd.DataFrame: """ Shows the capacities and their properties. @@ -2456,134 +1700,6 @@ def list_capacities() -> pd.DataFrame: return df -def get_notebook_definition( - notebook_name: str, workspace: Optional[str] = None, decode: Optional[bool] = True -): - """ - Obtains the notebook definition. - - Parameters - ---------- - notebook_name : str - The name of the notebook. - workspace : str, default=None - The name of the workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - decode : bool, default=True - If True, decodes the notebook definition file into .ipynb format. - If False, obtains the notebook definition file in base64 format. - - Returns - ------- - ipynb - The notebook definition. - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - dfI = fabric.list_items(workspace=workspace, type="Notebook") - dfI_filt = dfI[dfI["Display Name"] == notebook_name] - - if len(dfI_filt) == 0: - raise ValueError( - f"{icons.red_dot} The '{notebook_name}' notebook does not exist within the '{workspace}' workspace." - ) - - notebook_id = dfI_filt["Id"].iloc[0] - client = fabric.FabricRestClient() - response = client.post( - f"v1/workspaces/{workspace_id}/notebooks/{notebook_id}/getDefinition", - ) - - result = lro(client, response).json() - df_items = pd.json_normalize(result["definition"]["parts"]) - df_items_filt = df_items[df_items["path"] == "notebook-content.py"] - payload = df_items_filt["payload"].iloc[0] - - if decode: - result = _decode_b64(payload) - else: - result = payload - - return result - - -def import_notebook_from_web( - notebook_name: str, - url: str, - description: Optional[str] = None, - workspace: Optional[str] = None, -): - """ - Creates a new notebook within a workspace based on a Jupyter notebook hosted in the web. - - Parameters - ---------- - notebook_name : str - The name of the notebook to be created. - url : str - The url of the Jupyter Notebook (.ipynb) - description : str, default=None - The description of the notebook. - Defaults to None which does not place a description. - workspace : str, default=None - The name of the workspace. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - Returns - ------- - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - client = fabric.FabricRestClient() - dfI = fabric.list_items(workspace=workspace, type="Notebook") - dfI_filt = dfI[dfI["Display Name"] == notebook_name] - if len(dfI_filt) > 0: - raise ValueError( - f"{icons.red_dot} The '{notebook_name}' already exists within the '{workspace}' workspace." - ) - - # Fix links to go to the raw github file - starting_text = "https://github.com/" - starting_text_len = len(starting_text) - if url.startswith(starting_text): - url = f"https://raw.githubusercontent.com/{url[starting_text_len:]}".replace( - "/blob/", "/" - ) - - response = requests.get(url) - if response.status_code != 200: - raise FabricHTTPException(response) - file_content = response.content - notebook_payload = base64.b64encode(file_content) - - request_body = { - "displayName": notebook_name, - "definition": { - "format": "ipynb", - "parts": [ - { - "path": "notebook-content.py", - "payload": notebook_payload, - "payloadType": "InlineBase64", - } - ], - }, - } - if description is not None: - request_body["description"] = description - - response = client.post(f"v1/workspaces/{workspace_id}/notebooks", json=request_body) - - lro(client, response, status_codes=[201, 202]) - - print( - f"{icons.green_dot} The '{notebook_name}' notebook was created within the '{workspace}' workspace." - ) - - def list_reports_using_semantic_model( dataset: str, workspace: Optional[str] = None ) -> pd.DataFrame: diff --git a/src/sempy_labs/_notebooks.py b/src/sempy_labs/_notebooks.py new file mode 100644 index 00000000..8e804f2f --- /dev/null +++ b/src/sempy_labs/_notebooks.py @@ -0,0 +1,140 @@ +import sempy.fabric as fabric +import pandas as pd +import sempy_labs._icons as icons +from typing import Optional +import base64 +import requests +from sempy_labs._helper_functions import ( + resolve_workspace_name_and_id, + lro, + _decode_b64, +) +from sempy.fabric.exceptions import FabricHTTPException + + +def get_notebook_definition( + notebook_name: str, workspace: Optional[str] = None, decode: Optional[bool] = True +): + """ + Obtains the notebook definition. + + Parameters + ---------- + notebook_name : str + The name of the notebook. + workspace : str, default=None + The name of the workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + decode : bool, default=True + If True, decodes the notebook definition file into .ipynb format. + If False, obtains the notebook definition file in base64 format. + + Returns + ------- + ipynb + The notebook definition. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + dfI = fabric.list_items(workspace=workspace, type="Notebook") + dfI_filt = dfI[dfI["Display Name"] == notebook_name] + + if len(dfI_filt) == 0: + raise ValueError( + f"{icons.red_dot} The '{notebook_name}' notebook does not exist within the '{workspace}' workspace." + ) + + notebook_id = dfI_filt["Id"].iloc[0] + client = fabric.FabricRestClient() + response = client.post( + f"v1/workspaces/{workspace_id}/notebooks/{notebook_id}/getDefinition", + ) + + result = lro(client, response).json() + df_items = pd.json_normalize(result["definition"]["parts"]) + df_items_filt = df_items[df_items["path"] == "notebook-content.py"] + payload = df_items_filt["payload"].iloc[0] + + if decode: + result = _decode_b64(payload) + else: + result = payload + + return result + + +def import_notebook_from_web( + notebook_name: str, + url: str, + description: Optional[str] = None, + workspace: Optional[str] = None, +): + """ + Creates a new notebook within a workspace based on a Jupyter notebook hosted in the web. + + Parameters + ---------- + notebook_name : str + The name of the notebook to be created. + url : str + The url of the Jupyter Notebook (.ipynb) + description : str, default=None + The description of the notebook. + Defaults to None which does not place a description. + workspace : str, default=None + The name of the workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + client = fabric.FabricRestClient() + dfI = fabric.list_items(workspace=workspace, type="Notebook") + dfI_filt = dfI[dfI["Display Name"] == notebook_name] + if len(dfI_filt) > 0: + raise ValueError( + f"{icons.red_dot} The '{notebook_name}' already exists within the '{workspace}' workspace." + ) + + # Fix links to go to the raw github file + starting_text = "https://github.com/" + starting_text_len = len(starting_text) + if url.startswith(starting_text): + url = f"https://raw.githubusercontent.com/{url[starting_text_len:]}".replace( + "/blob/", "/" + ) + + response = requests.get(url) + if response.status_code != 200: + raise FabricHTTPException(response) + file_content = response.content + notebook_payload = base64.b64encode(file_content) + + request_body = { + "displayName": notebook_name, + "definition": { + "format": "ipynb", + "parts": [ + { + "path": "notebook-content.py", + "payload": notebook_payload, + "payloadType": "InlineBase64", + } + ], + }, + } + if description is not None: + request_body["description"] = description + + response = client.post(f"v1/workspaces/{workspace_id}/notebooks", json=request_body) + + lro(client, response, status_codes=[201, 202]) + + print( + f"{icons.green_dot} The '{notebook_name}' notebook was created within the '{workspace}' workspace." + ) diff --git a/src/sempy_labs/_spark.py b/src/sempy_labs/_spark.py new file mode 100644 index 00000000..39297ab4 --- /dev/null +++ b/src/sempy_labs/_spark.py @@ -0,0 +1,465 @@ +import sempy.fabric as fabric +import pandas as pd +import sempy_labs._icons as icons +from typing import Optional +from sempy_labs._helper_functions import ( + resolve_workspace_name_and_id, +) +from sempy.fabric.exceptions import FabricHTTPException + + +def list_custom_pools(workspace: Optional[str] = None) -> pd.DataFrame: + """ + Lists all `custom pools `_ within a workspace. + + Parameters + ---------- + workspace : str, default=None + The name of the Fabric workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + pandas.DataFrame + A pandas dataframe showing all the custom pools within the Fabric workspace. + """ + + # https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/list-workspace-custom-pools + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + df = pd.DataFrame( + columns=[ + "Custom Pool ID", + "Custom Pool Name", + "Type", + "Node Family", + "Node Size", + "Auto Scale Enabled", + "Auto Scale Min Node Count", + "Auto Scale Max Node Count", + "Dynamic Executor Allocation Enabled", + "Dynamic Executor Allocation Min Executors", + "Dynamic Executor Allocation Max Executors", + ] + ) + + client = fabric.FabricRestClient() + response = client.get(f"/v1/workspaces/{workspace_id}/spark/pools") + if response.status_code != 200: + raise FabricHTTPException(response) + + for i in response.json()["value"]: + + aScale = i.get("autoScale", {}) + d = i.get("dynamicExecutorAllocation", {}) + + new_data = { + "Custom Pool ID": i.get("id"), + "Custom Pool Name": i.get("name"), + "Type": i.get("type"), + "Node Family": i.get("nodeFamily"), + "Node Size": i.get("nodeSize"), + "Auto Scale Enabled": aScale.get("enabled"), + "Auto Scale Min Node Count": aScale.get("minNodeCount"), + "Auto Scale Max Node Count": aScale.get("maxNodeCount"), + "Dynamic Executor Allocation Enabled": d.get("enabled"), + "Dynamic Executor Allocation Min Executors": d.get("minExecutors"), + "Dynamic Executor Allocation Max Executors": d.get("maxExecutors"), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + + bool_cols = ["Auto Scale Enabled", "Dynamic Executor Allocation Enabled"] + int_cols = [ + "Auto Scale Min Node Count", + "Auto Scale Max Node Count", + "Dynamic Executor Allocation Enabled", + "Dynamic Executor Allocation Min Executors", + "Dynamic Executor Allocation Max Executors", + ] + + df[bool_cols] = df[bool_cols].astype(bool) + df[int_cols] = df[int_cols].astype(int) + + return df + + +def create_custom_pool( + pool_name: str, + node_size: str, + min_node_count: int, + max_node_count: int, + min_executors: int, + max_executors: int, + node_family: Optional[str] = "MemoryOptimized", + auto_scale_enabled: Optional[bool] = True, + dynamic_executor_allocation_enabled: Optional[bool] = True, + workspace: Optional[str] = None, +): + """ + Creates a `custom pool `_ within a workspace. + + Parameters + ---------- + pool_name : str + The custom pool name. + node_size : str + The `node size `_. + min_node_count : int + The `minimum node count `_. + max_node_count : int + The `maximum node count `_. + min_executors : int + The `minimum executors `_. + max_executors : int + The `maximum executors `_. + node_family : str, default='MemoryOptimized' + The `node family `_. + auto_scale_enabled : bool, default=True + The status of `auto scale `_. + dynamic_executor_allocation_enabled : bool, default=True + The status of the `dynamic executor allocation `_. + workspace : str, default=None + The name of the Fabric workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + request_body = { + "name": pool_name, + "nodeFamily": node_family, + "nodeSize": node_size, + "autoScale": { + "enabled": auto_scale_enabled, + "minNodeCount": min_node_count, + "maxNodeCount": max_node_count, + }, + "dynamicExecutorAllocation": { + "enabled": dynamic_executor_allocation_enabled, + "minExecutors": min_executors, + "maxExecutors": max_executors, + }, + } + + client = fabric.FabricRestClient() + response = client.post( + f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body + ) + + if response.status_code != 201: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The '{pool_name}' spark pool has been created within the '{workspace}' workspace." + ) + + +def update_custom_pool( + pool_name: str, + node_size: Optional[str] = None, + min_node_count: Optional[int] = None, + max_node_count: Optional[int] = None, + min_executors: Optional[int] = None, + max_executors: Optional[int] = None, + node_family: Optional[str] = None, + auto_scale_enabled: Optional[bool] = None, + dynamic_executor_allocation_enabled: Optional[bool] = None, + workspace: Optional[str] = None, +): + """ + Updates the properties of a `custom pool `_ within a workspace. + + Parameters + ---------- + pool_name : str + The custom pool name. + node_size : str, default=None + The `node size `_. + Defaults to None which keeps the existing property setting. + min_node_count : int, default=None + The `minimum node count `_. + Defaults to None which keeps the existing property setting. + max_node_count : int, default=None + The `maximum node count `_. + Defaults to None which keeps the existing property setting. + min_executors : int, default=None + The `minimum executors `_. + Defaults to None which keeps the existing property setting. + max_executors : int, default=None + The `maximum executors `_. + Defaults to None which keeps the existing property setting. + node_family : str, default=None + The `node family `_. + Defaults to None which keeps the existing property setting. + auto_scale_enabled : bool, default=None + The status of `auto scale `_. + Defaults to None which keeps the existing property setting. + dynamic_executor_allocation_enabled : bool, default=None + The status of the `dynamic executor allocation `_. + Defaults to None which keeps the existing property setting. + workspace : str, default=None + The name of the Fabric workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/update-workspace-custom-pool?tabs=HTTP + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + df = list_custom_pools(workspace=workspace) + df_pool = df[df["Custom Pool Name"] == pool_name] + + if len(df_pool) == 0: + raise ValueError( + f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}'. Please choose a valid custom pool." + ) + + if node_family is None: + node_family = df_pool["Node Family"].iloc[0] + if node_size is None: + node_size = df_pool["Node Size"].iloc[0] + if auto_scale_enabled is None: + auto_scale_enabled = bool(df_pool["Auto Scale Enabled"].iloc[0]) + if min_node_count is None: + min_node_count = int(df_pool["Min Node Count"].iloc[0]) + if max_node_count is None: + max_node_count = int(df_pool["Max Node Count"].iloc[0]) + if dynamic_executor_allocation_enabled is None: + dynamic_executor_allocation_enabled = bool( + df_pool["Dynami Executor Allocation Enabled"].iloc[0] + ) + if min_executors is None: + min_executors = int(df_pool["Min Executors"].iloc[0]) + if max_executors is None: + max_executors = int(df_pool["Max Executors"].iloc[0]) + + request_body = { + "name": pool_name, + "nodeFamily": node_family, + "nodeSize": node_size, + "autoScale": { + "enabled": auto_scale_enabled, + "minNodeCount": min_node_count, + "maxNodeCount": max_node_count, + }, + "dynamicExecutorAllocation": { + "enabled": dynamic_executor_allocation_enabled, + "minExecutors": min_executors, + "maxExecutors": max_executors, + }, + } + + client = fabric.FabricRestClient() + response = client.post( + f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body + ) + + if response.status_code != 200: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The '{pool_name}' spark pool within the '{workspace}' workspace has been updated." + ) + + +def delete_custom_pool(pool_name: str, workspace: Optional[str] = None): + """ + Deletes a `custom pool `_ within a workspace. + + Parameters + ---------- + pool_name : str + The custom pool name. + workspace : str, default=None + The name of the Fabric workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + dfL = list_custom_pools(workspace=workspace) + dfL_filt = dfL[dfL["Custom Pool Name"] == pool_name] + + if len(dfL_filt) == 0: + raise ValueError( + f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}' workspace." + ) + poolId = dfL_filt["Custom Pool ID"].iloc[0] + + client = fabric.FabricRestClient() + response = client.delete(f"/v1/workspaces/{workspace_id}/spark/pools/{poolId}") + + if response.status_code != 200: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The '{pool_name}' spark pool has been deleted from the '{workspace}' workspace." + ) + + +def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame: + """ + Shows the spark settings for a workspace. + + Parameters + ---------- + workspace : str, default=None + The name of the Fabric workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + pandas.DataFrame + A pandas dataframe showing the spark settings for a workspace. + """ + + # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/get-spark-settings?tabs=HTTP + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + df = pd.DataFrame( + columns=[ + "Automatic Log Enabled", + "High Concurrency Enabled", + "Customize Compute Enabled", + "Default Pool Name", + "Default Pool Type", + "Max Node Count", + "Max Executors", + "Environment Name", + "Runtime Version", + ] + ) + + client = fabric.FabricRestClient() + response = client.get(f"/v1/workspaces/{workspace_id}/spark/settings") + if response.status_code != 200: + raise FabricHTTPException(response) + + i = response.json() + p = i.get("pool") + dp = i.get("pool", {}).get("defaultPool", {}) + sp = i.get("pool", {}).get("starterPool", {}) + e = i.get("environment", {}) + + new_data = { + "Automatic Log Enabled": i.get("automaticLog").get("enabled"), + "High Concurrency Enabled": i.get("highConcurrency").get( + "notebookInteractiveRunEnabled" + ), + "Customize Compute Enabled": p.get("customizeComputeEnabled"), + "Default Pool Name": dp.get("name"), + "Default Pool Type": dp.get("type"), + "Max Node Count": sp.get("maxNodeCount"), + "Max Node Executors": sp.get("maxExecutors"), + "Environment Name": e.get("name"), + "Runtime Version": e.get("runtimeVersion"), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + + bool_cols = [ + "Automatic Log Enabled", + "High Concurrency Enabled", + "Customize Compute Enabled", + ] + int_cols = ["Max Node Count", "Max Executors"] + + df[bool_cols] = df[bool_cols].astype(bool) + df[int_cols] = df[int_cols].astype(int) + + return df + + +def update_spark_settings( + automatic_log_enabled: Optional[bool] = None, + high_concurrency_enabled: Optional[bool] = None, + customize_compute_enabled: Optional[bool] = None, + default_pool_name: Optional[str] = None, + max_node_count: Optional[int] = None, + max_executors: Optional[int] = None, + environment_name: Optional[str] = None, + runtime_version: Optional[str] = None, + workspace: Optional[str] = None, +): + """ + Updates the spark settings for a workspace. + + Parameters + ---------- + automatic_log_enabled : bool, default=None + The status of the `automatic log `_. + Defaults to None which keeps the existing property setting. + high_concurrency_enabled : bool, default=None + The status of the `high concurrency `_ for notebook interactive run. + Defaults to None which keeps the existing property setting. + customize_compute_enabled : bool, default=None + `Customize compute `_ configurations for items. + Defaults to None which keeps the existing property setting. + default_pool_name : str, default=None + `Default pool `_ for workspace. + Defaults to None which keeps the existing property setting. + max_node_count : int, default=None + The `maximum node count `_. + Defaults to None which keeps the existing property setting. + max_executors : int, default=None + The `maximum executors `_. + Defaults to None which keeps the existing property setting. + environment_name : str, default=None + The name of the `default environment `_. Empty string indicated there is no workspace default environment + Defaults to None which keeps the existing property setting. + runtime_version : str, default=None + The `runtime version `_. + Defaults to None which keeps the existing property setting. + workspace : str, default=None + The name of the Fabric workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + dfS = get_spark_settings(workspace=workspace) + + if automatic_log_enabled is None: + automatic_log_enabled = bool(dfS["Automatic Log Enabled"].iloc[0]) + if high_concurrency_enabled is None: + high_concurrency_enabled = bool(dfS["High Concurrency Enabled"].iloc[0]) + if customize_compute_enabled is None: + customize_compute_enabled = bool(dfS["Customize Compute Enabled"].iloc[0]) + if default_pool_name is None: + default_pool_name = dfS["Default Pool Name"].iloc[0] + if max_node_count is None: + max_node_count = int(dfS["Max Node Count"].iloc[0]) + if max_executors is None: + max_executors = int(dfS["Max Executors"].iloc[0]) + if environment_name is None: + environment_name = dfS["Environment Name"].iloc[0] + if runtime_version is None: + runtime_version = dfS["Runtime Version"].iloc[0] + + request_body = { + "automaticLog": {"enabled": automatic_log_enabled}, + "highConcurrency": {"notebookInteractiveRunEnabled": high_concurrency_enabled}, + "pool": { + "customizeComputeEnabled": customize_compute_enabled, + "defaultPool": {"name": default_pool_name, "type": "Workspace"}, + "starterPool": { + "maxNodeCount": max_node_count, + "maxExecutors": max_executors, + }, + }, + "environment": {"name": environment_name, "runtimeVersion": runtime_version}, + } + + client = fabric.FabricRestClient() + response = client.patch( + f"/v1/workspaces/{workspace_id}/spark/settings", json=request_body + ) + + if response.status_code != 200: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The spark settings within the '{workspace}' workspace have been updated accordingly." + ) diff --git a/src/sempy_labs/_workspaces.py b/src/sempy_labs/_workspaces.py new file mode 100644 index 00000000..c4347d63 --- /dev/null +++ b/src/sempy_labs/_workspaces.py @@ -0,0 +1,302 @@ +import sempy.fabric as fabric +import pandas as pd +import sempy_labs._icons as icons +from typing import Optional +from sempy_labs._helper_functions import ( + resolve_workspace_name_and_id, + pagination, +) +from sempy.fabric.exceptions import FabricHTTPException + + +def delete_user_from_workspace(email_address: str, workspace: Optional[str] = None): + """ + Removes a user from a workspace. + + Parameters + ---------- + email_address : str + The email address of the user. + workspace : str, default=None + The name of the workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + client = fabric.PowerBIRestClient() + response = client.delete(f"/v1.0/myorg/groups/{workspace_id}/users/{email_address}") + + if response.status_code != 200: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The '{email_address}' user has been removed from accessing the '{workspace}' workspace." + ) + + +def update_workspace_user( + email_address: str, + role_name: str, + principal_type: Optional[str] = "User", + workspace: Optional[str] = None, +): + """ + Updates a user's role within a workspace. + + Parameters + ---------- + email_address : str + The email address of the user. + role_name : str + The `role `_ of the user within the workspace. + principal_type : str, default='User' + The `principal type `_. + workspace : str, default=None + The name of the workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + role_names = ["Admin", "Member", "Viewer", "Contributor"] + role_name = role_name.capitalize() + if role_name not in role_names: + raise ValueError( + f"{icons.red_dot} Invalid role. The 'role_name' parameter must be one of the following: {role_names}." + ) + principal_types = ["App", "Group", "None", "User"] + principal_type = principal_type.capitalize() + if principal_type not in principal_types: + raise ValueError( + f"{icons.red_dot} Invalid princpal type. Valid options: {principal_types}." + ) + + request_body = { + "emailAddress": email_address, + "groupUserAccessRight": role_name, + "principalType": principal_type, + "identifier": email_address, + } + + client = fabric.PowerBIRestClient() + response = client.put(f"/v1.0/myorg/groups/{workspace_id}/users", json=request_body) + + if response.status_code != 200: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The '{email_address}' user has been updated to a '{role_name}' within the '{workspace}' workspace." + ) + + +def list_workspace_users(workspace: Optional[str] = None) -> pd.DataFrame: + """ + A list of all the users of a workspace and their roles. + + Parameters + ---------- + workspace : str, default=None + The name of the workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + pandas.DataFrame + A pandas dataframe the users of a workspace and their properties. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + df = pd.DataFrame(columns=["User Name", "Email Address", "Role", "Type", "User ID"]) + client = fabric.FabricRestClient() + response = client.get(f"/v1/workspaces/{workspace_id}/roleAssignments") + if response.status_code != 200: + raise FabricHTTPException(response) + + responses = pagination(client, response) + + for r in responses: + for v in r.get("value", []): + p = v.get("principal", {}) + new_data = { + "User Name": p.get("displayName"), + "User ID": p.get("id"), + "Type": p.get("type"), + "Role": v.get("role"), + "Email Address": p.get("userDetails", {}).get("userPrincipalName"), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + + return df + + +def add_user_to_workspace( + email_address: str, + role_name: str, + principal_type: Optional[str] = "User", + workspace: Optional[str] = None, +): + """ + Adds a user to a workspace. + + Parameters + ---------- + email_address : str + The email address of the user. + role_name : str + The `role `_ of the user within the workspace. + principal_type : str, default='User' + The `principal type `_. + workspace : str, default=None + The name of the workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + role_names = ["Admin", "Member", "Viewer", "Contributor"] + role_name = role_name.capitalize() + if role_name not in role_names: + raise ValueError( + f"{icons.red_dot} Invalid role. The 'role_name' parameter must be one of the following: {role_names}." + ) + plural = "n" if role_name == "Admin" else "" + principal_types = ["App", "Group", "None", "User"] + principal_type = principal_type.capitalize() + if principal_type not in principal_types: + raise ValueError( + f"{icons.red_dot} Invalid princpal type. Valid options: {principal_types}." + ) + + client = fabric.PowerBIRestClient() + + request_body = { + "emailAddress": email_address, + "groupUserAccessRight": role_name, + "principalType": principal_type, + "identifier": email_address, + } + + response = client.post( + f"/v1.0/myorg/groups/{workspace_id}/users", json=request_body + ) + + if response.status_code != 200: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The '{email_address}' user has been added as a{plural} '{role_name}' within the '{workspace}' workspace." + ) + + +def assign_workspace_to_capacity(capacity_name: str, workspace: Optional[str] = None): + """ + Assigns a workspace to a capacity. + + Parameters + ---------- + capacity_name : str + The name of the capacity. + workspace : str, default=None + The name of the Fabric workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + dfC = fabric.list_capacities() + dfC_filt = dfC[dfC["Display Name"] == capacity_name] + + if len(dfC_filt) == 0: + raise ValueError( + f"{icons.red_dot} The '{capacity_name}' capacity does not exist." + ) + + capacity_id = dfC_filt["Id"].iloc[0] + + request_body = {"capacityId": capacity_id} + + client = fabric.FabricRestClient() + response = client.post( + f"/v1/workspaces/{workspace_id}/assignToCapacity", + json=request_body, + ) + + if response.status_code not in [200, 202]: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The '{workspace}' workspace has been assigned to the '{capacity_name}' capacity." + ) + + +def unassign_workspace_from_capacity(workspace: Optional[str] = None): + """ + Unassigns a workspace from its assigned capacity. + + Parameters + ---------- + workspace : str, default=None + The name of the Fabric workspace. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + # https://learn.microsoft.com/en-us/rest/api/fabric/core/workspaces/unassign-from-capacity?tabs=HTTP + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + client = fabric.FabricRestClient() + response = client.post(f"/v1/workspaces/{workspace_id}/unassignFromCapacity") + + if response.status_code not in [200, 202]: + raise FabricHTTPException(response) + print( + f"{icons.green_dot} The '{workspace}' workspace has been unassigned from its capacity." + ) + + +def list_workspace_role_assignments(workspace: Optional[str] = None) -> pd.DataFrame: + """ + Shows the members of a given workspace. + + Parameters + ---------- + workspace : str, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + pandas.DataFrame + A pandas dataframe showing the members of a given workspace and their roles. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + df = pd.DataFrame(columns=["User Name", "User Email", "Role Name", "Type"]) + + client = fabric.FabricRestClient() + response = client.get(f"/v1/workspaces/{workspace_id}/roleAssignments") + if response.status_code != 200: + raise FabricHTTPException(response) + + responses = pagination(client, response) + + for r in responses: + for i in r.get("value", []): + principal = i.get("principal", {}) + new_data = { + "User Name": principal.get("displayName"), + "Role Name": i.get("role"), + "Type": principal.get("type"), + "User Email": principal.get("userDetails", {}).get("userPrincipalName"), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + + return df From f6b3f18f10f26ce526fc2fc4a7daa75099bbb75e Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 14 Sep 2024 21:54:52 +0300 Subject: [PATCH 3/4] updates per comments --- src/sempy_labs/__init__.py | 2 ++ src/sempy_labs/_helper_functions.py | 31 +++++++++++++++++++++++++++++ src/sempy_labs/_icons.py | 2 ++ src/sempy_labs/_notebooks.py | 3 +++ src/sempy_labs/_workspaces.py | 20 ++++++------------- 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/sempy_labs/__init__.py b/src/sempy_labs/__init__.py index 781be4c3..91c8131e 100644 --- a/src/sempy_labs/__init__.py +++ b/src/sempy_labs/__init__.py @@ -103,6 +103,7 @@ ) from sempy_labs._helper_functions import ( + resolve_capacity_id, resolve_warehouse_id, resolve_workspace_capacity, create_abfss_path, @@ -276,4 +277,5 @@ "disconnect_workspace_from_git", "create_environment", "delete_environment", + "resolve_capacity_id", ] diff --git a/src/sempy_labs/_helper_functions.py b/src/sempy_labs/_helper_functions.py index 8864e4e7..738f8d8b 100644 --- a/src/sempy_labs/_helper_functions.py +++ b/src/sempy_labs/_helper_functions.py @@ -781,6 +781,37 @@ def resolve_capacity_name(capacity_id: Optional[UUID] = None) -> str: return dfC_filt["Display Name"].iloc[0] +def resolve_capacity_id(capacity_name: Optional[str] = None) -> UUID: + """ + Obtains the capacity Id for a given capacity name. + + Parameters + ---------- + capacity_name : str, default=None + The capacity name. + Defaults to None which resolves to the capacity id of the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the capacity name of the workspace of the notebook. + + Returns + ------- + UUID + The capacity Id. + """ + + if capacity_name is None: + return get_capacity_id() + + dfC = fabric.list_capacities() + dfC_filt = dfC[dfC["Display Name"] == capacity_name] + + if len(dfC_filt) == 0: + raise ValueError( + f"{icons.red_dot} The '{capacity_name}' capacity does not exist." + ) + + return dfC_filt["Id"].iloc[0] + + def retry(sleep_time: int, timeout_error_message: str): def decorator(func): @wraps(func) diff --git a/src/sempy_labs/_icons.py b/src/sempy_labs/_icons.py index 29463168..59bff22b 100644 --- a/src/sempy_labs/_icons.py +++ b/src/sempy_labs/_icons.py @@ -31,3 +31,5 @@ report_bpa_name = "ReportBPA" severity_mapping = {warning: "Warning", error: "Error", info: "Info"} special_characters = ['"', "/", '"', ":", "|", "<", ">", "*", "?", "'", "!"] +workspace_roles = ["Admin", "Member", "Viewer", "Contributor"] +principal_types = ["App", "Group", "None", "User"] \ No newline at end of file diff --git a/src/sempy_labs/_notebooks.py b/src/sempy_labs/_notebooks.py index 8e804f2f..4ca15e83 100644 --- a/src/sempy_labs/_notebooks.py +++ b/src/sempy_labs/_notebooks.py @@ -74,6 +74,9 @@ def import_notebook_from_web( """ Creates a new notebook within a workspace based on a Jupyter notebook hosted in the web. + Note: When specifying a notebook from GitHub, please use the raw file path. Note that if the non-raw file path is specified, the url will be + converted to the raw URL as the raw URL is needed to obtain the notebook content. + Parameters ---------- notebook_name : str diff --git a/src/sempy_labs/_workspaces.py b/src/sempy_labs/_workspaces.py index c4347d63..1bb54872 100644 --- a/src/sempy_labs/_workspaces.py +++ b/src/sempy_labs/_workspaces.py @@ -5,6 +5,7 @@ from sempy_labs._helper_functions import ( resolve_workspace_name_and_id, pagination, + resolve_capacity_id ) from sempy.fabric.exceptions import FabricHTTPException @@ -63,13 +64,13 @@ def update_workspace_user( (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - role_names = ["Admin", "Member", "Viewer", "Contributor"] + role_names = icons.workspace_roles role_name = role_name.capitalize() if role_name not in role_names: raise ValueError( f"{icons.red_dot} Invalid role. The 'role_name' parameter must be one of the following: {role_names}." ) - principal_types = ["App", "Group", "None", "User"] + principal_types = icons.principal_types principal_type = principal_type.capitalize() if principal_type not in principal_types: raise ValueError( @@ -160,14 +161,14 @@ def add_user_to_workspace( (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - role_names = ["Admin", "Member", "Viewer", "Contributor"] + role_names = icons.workspace_roles role_name = role_name.capitalize() if role_name not in role_names: raise ValueError( f"{icons.red_dot} Invalid role. The 'role_name' parameter must be one of the following: {role_names}." ) plural = "n" if role_name == "Admin" else "" - principal_types = ["App", "Group", "None", "User"] + principal_types = icons.principal_types principal_type = principal_type.capitalize() if principal_type not in principal_types: raise ValueError( @@ -209,16 +210,7 @@ def assign_workspace_to_capacity(capacity_name: str, workspace: Optional[str] = """ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - dfC = fabric.list_capacities() - dfC_filt = dfC[dfC["Display Name"] == capacity_name] - - if len(dfC_filt) == 0: - raise ValueError( - f"{icons.red_dot} The '{capacity_name}' capacity does not exist." - ) - - capacity_id = dfC_filt["Id"].iloc[0] + capacity_id = resolve_capacity_id(capacity_name=capacity_name) request_body = {"capacityId": capacity_id} From 9082b48e84866178382187f17ef47ecb20931bee Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 15 Sep 2024 13:51:20 +0300 Subject: [PATCH 4/4] added publish_environment, resolve_environment_id --- src/sempy_labs/__init__.py | 4 +++ src/sempy_labs/_environments.py | 48 ++++++++++++++++++++++++----- src/sempy_labs/_helper_functions.py | 28 +++++++++++++++++ src/sempy_labs/_icons.py | 2 +- src/sempy_labs/_workspaces.py | 2 +- 5 files changed, 74 insertions(+), 10 deletions(-) diff --git a/src/sempy_labs/__init__.py b/src/sempy_labs/__init__.py index 91c8131e..0d4ec5ee 100644 --- a/src/sempy_labs/__init__.py +++ b/src/sempy_labs/__init__.py @@ -1,6 +1,7 @@ from sempy_labs._environments import ( create_environment, delete_environment, + publish_environment, ) from sempy_labs._spark import ( @@ -103,6 +104,7 @@ ) from sempy_labs._helper_functions import ( + resolve_environment_id, resolve_capacity_id, resolve_warehouse_id, resolve_workspace_capacity, @@ -277,5 +279,7 @@ "disconnect_workspace_from_git", "create_environment", "delete_environment", + "publish_environment", "resolve_capacity_id", + "resolve_environment_id", ] diff --git a/src/sempy_labs/_environments.py b/src/sempy_labs/_environments.py index 0d0564b5..b7d2faf5 100644 --- a/src/sempy_labs/_environments.py +++ b/src/sempy_labs/_environments.py @@ -101,15 +101,12 @@ def delete_environment(environment: str, workspace: Optional[str] = None): or if no lakehouse attached, resolves to the workspace of the notebook. """ - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + from sempy_labs._helper_functions import resolve_environment_id - dfE = list_environments(workspace=workspace) - dfE_filt = dfE[dfE["Environment Name"] == environment] - if len(dfE_filt) == 0: - raise ValueError( - f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace." - ) - environment_id = dfE_filt["Environment Id"].iloc[0] + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + environment_id = resolve_environment_id( + environment=environment, workspace=workspace + ) client = fabric.FabricRestClient() response = client.delete( @@ -122,3 +119,38 @@ def delete_environment(environment: str, workspace: Optional[str] = None): print( f"{icons.green_dot} The '{environment}' environment within the '{workspace}' workspace has been deleted." ) + + +def publish_environment(environment: str, workspace: Optional[str] = None): + """ + Publishes a Fabric environment. + + Parameters + ---------- + environment: str + Name of the environment. + workspace : str, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + # https://learn.microsoft.com/en-us/rest/api/fabric/environment/spark-libraries/publish-environment?tabs=HTTP + + from sempy_labs._helper_functions import resolve_environment_id + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + environment_id = resolve_environment_id( + environment=environment, workspace=workspace + ) + + client = fabric.FabricRestClient() + response = client.post( + f"/v1/workspaces/{workspace_id}/environments/{environment_id}/staging/publish" + ) + + lro(client, response) + + print( + f"{icons.green_dot} The '{environment}' environment within the '{workspace}' workspace has been published." + ) diff --git a/src/sempy_labs/_helper_functions.py b/src/sempy_labs/_helper_functions.py index 738f8d8b..550ceb1f 100644 --- a/src/sempy_labs/_helper_functions.py +++ b/src/sempy_labs/_helper_functions.py @@ -942,3 +942,31 @@ def resolve_warehouse_id(warehouse: str, workspace: Optional[str]): ) return warehouse_id + + +def resolve_environment_id(environment: str, workspace: Optional[str] = None) -> UUID: + """ + Obtains the environment Id for a given environment. + + Parameters + ---------- + environment: str + Name of the environment. + workspace : str, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + """ + + from sempy_labs._environments import list_environments + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + dfE = list_environments(workspace=workspace) + dfE_filt = dfE[dfE["Environment Name"] == environment] + if len(dfE_filt) == 0: + raise ValueError( + f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace." + ) + + return dfE_filt["Environment Id"].iloc[0] diff --git a/src/sempy_labs/_icons.py b/src/sempy_labs/_icons.py index 59bff22b..6e374f51 100644 --- a/src/sempy_labs/_icons.py +++ b/src/sempy_labs/_icons.py @@ -32,4 +32,4 @@ severity_mapping = {warning: "Warning", error: "Error", info: "Info"} special_characters = ['"', "/", '"', ":", "|", "<", ">", "*", "?", "'", "!"] workspace_roles = ["Admin", "Member", "Viewer", "Contributor"] -principal_types = ["App", "Group", "None", "User"] \ No newline at end of file +principal_types = ["App", "Group", "None", "User"] diff --git a/src/sempy_labs/_workspaces.py b/src/sempy_labs/_workspaces.py index 1bb54872..aa62ecf7 100644 --- a/src/sempy_labs/_workspaces.py +++ b/src/sempy_labs/_workspaces.py @@ -5,7 +5,7 @@ from sempy_labs._helper_functions import ( resolve_workspace_name_and_id, pagination, - resolve_capacity_id + resolve_capacity_id, ) from sempy.fabric.exceptions import FabricHTTPException