Skip to content

Commit

Permalink
added icons to more functions, fixed regex issues, renamed run_dax fu…
Browse files Browse the repository at this point in the history
…nction name, fixed update_directlake_partitionentity parameters.
  • Loading branch information
m-kovalsky committed Jun 13, 2024
1 parent 71c61df commit 94ce4f0
Show file tree
Hide file tree
Showing 30 changed files with 412 additions and 395 deletions.
4 changes: 2 additions & 2 deletions src/sempy_labs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# create_connection_vnet,
# create_connection_on_prem
# )
from sempy_labs._dax import run_dax
from sempy_labs._dax import evaluate_dax_impersonation
from sempy_labs._generate_semantic_model import (
create_blank_semantic_model,
create_semantic_model_from_bim,
Expand Down Expand Up @@ -89,7 +89,7 @@
# create_connection_cloud,
# create_connection_vnet,
# create_connection_on_prem,
"run_dax",
"evaluate_dax_impersonation",
"create_blank_semantic_model",
"create_semantic_model_from_bim",
#'deploy_semantic_model',
Expand Down
50 changes: 25 additions & 25 deletions src/sempy_labs/_ai.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from pyspark.sql import SparkSession
from typing import List, Optional, Union
from IPython.display import display

import sempy_labs._icons as icons

def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):

Expand Down Expand Up @@ -39,7 +39,7 @@ def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):

if len(fallback_filt) > 0:
print(
f"The '{dataset}' semantic model is a Direct Lake semantic model which contains views. Since views always fall back to DirectQuery, it is recommended to only use lakehouse tables and not views."
f"{icons.yellow_dot} The '{dataset}' semantic model is a Direct Lake semantic model which contains views. Since views always fall back to DirectQuery, it is recommended to only use lakehouse tables and not views."
)

# Potential model reduction estimate
Expand All @@ -56,11 +56,11 @@ def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
totSize = df["Total Size"].sum()
if len(df_filt) > 0:
print(
f"Potential savings of {totSize} bytes from following the '{rule}' rule."
f"{icons.yellow_dot} Potential savings of {totSize} bytes from following the '{rule}' rule."
)
display(df_filt)
else:
print(f"The '{rule}' rule has been followed.")
print(f"{icons.green_dot} The '{rule}' rule has been followed.")


def generate_measure_descriptions(
Expand All @@ -78,7 +78,7 @@ def generate_measure_descriptions(
validModels = ["gpt-35-turbo", "gpt-35-turbo-16k", "gpt-4"]
if gpt_model not in validModels:
print(
f"The '{gpt_model}' model is not a valid model. Enter a gpt_model from this list: {validModels}."
f"{icons.red_dot} The '{gpt_model}' model is not a valid model. Enter a gpt_model from this list: {validModels}."
)
return

Expand Down Expand Up @@ -173,7 +173,7 @@ def generate_aggs(

if any(value not in aggTypes for value in columns.values()):
print(
f"Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}."
f"{icons.red_dot} Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}."
)
return

Expand All @@ -183,15 +183,15 @@ def generate_aggs(
dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
print(
f"The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models."
f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models."
)
return

dfC_filtT = dfC[dfC["Table Name"] == table_name]

if len(dfC_filtT) == 0:
print(
f"The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace."
f"{icons.red_dot} The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace."
)
return

Expand All @@ -201,7 +201,7 @@ def generate_aggs(

if len(columns) != len(dfC_filt):
print(
f"Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
f"{icons.red_dot} Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
)
return

Expand All @@ -211,7 +211,7 @@ def generate_aggs(
dataType = dfC_col["Data Type"].iloc[0]
if agg in aggTypesAggregate and dataType not in numericTypes:
print(
f"The '{col}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types can be aggregated as '{aggTypesAggregate}' aggregation types."
f"{icons.red_dot} The '{col}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types can be aggregated as '{aggTypesAggregate}' aggregation types."
)
return

Expand All @@ -230,7 +230,7 @@ def generate_aggs(

if len(dfI_filt) == 0:
print(
f"The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter."
f"{icons.red_dot} The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter."
)
return

Expand Down Expand Up @@ -278,16 +278,16 @@ def generate_aggs(
delta_table_name=aggLakeTName,
)
spark_df.write.mode("overwrite").format("delta").save(aggFilePath)
f"The '{aggLakeTName}' table has been created/updated in the lakehouse."
f"{icons.green_dot} The '{aggLakeTName}' table has been created/updated in the lakehouse."

# Create/update semantic model agg table
tom_server = fabric.create_tom_server(readonly=False, workspace=workspace)
m = tom_server.Databases.GetByName(dataset).Model
f"\nUpdating the '{dataset}' semantic model..."
f"\n{icons.in_progress} Updating the '{dataset}' semantic model..."
dfC_agg = dfC[dfC["Table Name"] == aggTableName]

if len(dfC_agg) == 0:
print(f"Creating the '{aggTableName}' table...")
print(f"{icons.in_progress} Creating the '{aggTableName}' table...")
exp = m.Expressions["DatabaseQuery"]
tbl = TOM.Table()
tbl.Name = aggTableName
Expand Down Expand Up @@ -318,15 +318,15 @@ def generate_aggs(

tbl.Columns.Add(col)
print(
f"The '{aggTableName}'[{cName}] column has been added to the '{dataset}' semantic model."
f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added to the '{dataset}' semantic model."
)

m.Tables.Add(tbl)
print(
f"The '{aggTableName}' table has been added to the '{dataset}' semantic model."
f"{icons.green_dot} The '{aggTableName}' table has been added to the '{dataset}' semantic model."
)
else:
print(f"Updating the '{aggTableName}' table's columns...")
print(f"{icons.in_progress} Updating the '{aggTableName}' table's columns...")
# Remove existing columns
for t in m.Tables:
tName = t.Name
Expand All @@ -347,12 +347,12 @@ def generate_aggs(
col.DataType = System.Enum.Parse(TOM.DataType, dType)

m.Tables[aggTableName].Columns.Add(col)
print(f"The '{aggTableName}'[{cName}] column has been added.")
print(f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added.")

# Create relationships
relMap = {"m": "Many", "1": "One", "0": "None"}

print(f"\nGenerating necessary relationships...")
print(f"\n{icons.in_progress} Generating necessary relationships...")
for i, r in dfR.iterrows():
fromTable = r["From Table"]
fromColumn = r["From Column"]
Expand Down Expand Up @@ -384,27 +384,27 @@ def generate_aggs(
rel.FromColumn = m.Tables[aggTableName].Columns[fromColumn]
m.Relationships.Add(rel)
print(
f"'{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has been added."
f"{icons.green_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has been added."
)
except:
print(
f"'{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has not been created."
f"{icons.red_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has not been created."
)
elif toTable == table_name:
try:
rel.ToColumn = m.Tables[aggTableName].Columns[toColumn]
m.Relationships.Add(rel)
print(
f"'{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has been added."
f"{icons.green_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has been added."
)
except:
print(
f"'{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has not been created."
f"{icons.red_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has not been created."
)
f"Relationship creation is complete."

# Create IF measure
f"\nCreating measure to check if the agg table can be used..."
f"\n{icons.in_progress} Creating measure to check if the agg table can be used..."
aggChecker = "IF("
dfR_filt = dfR[
(dfR["From Table"] == table_name) & (~dfR["From Column"].isin(columnValues))
Expand All @@ -419,7 +419,7 @@ def generate_aggs(
print(aggChecker)

# Todo: add IFISFILTERED clause for columns
f"\n Creating the base measures in the agg table..."
f"\n{icons.in_progress} Creating the base measures in the agg table..."
# Create base agg measures
dep = fabric.evaluate_dax(
dataset=dataset,
Expand Down
1 change: 1 addition & 0 deletions src/sempy_labs/_clear_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
def clear_cache(dataset: str, workspace: Optional[str] = None):
"""
Clears the cache of a semantic model.
See `here <https://learn.microsoft.com/analysis-services/instances/clear-the-analysis-services-caches?view=asallproducts-allversions>`_ for documentation.
Parameters
----------
Expand Down
2 changes: 1 addition & 1 deletion src/sempy_labs/_dax.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@


@log
def run_dax(
def evaluate_dax_impersonation(
dataset: str,
dax_query: str,
user_name: Optional[str] = None,
Expand Down
5 changes: 5 additions & 0 deletions src/sempy_labs/_icons.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,8 @@
yellow_dot = "\U0001F7E1"
red_dot = "\U0001F534"
in_progress = "⌛"
checked = "\u2611"
unchecked = "\u2610"
start_bold = "\033[1m"
end_bold = "\033[0m"
bullet = "\u2022"
2 changes: 1 addition & 1 deletion src/sempy_labs/_list_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -1308,7 +1308,7 @@ def list_kpis(dataset: str, workspace: Optional[str] = None):
]
)

for t in tom.model.Tables:
for t in tom._model.Tables:
for m in t.Measures:
if m.KPI is not None:
new_data = {
Expand Down
20 changes: 10 additions & 10 deletions src/sempy_labs/_model_bpa.py
Original file line number Diff line number Diff line change
Expand Up @@ -1008,7 +1008,7 @@ def run_model_bpa(
dfM["Referenced By"].fillna(0, inplace=True)
dfM["Referenced By"] = dfM["Referenced By"].fillna(0).astype(int)

pattern = "[^\( ][a-zA-Z0-9_()-]+\[[^\[]+\]|'[^']+'\[[^\[]+\]|\[[^\[]+\]"
pattern = r"[^\( ][a-zA-Z0-9_()-]+\[[^\[]+\]|'[^']+'\[[^\[]+\]|\[[^\[]+\]"

dfM["Has Fully Qualified Measure Reference"] = False
dfM["Has Unqualified Column Reference"] = False
Expand Down Expand Up @@ -1041,15 +1041,15 @@ def run_model_bpa(

dfM_filt = dfM[
dfM["Measure Expression"].str.contains(
"(?i)USERELATIONSHIP\s*\(\s*'*"
+ fromTable
+ "'*\["
+ fromColumn
+ "\]\s*,\s*'*"
+ toTable
+ "'*\["
+ toColumn
+ "\]",
r"(?i)USERELATIONSHIP\s*\(\s*'*"
+ re.escape(fromTable)
+ r"'*\["
+ re.escape(fromColumn)
+ r"\]\s*,\s*'*"
+ re.escape(toTable)
+ r"'*\["
+ re.escape(toColumn)
+ r"\]",
regex=True,
)
]
Expand Down
18 changes: 9 additions & 9 deletions src/sempy_labs/_one_lake_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from typing import Optional
from sempy._utils._log import log
from sempy_labs._helper_functions import resolve_workspace_name_and_id

import sempy_labs._icons as icons

@log
def export_model_to_onelake(
Expand Down Expand Up @@ -42,7 +42,7 @@ def export_model_to_onelake(

if len(dfD_filt) == 0:
print(
f"The '{dataset}' semantic model does not exist in the '{workspace}' workspace."
f"{icons.red_dot} The '{dataset}' semantic model does not exist in the '{workspace}' workspace."
)
return

Expand All @@ -64,11 +64,11 @@ def export_model_to_onelake(
try:
fabric.execute_tmsl(script=tmsl, workspace=workspace)
print(
f"The '{dataset}' semantic model's tables have been exported as delta tables to the '{workspace}' workspace.\n"
f"{icons.green_dot} The '{dataset}' semantic model's tables have been exported as delta tables to the '{workspace}' workspace.\n"
)
except:
print(
f"ERROR: The '{dataset}' semantic model's tables have not been exported as delta tables to the '{workspace}' workspace."
f"{icons.red_dot} The '{dataset}' semantic model's tables have not been exported as delta tables to the '{workspace}' workspace."
)
print(
f"Make sure you enable OneLake integration for the '{dataset}' semantic model. Follow the instructions here: https://learn.microsoft.com/power-bi/enterprise/onelake-integration-overview#enable-onelake-integration"
Expand All @@ -83,14 +83,14 @@ def export_model_to_onelake(

if len(dfI_filt) == 0:
print(
f"The '{destination_lakehouse}' lakehouse does not exist within the '{destination_workspace}' workspace."
f"{icons.red_dot} The '{destination_lakehouse}' lakehouse does not exist within the '{destination_workspace}' workspace."
)
# Create lakehouse
destination_lakehouse_id = fabric.create_lakehouse(
display_name=destination_lakehouse, workspace=destination_workspace
)
print(
f"The '{destination_lakehouse}' lakehouse has been created within the '{destination_workspace}' workspace.\n"
f"{icons.green_dot} The '{destination_lakehouse}' lakehouse has been created within the '{destination_workspace}' workspace.\n"
)
else:
destination_lakehouse_id = dfI_filt["Id"].iloc[0]
Expand Down Expand Up @@ -122,7 +122,7 @@ def export_model_to_onelake(

client = fabric.FabricRestClient()

print("Creating shortcuts...\n")
print(f"{icons.in_progress} Creating shortcuts...\n")
for tableName in tables:
tablePath = "Tables/" + tableName
shortcutName = tableName.replace(" ", "")
Expand All @@ -145,11 +145,11 @@ def export_model_to_onelake(
)
if response.status_code == 201:
print(
f"\u2022 The shortcut '{shortcutName}' was created in the '{destination_lakehouse}' lakehouse within the '{destination_workspace}' workspace. It is based on the '{tableName}' table in the '{dataset}' semantic model within the '{workspace}' workspace.\n"
f"{icons.bullet} The shortcut '{shortcutName}' was created in the '{destination_lakehouse}' lakehouse within the '{destination_workspace}' workspace. It is based on the '{tableName}' table in the '{dataset}' semantic model within the '{workspace}' workspace.\n"
)
else:
print(response.status_code)
except:
print(
f"ERROR: Failed to create a shortcut for the '{tableName}' table."
f"{icons.red_dot} Failed to create a shortcut for the '{tableName}' table."
)
Loading

0 comments on commit 94ce4f0

Please sign in to comment.