Merge branch 'm-kovalsky/visualizerefresh'

microsoft · Oct 23, 2024 · b897b8e · b897b8e
2 parents b961377 + fc96c4a
commit b897b8e
Show file tree

Hide file tree

Showing 6 changed files with 446 additions and 59 deletions.
diff --git a/environment.yml b/environment.yml
@@ -6,7 +6,7 @@ dependencies:
 - pytest-cov
 - pytest-mock
 - pip:
-  - semantic-link-sempy>=0.8.0
+  - semantic-link-sempy>=0.8.1
   - azure-identity==1.7.1
   - azure-storage-blob>=12.9.0
   - pandas-stubs

diff --git a/pyproject.toml b/pyproject.toml
@@ -23,7 +23,7 @@ classifiers = [
 license= { text = "MIT License" }
 
 dependencies = [
-    "semantic-link-sempy>=0.8.0",
+    "semantic-link-sempy>=0.8.1",
     "anytree",
     "powerbiclient",
     "polib",

diff --git a/src/sempy_labs/__init__.py b/src/sempy_labs/__init__.py
@@ -215,6 +215,7 @@
 from sempy_labs._refresh_semantic_model import (
     refresh_semantic_model,
     cancel_dataset_refresh,
+    get_semantic_model_refresh_history,
 )
 from sempy_labs._translations import translate_semantic_model
 from sempy_labs._vertipaq import (
@@ -383,4 +384,5 @@
     "patch_workload",
     "update_notebook_definition",
     "create_notebook",
+    "get_semantic_model_refresh_history",
 ]
diff --git a/src/sempy_labs/_helper_functions.py b/src/sempy_labs/_helper_functions.py
@@ -13,6 +13,8 @@
 import sempy_labs._icons as icons
 import urllib.parse
 from azure.core.credentials import TokenCredential, AccessToken
+import numpy as np
+from IPython.display import display, HTML
 
 
 def create_abfss_path(
@@ -1147,6 +1149,150 @@ def _get_max_run_id(lakehouse: str, table_name: str) -> int:
     return max_run_id
 
 
-def make_list_unique(my_list):
+def _make_list_unique(my_list):
 
     return list(set(my_list))
+
+
+def _get_partition_map(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
+
+    if workspace is None:
+        workspace = fabric.resolve_workspace_name()
+
+    partitions = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+    select [ID] AS [PartitionID], [TableID], [Name] AS [PartitionName] from $system.tmschema_partitions
+    """,
+    )
+
+    tables = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+    select [ID] AS [TableID], [Name] AS [TableName] from $system.tmschema_tables
+    """,
+    )
+
+    partition_map = pd.merge(partitions, tables, on="TableID", how="left")
+    partition_map["PartitionID"] = partition_map["PartitionID"].astype(str)
+    partition_counts = partition_map.groupby("TableID")["PartitionID"].transform(
+        "count"
+    )
+    partition_map["Object Name"] = partition_map.apply(
+        lambda row: (
+            f"'{row['TableName']}'[{row['PartitionName']}]"
+            if partition_counts[row.name] > 1
+            else row["TableName"]
+        ),
+        axis=1,
+    )
+    return partition_map
+
+
+def _show_chart(spec, title):
+
+    h = f"""
+    <!DOCTYPE html>
+    <html>
+        <head>
+            <script src="https://cdn.jsdelivr.net/npm/vega@5"></script>
+            <script src="https://cdn.jsdelivr.net/npm/vega-lite@5"></script>
+            <script src="https://cdn.jsdelivr.net/npm/vega-embed@6"></script>
+            <style>
+                table, th, td {{
+                border: 10px solid #e7e9eb;
+                border-collapse: collapse;
+                }}
+            </style>
+        </head>
+        <body>
+            <table>
+                <tr>
+                    <td style="text-align: center;">
+                        <h1>{title}</h1>
+                    </td>
+                </tr>
+                <tr>
+                    <td>
+                        <div id="vis"></div>
+                    </td>
+                </tr>
+            </table>    
+            <script type="text/javascript">
+                var spec = {spec};
+                var opt = {{"renderer": "canvas", "actions": false}};
+                vegaEmbed("#vis", spec, opt);
+            </script>
+        </body>
+    </html>"""
+
+    display(HTML(h))
+
+
+def _process_and_display_chart(df, title, widget):
+
+    # Convert time columns to milliseconds
+    df["Start"] = df["Start Time"].astype(np.int64) / int(1e6)
+    df["End"] = df["End Time"].astype(np.int64) / int(1e6)
+
+    # Calculate the time offset for proper Gantt chart rendering
+    Offset = min(df["Start"])
+    df["Start"] = df["Start"] - Offset
+    df["End"] = df["End"] - Offset
+
+    # Vega-Lite spec for Gantt chart
+    spec = (
+        """{
+        "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+        "description": "A simple bar chart with ranged data (aka Gantt Chart).",
+        "data": { "values": """
+        + df.to_json(orient="records")
+        + """ },
+        "width": 700,
+        "height": 400,
+        "mark": "bar",
+        "encoding": {
+            "y": {
+                "field": "Object Name",
+                "type": "ordinal",
+                "axis": {
+                    "labelFontSize": 15,
+                    "titleFontSize": 20,
+                    "title": "Object"
+                }
+            },
+            "x": {
+                "field": "Start", 
+                "type": "quantitative", 
+                "title": "milliseconds",
+                "axis": {
+                    "titleFontSize": 20
+                }
+            },
+            "x2": {"field": "End"},
+            "color": {
+                "field": "Event Subclass",
+                "scale": {
+                    "domain": ["Process", "ExecuteSql"],
+                    "range": ["#FFC000","#0070C0"]
+                },
+                "legend": {
+                    "labelFontSize": 20,
+                    "titleFontSize": 20,
+                    "title": "Event Type"
+                }
+            },
+            "tooltip": [
+                {"field": "Duration", "type": "quantitative", "format": ","},
+                {"field": "Cpu Time", "type": "quantitative", "format": ","},
+                {"field": "Event Subclass", "type": "nominal"}
+            ]
+        }
+    }"""
+    )
+
+    with widget:
+        widget.clear_output(wait=True)
+    _show_chart(spec, title=title)
diff --git a/src/sempy_labs/_icons.py b/src/sempy_labs/_icons.py
@@ -93,14 +93,15 @@
     "P4": "F512",
     "P5": "F1024",
 }
-refreshTypes = [
-    "full",
-    "automatic",
-    "dataOnly",
-    "calculate",
-    "clearValues",
-    "defragment",
-]
+
+refresh_type_mapping = {
+    "full": "full",
+    "auto": "automatic",
+    "data": "dataOnly",
+    "calc": "calculate",
+    "clear": "clearValues",
+    "defrag": "defragment",
+}
 
 itemTypes = {
     "DataPipeline": "dataPipelines",
@@ -146,3 +147,17 @@
 sll_ann_name = "PBI_ProTooling"
 sll_prefix = "SLL_"
 sll_tags = []
+base_cols = ["EventClass", "EventSubclass", "CurrentTime", "TextData"]
+end_cols = base_cols + [
+    "StartTime",
+    "EndTime",
+    "Duration",
+    "CpuTime",
+    "Success",
+    "IntegerData",
+    "ObjectID",
+]
+refresh_event_schema = {
+    "JobGraph": base_cols,
+    "ProgressReportEnd": end_cols,
+}