Merge branch 'm-kovalsky/sourcelineagetagfix'

microsoft · Oct 14, 2024 · 9afa8b8 · 9afa8b8
2 parents b4bd93d + 4f9865d
commit 9afa8b8
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 35 deletions.
diff --git a/README.md b/README.md
@@ -33,7 +33,7 @@ Semantic Link Labs is a Python library designed for use in [Microsoft Fabric not
     * [Migrating a Power BI Premium capacity (P sku) to a Fabric capacity (F sku)](https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Capacity%20Migration.ipynb)
     * [Create](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.html#sempy_labs.create_fabric_capacity)/[update](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.html#sempy_labs.update_fabric_capacity)/[suspend](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.html#sempy_labs.suspend_fabric_capacity)/[resume](https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.html#sempy_labs.resume_fabric_capacity) Fabric capacities.
 * APIs
-    * Wrapper functions for [Power BI](https://learn.microsoft.com/rest/api/power-bi/), [Fabric](https://learn.microsoft.com/rest/api/fabric/articles/using-fabric-apis), and [Azure](https://learn.microsoft.com/rest/api/azure/?view=rest-power-bi-embedded-2021-01-01) APIs
+    * Wrapper functions for [Power BI](https://learn.microsoft.com/rest/api/power-bi/), [Fabric](https://learn.microsoft.com/rest/api/fabric/articles/using-fabric-apis), and [Azure (Fabric Capacity)](https://learn.microsoft.com/rest/api/microsoftfabric/fabric-capacities?view=rest-microsoftfabric-2023-11-01) APIs
 
 
 ### Check out the [helper notebooks](https://github.com/microsoft/semantic-link-labs/tree/main/notebooks) for getting started!

diff --git a/src/sempy_labs/_model_bpa_bulk.py b/src/sempy_labs/_model_bpa_bulk.py
@@ -49,8 +49,6 @@ def run_model_bpa_bulk(
         The semantic models to always skip when running this analysis.
     """
 
-    import pyspark.sql.functions as F
-
     if not lakehouse_attached():
         raise ValueError(
             f"{icons.red_dot} No lakehouse is attached to this notebook. Must attach a lakehouse to the notebook."

diff --git a/src/sempy_labs/_notebooks.py b/src/sempy_labs/_notebooks.py
@@ -8,7 +8,6 @@
     resolve_workspace_name_and_id,
     lro,
     _decode_b64,
-    resolve_notebook_id,
 )
 from sempy.fabric.exceptions import FabricHTTPException
 

diff --git a/src/sempy_labs/report/_generate_report.py b/src/sempy_labs/report/_generate_report.py
@@ -11,7 +11,6 @@
 )
 import sempy_labs._icons as icons
 from sempy._utils._log import log
-from sempy.fabric.exceptions import FabricHTTPException
 
 
 def create_report_from_reportjson(

diff --git a/src/sempy_labs/report/_report_rebind.py b/src/sempy_labs/report/_report_rebind.py
@@ -1,5 +1,8 @@
 import sempy.fabric as fabric
-from sempy_labs._helper_functions import resolve_dataset_id, resolve_report_id
+from sempy_labs._helper_functions import (
+    resolve_dataset_id,
+    resolve_report_id,
+)
 from typing import Optional, List
 from sempy._utils._log import log
 import sempy_labs._icons as icons

diff --git a/src/sempy_labs/tom/_model.py b/src/sempy_labs/tom/_model.py
@@ -33,6 +33,8 @@ class TOMWrapper:
     _workspace: str
     _readonly: bool
     _tables_added: List[str]
+    _table_map = dict
+    _column_map = dict
 
     def __init__(self, dataset, workspace, readonly):
         self._dataset = dataset
@@ -45,6 +47,18 @@ def __init__(self, dataset, workspace, readonly):
         )
         self.model = self._tom_server.Databases.GetByName(dataset).Model
 
+        self._table_map = {}
+        self._column_map = {}
+        for t in self.model.Tables:
+            if len(t.LineageTag) == 0:
+                t.LineageTag = generate_guid()
+            self._table_map[t.LineageTag] = t.Name
+
+        for c in self.all_columns():
+            if len(c.LineageTag) == 0:
+                c.LineageTag = generate_guid()
+            self._column_map[c.LineageTag] = [c.Name, c.DataType]
+
     def all_columns(self):
         """
         Outputs a list of all columns within all tables in the semantic model.
@@ -291,8 +305,6 @@ def add_measure(
             obj.LineageTag = generate_guid()
         if source_lineage_tag is not None:
             obj.SourceLineageTag = source_lineage_tag
-        else:
-            obj.SourceLineageTag = generate_guid()
         if detail_rows_expression is not None:
             drd = TOM.DetailRowsDefinition()
             drd.Expression = detail_rows_expression
@@ -388,8 +400,6 @@ def add_calculated_table_column(
             obj.LineageTag = generate_guid()
         if source_lineage_tag is not None:
             obj.SourceLineageTag = source_lineage_tag
-        else:
-            obj.SourceLineageTag = generate_guid()
         self.model.Tables[table_name].Columns.Add(obj)
 
     def add_data_column(
@@ -478,8 +488,6 @@ def add_data_column(
             obj.LineageTag = generate_guid()
         if source_lineage_tag is not None:
             obj.SourceLineageTag = source_lineage_tag
-        else:
-            obj.SourceLineagetTag = generate_guid()
         self.model.Tables[table_name].Columns.Add(obj)
 
     def add_calculated_column(
@@ -568,8 +576,6 @@ def add_calculated_column(
             obj.LineageTag = generate_guid()
         if source_lineage_tag is not None:
             obj.SourceLineageTag = source_lineage_tag
-        else:
-            obj.SourceLineagetTag = generate_guid()
         self.model.Tables[table_name].Columns.Add(obj)
 
     def add_calculation_item(
@@ -785,8 +791,6 @@ def add_hierarchy(
             obj.LineageTag = generate_guid()
         if source_lineage_tag is not None:
             obj.SourceLineageTag = source_lineage_tag
-        else:
-            obj.SourceLineagetTag = generate_guid()
         self.model.Tables[table_name].Hierarchies.Add(obj)
 
         for col in columns:
@@ -795,7 +799,6 @@ def add_hierarchy(
             lvl.Name = levels[columns.index(col)]
             lvl.Ordinal = columns.index(col)
             lvl.LineageTag = generate_guid()
-            lvl.SourceLineageTag = generate_guid()
             self.model.Tables[table_name].Hierarchies[hierarchy_name].Levels.Add(lvl)
 
     def add_relationship(
@@ -969,8 +972,6 @@ def add_expression(
             exp.LineageTag = generate_guid()
         if source_lineage_tag is not None:
             exp.SourceLineageTag = source_lineage_tag
-        else:
-            exp.SourceLineageTag = generate_guid()
         exp.Kind = TOM.ExpressionKind.M
         exp.Expression = expression
 
@@ -2654,8 +2655,6 @@ def add_table(
             t.LineageTag = generate_guid()
         if source_lineage_tag is not None:
             t.SourceLineageTag = source_lineage_tag
-        else:
-            t.SourceLineagetTag = generate_guid()
         t.Hidden = hidden
         self.model.Tables.Add(t)
 
@@ -2710,8 +2709,6 @@ def add_calculated_table(
             t.LineageTag = generate_guid()
         if source_lineage_tag is not None:
             t.SourceLineageTag = source_lineage_tag
-        else:
-            t.SourceLineagetTag = generate_guid()
         t.Hidden = hidden
         t.Partitions.Add(par)
         self.model.Tables.Add(t)
@@ -4281,33 +4278,57 @@ def update_lineage_tags(self):
                             icons.default_schema
                         )
                     t.SourceLineageTag = f"[{schema_name}].[{entity_name}]"
-                else:
-                    t.SourceLineageTag = generate_guid()
         for c in self.all_columns():
             if len(c.LineageTag) == 0:
                 c.LineageTag = generate_guid()
-            if len(c.SourceLineageTag) == 0:
-                c.SourceLineageTag = generate_guid()
         for m in self.all_measures():
             if len(m.LineageTag) == 0:
                 m.LineageTag = generate_guid()
-            if len(m.SourceLineageTag) == 0:
-                m.SourceLineageTag = generate_guid()
         for h in self.all_hierarchies():
             if len(h.LineageTag) == 0:
                 h.LineageTag = generate_guid()
-            if len(h.SourceLineageTag) == 0:
-                h.SourceLineageTag = generate_guid()
         for lvl in self.all_levels():
             if len(lvl.LineageTag) == 0:
                 lvl.LineageTag = generate_guid()
-            if len(lvl.SourceLineageTag) == 0:
-                lvl.SourceLineageTag = generate_guid()
         for e in self.model.Expressions():
             if len(e.LineageTag) == 0:
                 e.LineageTag = generate_guid()
-            if len(e.SourceLineageTag) == 0:
-                e.SourceLineageTag = generate_guid()
+
+    def add_changed_property(self, object, property: str):
+        """
+        Adds a `ChangedProperty <https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.changedproperty.property?view=analysisservices-dotnet#microsoft-analysisservices-tabular-changedproperty-property>`_ property to a semantic model object. Only adds the property if it does not already exist for the object.
+
+        Parameters
+        ----------
+        object : TOM Object
+            The TOM object within the semantic model.
+        property : str
+            The property to set (i.e. 'Name', 'DataType').
+        """
+
+        import Microsoft.AnalysisServices.Tabular as TOM
+
+        # Only add the property if it does not already exist for that object
+        if not any(c.Property == property for c in object.ChangedProperties):
+            cp = TOM.ChangedProperty()
+            cp.Property = property
+            object.ChangedProperties.Add(cp)
+
+    def remove_changed_property(self, object, property: str):
+        """
+        Removes a `ChangedProperty <https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.changedproperty.property?view=analysisservices-dotnet#microsoft-analysisservices-tabular-changedproperty-property>`_ property to a semantic model object. Only adds the property if it does not already exist for the object.
+
+        Parameters
+        ----------
+        object : TOM Object
+            The TOM object within the semantic model.
+        property : str
+            The property to set (i.e. 'Name', 'DataType').
+        """
+
+        for cp in object.ChangedProperties:
+            if cp.Property == property:
+                object.ChangedProperties.Remove(cp)
 
     def generate_measure_descriptions(
         self,
@@ -4456,6 +4477,29 @@ def generate_measure_descriptions(
 
     def close(self):
         if not self._readonly and self.model is not None:
+
+            import Microsoft.AnalysisServices.Tabular as TOM
+
+            # ChangedProperty logic
+            for t in self.model.Tables:
+                if any(
+                    p.SourceType == TOM.PartitionSourceType.Entity for p in t.Partitions
+                ):
+                    if t.LineageTag in list(self._table_map.keys()):
+                        if self._table_map.get(t.LineageTag) != t.Name:
+                            self.add_changed_property(object=t, property="Name")
+
+            for c in self.all_columns():
+                if c.LineageTag in list(self._column_map.keys()):
+                    if any(
+                        p.SourceType == TOM.PartitionSourceType.Entity
+                        for p in c.Parent.Partitions
+                    ):
+                        if self._column_map.get(c.LineageTag)[0] != c.Name:
+                            self.add_changed_property(object=c, property="Name")
+                    if self._column_map.get(c.LineageTag)[1] != c.DataType:
+                        self.add_changed_property(object=c, property="DataType")
+
             self.model.SaveChanges()
 
             if len(self._tables_added) > 0: