Merge SAI spec.

dash-pipeline/SAI/sai_api_gen.py

@@ -68,7 +68,8 @@
     print("Outputting new SAI spec to " + sai_spec_dir)
     yaml_inc_ctor.autoload = False
     new_sai_spec = dash_sai_exts.to_sai()
-    new_sai_spec.serialize(sai_spec_dir)
+    sai_spec.merge(new_sai_spec)
+    sai_spec.serialize(sai_spec_dir)
 
     # Generate and update all SAI files
     SAIGenerator(dash_sai_exts).generate()

dash-pipeline/SAI/utils/sai_spec/sai_api.py

@@ -4,6 +4,7 @@
 from .sai_enum import SaiEnum
 from .sai_struct import SaiStruct
 from .sai_api_p4_meta import SaiApiP4Meta
+from . import sai_spec_utils
 
 
 class SaiApi(SaiCommon):
@@ -19,3 +20,16 @@ def __init__(self, name: str, description: str, is_object: bool = False):
         self.attributes: List[SaiAttribute] = []
         self.stats: List[SaiAttribute] = []
         self.p4_meta: SaiApiP4Meta = SaiApiP4Meta()
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+
+        self.is_object = other.is_object
+        sai_spec_utils.merge_sai_common_lists(self.enums, other.enums)
+        sai_spec_utils.merge_sai_common_lists(self.structs, other.structs)
+        sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+        sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)
+
+        # The P4 meta can be merged by replacing the old one, since it doesn't affect the ABI,
+        # and the new one is always more up-to-date.
+        self.p4_meta = other.p4_meta

dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py

@@ -1,5 +1,6 @@
 from typing import List
 from .sai_attribute import SaiAttribute
+from . import sai_spec_utils
 
 
 class SaiApiExtension:
@@ -12,3 +13,7 @@ class SaiApiExtension:
     def __init__(self):
         self.attributes: List[SaiAttribute] = []
         self.stats: List[SaiAttribute] = []
+
+    def merge(self, other: "SaiApiExtension"):
+        sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+        sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)

dash-pipeline/SAI/utils/sai_spec/sai_api_group.py

@@ -1,6 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_api import SaiApi
+from . import sai_spec_utils
 
 
 class SaiApiGroup(SaiCommon):
@@ -11,3 +12,20 @@ class SaiApiGroup(SaiCommon):
     def __init__(self, name: str, description: str):
         super().__init__(name, description)
         self.sai_apis: List[SaiApi] = []
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.sai_apis, other.sai_apis)
+
+    def deprecate(self) -> bool:
+        """
+        Deprecate API group.
+
+        When deprecating the API group, we can safely remove it from the list as the
+        net effect is the same as keeping it:
+        - The old API type, object type and object entries will not be changed.
+        - The SAI headers will not be changed, because their API groups are present.
+        - The DASH libsai will not be generated anymore, but it is ok, since we will not
+          use them in the BMv2 anyway.
+        """
+        return True

dash-pipeline/SAI/utils/sai_spec/sai_attribute.py

@@ -31,3 +31,11 @@ def __init__(
         self.allow_null = allow_null
         self.valid_only = valid_only
         self.deprecated = deprecated
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.__dict__.update(other.__dict__)
+
+    def deprecate(self) -> bool:
+        self.deprecated = True
+        return False

dash-pipeline/SAI/utils/sai_spec/sai_common.py

@@ -6,3 +6,21 @@ class SaiCommon:
     def __init__(self, name: str, description: str):
         self.name: str = name
         self.description: str = description
+
+    def merge(self, other: "SaiCommon"):
+        """
+        Merge the other SaiCommon object into this object.
+        """
+        if not isinstance(other, type(self)):
+            raise TypeError(f"Cannot merge {type(self)} with {type(other)}")
+
+        self.description = other.description
+
+    def deprecate(self) -> bool:
+        """
+        Deprecate this object.
+        
+        If the value doesn't support deprecation marking, we don't do anything
+        but return False to keep it in the list.
+        """
+        return False

dash-pipeline/SAI/utils/sai_spec/sai_enum.py

@@ -1,6 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_enum_member import SaiEnumMember
+from . import sai_spec_utils
 
 
 class SaiEnum(SaiCommon):
@@ -11,3 +12,7 @@ class SaiEnum(SaiCommon):
     def __init__(self, name: str, description: str, members: List[SaiEnumMember] = []):
         super().__init__(name, description)
         self.members: List[SaiEnumMember] = members
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.members, other.members)

dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py

@@ -10,3 +10,7 @@ class SaiEnumMember(SaiCommon):
     def __init__(self, name: str, description: str, value: str):
         super().__init__(name, description)
         self.value: str = value
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.value = other.value

dash-pipeline/SAI/utils/sai_spec/sai_spec.py

@@ -6,6 +6,7 @@
 from .sai_api_group import SaiApiGroup
 from .sai_api_extension import SaiApiExtension
 from .sai_struct_entry import SaiStructEntry
+from . import sai_spec_utils
 
 
 class SaiSpec:
@@ -24,12 +25,16 @@ def __init__(self):
     def serialize(self, spec_dir: str):
         yaml_inc_files = []
         for api_group in self.api_groups:
-            sai_api_group_spec_file_path = os.path.join(spec_dir, api_group.name + ".yaml")
+            sai_api_group_spec_file_path = os.path.join(
+                spec_dir, api_group.name + ".yaml"
+            )
 
             with open(sai_api_group_spec_file_path, "w") as f:
                 f.write(yaml.dump(api_group, indent=2, sort_keys=False))
-
-            yaml_inc_files.append(yaml_include.Data(urlpath=sai_api_group_spec_file_path))
+
+            yaml_inc_files.append(
+                yaml_include.Data(urlpath=sai_api_group_spec_file_path)
+            )
 
         api_groups = self.api_groups
         self.api_groups = yaml_inc_files
@@ -43,3 +48,21 @@ def serialize(self, spec_dir: str):
     def deserialize(spec_dir: str):
         with open(os.path.join(spec_dir, "sai_spec.yaml")) as f:
             return yaml.unsafe_load(f)
+
+    def merge(self, other: "SaiSpec"):
+        sai_spec_utils.merge_sai_value_lists(
+            self.api_types, other.api_types, lambda x: x
+        )
+        sai_spec_utils.merge_sai_value_lists(
+            self.object_types, other.object_types, lambda x: x
+        )
+        sai_spec_utils.merge_sai_common_lists(self.object_entries, other.object_entries)
+
+        # The global enums are generated from the P4 enum types, so we can respect whatever in the
+        # new spec and simply replace them, because:
+        # - It doesn't matter if the order of enum itself changes.
+        # - We cannot move the enum members as we want, as their order changes their values.
+        self.enums = other.enums
+
+        self.port_extenstion.merge(other.port_extenstion)
+        sai_spec_utils.merge_sai_common_lists(self.api_groups, other.api_groups)

dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py

@@ -0,0 +1,50 @@
+from typing import Any, List, Callable
+from .sai_common import SaiCommon
+
+
+def merge_sai_value_lists(
+    target: List[Any],
+    source: List[Any],
+    get_key: Callable[[Any], str],
+    on_conflict: Callable[[Any, Any], None] = lambda x, y: x,
+    on_deprecate: Callable[[Any], bool] = lambda x: False
+) -> None:
+    """
+    Merge 2 SAI value lists from source list into target.
+
+    Since we could not remove the old value or change the order of old values, the merge
+    is done as below:
+    - Any new values will be added in the end of the list.
+    - Any values that collapse with existing values will invoke on_conflict callback to resolve.
+    - Any values that needs to be removed will invoke on_deprecate function to deprecate. By default,
+      it will not be removed from the old list.
+    """
+    target_dict = {get_key(item): item for item in target}
+
+    source_keys = set()
+    for source_item in source:
+        source_key = get_key(source_item)
+        source_keys.add(source_key)
+
+        if source_key in target_dict:
+            target_item = target_dict[source_key]
+            on_conflict(target_item, source_item)
+        else:
+            target.append(source_item)
+            target_dict[source_key] = source_item
+
+    # Remove all items in target, if its key doesn't exist in source_keys and on_deprecate returns True.
+    target[:] = [item for item in target if get_key(item) in source_keys or not on_deprecate(item)]
+
+
+def merge_sai_common_lists(
+    target: List[SaiCommon],
+    source: List[SaiCommon],
+) -> None:
+    merge_sai_value_lists(
+        target,
+        source,
+        get_key=lambda x: x.name,
+        on_conflict=lambda x, y: x.merge(y),
+        on_deprecate=lambda x: x.deprecate(),
+    )

dash-pipeline/SAI/utils/sai_spec/sai_struct.py

@@ -1,7 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_struct_entry import SaiStructEntry
-
+from . import sai_spec_utils
 
 class SaiStruct(SaiCommon):
     """
@@ -11,3 +11,7 @@ class SaiStruct(SaiCommon):
     def __init__(self, name: str, description: str, members: List[SaiStructEntry] = []):
         super().__init__(name, description)
         self.members: List[SaiStructEntry] = members
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.members, other.members)

dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py

@@ -19,3 +19,7 @@ def __init__(
         self.type = type
         self.objects = objects
         self.valid_only = valid_only
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.__dict__.update(other.__dict__)