Add P4 meta in SAI spec for lib generation, support merge SAI spec.

dash-pipeline/Makefile

@@ -177,8 +177,9 @@ sai: sai-headers sai-meta libsai
 sai-headers: p4 docker-saithrift-bldr-image-exists | SAI/SAI
 	@echo "Generate SAI library headers and implementation..."
 
-	# Once the specs are checked in, we can use this to revert any local changes before generating the new specs.
-	# git checkout SAI/specs/*
+	# Revert any local changes before generating the new specs.
+	git clean -xf SAI/specs
+	git checkout SAI/specs/*
 
 	mkdir -p SAI/lib
 

dash-pipeline/SAI/sai_api_gen.py

@@ -68,7 +68,8 @@
     print("Outputting new SAI spec to " + sai_spec_dir)
     yaml_inc_ctor.autoload = False
     new_sai_spec = dash_sai_exts.to_sai()
-    new_sai_spec.serialize(sai_spec_dir)
+    sai_spec.merge(new_sai_spec)
+    sai_spec.serialize(sai_spec_dir)
 
     # Generate and update all SAI files
     SAIGenerator(dash_sai_exts).generate()

dash-pipeline/SAI/utils/dash_p4/dash_p4_counter.py

@@ -147,13 +147,13 @@ def generate_counter_sai_attributes(self) -> "Iterator[DashP4Counter]":
     #
     # Functions for generating SAI specs.
     #
-    def _get_sai_name(self, table_name: str) -> str:
+    def get_sai_name(self, table_name: str) -> str:
         if self.attr_type == "stats":
             return f"SAI_{table_name.upper()}_STAT_{self.name.upper()}"
 
         return f"SAI_{table_name.upper()}_{self.name.upper()}"
 
-    def _get_sai_description(self, table_name: str):
+    def get_sai_description(self, table_name: str):
         if self.attr_type == "stats":
             return f"DASH {table_name.upper()} {self.name.upper()} stat count"
 

dash-pipeline/SAI/utils/dash_p4/dash_p4_table.py

@@ -5,7 +5,7 @@
 from .dash_p4_table_action_param import *
 from .dash_p4_table_key import *
 from .dash_p4_table_action import *
-from ..sai_spec import SaiApi, SaiStruct, SaiEnum, SaiEnumMember, SaiAttribute
+from ..sai_spec import SaiApi, SaiStruct, SaiEnum, SaiEnumMember, SaiAttribute, SaiApiP4MetaAction, SaiApiP4MetaTable
 
 
 @dash_p4rt_parser
@@ -264,6 +264,7 @@ def __build_sai_attributes_after_parsing(self):
     #
     def to_sai(self) -> SaiApi:
         sai_api = SaiApi(self.name, "", self.is_object != "false")
+        sai_api.p4_meta.tables.append(SaiApiP4MetaTable(self.id))
 
         self.create_sai_action_enum(sai_api)
         self.create_sai_structs(sai_api)
@@ -276,19 +277,36 @@ def create_sai_action_enum(self, sai_api: SaiApi) -> None:
         # If the table represents an SAI object, it should not have an action enum.
         # If the table has only 1 action, we don't need to create the action enum.
         if len(self.actions) <= 1 and self.is_object != "false":
+            # We still need to create the p4 meta action here for generating default action code in libsai.
+            if len(self.actions) == 1:
+                sai_api.p4_meta.tables[0].actions["default"] = SaiApiP4MetaAction("default", self.actions[0].id)
             return
 
         action_enum_member_value = 0
         action_enum_members: List[SaiEnumMember] = []
         for action in self.actions:
+            action_enum_member_name = f"SAI_{self.name.upper()}_ACTION_{action.name.upper()}"
+
             action_enum_members.append(
                 SaiEnumMember(
-                    name=f"SAI_{self.name.upper()}_ACTION_{action.name.upper()}",
+                    name=action_enum_member_name,
                     description="",
                     value=str(action_enum_member_value),
                 )
             )
 
+            p4_meta_action = SaiApiP4MetaAction(
+                name=action_enum_member_name,
+                id=action.id,
+            )
+
+            for action_param in action.params:
+                p4_meta_action.attr_param_id[action_param.get_sai_name(self.name)] = action_param.id
+
+            sai_api.p4_meta.tables[0].actions[action_enum_member_name] = p4_meta_action
+
+            action_enum_member_value += 1
+
         action_enum_type_name = f"sai_{self.name.lower()}_action_t"
 
         action_enum = SaiEnum(
@@ -307,6 +325,7 @@ def create_sai_action_enum(self, sai_api: SaiApi) -> None:
         )
         sai_api.attributes.append(sai_attr_action)
 
+
     def create_sai_structs(self, sai_api: SaiApi) -> None:
         # The entry struct is only needed for non-object tables.
         if self.is_object != "false":
@@ -329,9 +348,9 @@ def create_sai_stats(self, sai_api: SaiApi) -> None:
         ]
 
     def create_sai_attributes(self, sai_api: SaiApi) -> None:
-        sai_api.attributes.extend(
-            [attr.to_sai_attribute(self.name) for attr in self.sai_attributes if attr.skipattr != "true"]
-        )
+        for attr in self.sai_attributes:
+            if attr.skipattr != "true":
+                sai_api.attributes.append(attr.to_sai_attribute(self.name))
 
         # If the table has an counter attached, we need to create a counter attribute for it.
         # The counter attribute only counts that packets that hits any entry, but not the packet that misses all entries.

dash-pipeline/SAI/utils/dash_p4/dash_p4_table_attribute.py

@@ -95,8 +95,8 @@ def set_sai_type(self, sai_type_info: SAITypeInfo) -> None:
     # Functions for generating SAI specs.
     #
     def to_sai_struct_entry(self, table_name: str) -> SaiStructEntry:
-        name = self._get_sai_name(table_name)
-        description = self._get_sai_description(table_name)
+        name = self.get_sai_name(table_name)
+        description = self.get_sai_description(table_name)
         object_name = f"SAI_OBJECT_TYPE_{self.object_name.upper()}" if self.object_name else None
 
         return SaiStructEntry(
@@ -108,8 +108,8 @@ def to_sai_struct_entry(self, table_name: str) -> SaiStructEntry:
         )
 
     def to_sai_attribute(self, table_name: str) -> SaiAttribute:
-        name = self._get_sai_name(table_name)
-        description = self._get_sai_description(table_name)
+        name = self.get_sai_name(table_name)
+        description = self.get_sai_description(table_name)
 
         default_value = None if self.isreadonly == "true" else self.default
         object_name = f"SAI_OBJECT_TYPE_{self.object_name.upper()}" if self.object_name else None
@@ -129,8 +129,8 @@ def to_sai_attribute(self, table_name: str) -> SaiAttribute:
             valid_only = self.validonly,
         )
 
-    def _get_sai_name(self, table_name: str) -> str:
+    def get_sai_name(self, table_name: str) -> str:
         return f"SAI_{table_name.upper()}_{self.name.upper()}"
 
-    def _get_sai_description(self, table_name: str):
+    def get_sai_description(self, table_name: str):
         return f"Action parameter {self.name.upper()}"

dash-pipeline/SAI/utils/dash_p4/dash_p4_table_group.py

@@ -1,7 +1,7 @@
 from typing import List, Optional
 from .common import *
 from .dash_p4_table import DashP4Table
-from ..sai_spec import SaiApiGroup
+from ..sai_spec import SaiApiGroup, SaiApi
 
 
 class DashP4TableGroup(DashP4Object):
@@ -25,20 +25,23 @@ def add_table(self, table: DashP4Table) -> None:
         self.tables.append(table)
 
     def post_parsing_process(self, all_table_names: List[str]) -> None:
-        self.__ignore_duplicated_tables_in_headers()
-
         for table in self.tables:
             table.post_parsing_process(all_table_names)
-
-    def __ignore_duplicated_tables_in_headers(self) -> None:
-        table_names = set()
+
+    def to_sai(self) -> SaiApiGroup:
+        sai_api_list: List[SaiApi] = []
+        sai_api_map: Dict[str, SaiApi] = {}
 
         for table in self.tables:
-            if table.name in table_names:
+            sai_api = table.to_sai()
+
+            if table.name in sai_api_map:
                 table.ignored_in_header = True
-            table_names.add(table.name)
+                sai_api_map[table.name].p4_meta.tables.extend(sai_api.p4_meta.tables)
+            else:
+                sai_api_map[table.name] = sai_api
+                sai_api_list.append(sai_api)
 
-    def to_sai(self) -> SaiApiGroup:
         sai_api_group = SaiApiGroup(self.app_name, "")
-        sai_api_group.sai_apis = [table.to_sai() for table in self.tables if not table.ignored_in_header]
-        return sai_api_group
+        sai_api_group.sai_apis = sai_api_list
+        return sai_api_group

dash-pipeline/SAI/utils/dash_p4/dash_p4_table_key.py

@@ -61,11 +61,11 @@ def parse_p4rt(self, p4rt_table_key: Dict[str, Any]) -> None:
     #
     # Functions for generating SAI specs.
     #
-    def _get_sai_name(self, table_name: str) -> str:
+    def get_sai_name(self, table_name: str) -> str:
         if self.is_entry_key:
             return self.name
 
         return f"SAI_{table_name.upper()}_{self.name.upper()}"
 
-    def _get_sai_description(self, table_name: str):
+    def get_sai_description(self, table_name: str):
         return f"{self.match_type.capitalize()} matched key {self.name}"

dash-pipeline/SAI/utils/dash_p4/dash_sai_extensions.py

@@ -119,12 +119,14 @@ def post_parsing_process(self) -> None:
     #
     def to_sai(self) -> SaiSpec:
         sai_spec = SaiSpec()
+        sai_spec.api_groups = [api_group.to_sai() for api_group in self.table_groups]
+
         self.create_sai_api_types(sai_spec)
         self.create_sai_object_types(sai_spec)
         self.create_sai_object_entries(sai_spec)
         self.create_sai_enums(sai_spec)
         self.create_sai_port_counters(sai_spec.port_extenstion)
-        sai_spec.api_groups = [api_group.to_sai() for api_group in self.table_groups]
+
         return sai_spec
 
     def create_sai_api_types(self, sai_spec: SaiSpec):

dash-pipeline/SAI/utils/sai_spec/__init__.py

@@ -7,3 +7,4 @@
 from .sai_struct import SaiStruct
 from .sai_struct_entry import SaiStructEntry
 from .sai_attribute import SaiAttribute
+from .sai_api_p4_meta import SaiApiP4Meta, SaiApiP4MetaTable, SaiApiP4MetaAction

dash-pipeline/SAI/utils/sai_spec/sai_api.py

@@ -3,6 +3,8 @@
 from .sai_attribute import SaiAttribute
 from .sai_enum import SaiEnum
 from .sai_struct import SaiStruct
+from .sai_api_p4_meta import SaiApiP4Meta
+from . import sai_spec_utils
 
 
 class SaiApi(SaiCommon):
@@ -17,3 +19,17 @@ def __init__(self, name: str, description: str, is_object: bool = False):
         self.structs: List[SaiStruct] = []
         self.attributes: List[SaiAttribute] = []
         self.stats: List[SaiAttribute] = []
+        self.p4_meta: SaiApiP4Meta = SaiApiP4Meta()
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+
+        self.is_object = other.is_object
+        sai_spec_utils.merge_sai_common_lists(self.enums, other.enums)
+        sai_spec_utils.merge_sai_common_lists(self.structs, other.structs)
+        sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+        sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)
+
+        # The P4 meta can be merged by replacing the old one, since it doesn't affect the ABI,
+        # and the new one is always more up-to-date.
+        self.p4_meta = other.p4_meta

dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py

@@ -1,5 +1,6 @@
 from typing import List
 from .sai_attribute import SaiAttribute
+from . import sai_spec_utils
 
 
 class SaiApiExtension:
@@ -12,3 +13,7 @@ class SaiApiExtension:
     def __init__(self):
         self.attributes: List[SaiAttribute] = []
         self.stats: List[SaiAttribute] = []
+
+    def merge(self, other: "SaiApiExtension"):
+        sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+        sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)

dash-pipeline/SAI/utils/sai_spec/sai_api_group.py

@@ -1,6 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_api import SaiApi
+from . import sai_spec_utils
 
 
 class SaiApiGroup(SaiCommon):
@@ -11,3 +12,20 @@ class SaiApiGroup(SaiCommon):
     def __init__(self, name: str, description: str):
         super().__init__(name, description)
         self.sai_apis: List[SaiApi] = []
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.sai_apis, other.sai_apis)
+
+    def deprecate(self) -> bool:
+        """
+        Deprecate API group.
+
+        When deprecating the API group, we can safely remove it from the list as the
+        net effect is the same as keeping it:
+        - The old API type, object type and object entries will not be changed.
+        - The SAI headers will not be changed, because their API groups are present.
+        - The DASH libsai will not be generated anymore, but it is ok, since we will not
+          use them in the BMv2 anyway.
+        """
+        return True

dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py

@@ -0,0 +1,19 @@
+from typing import Dict, List
+
+
+class SaiApiP4MetaAction:
+    def __init__(self, name: str, id: int):
+        self.name: str = name
+        self.id: int = id
+        self.attr_param_id: Dict[str, int] = {}
+
+
+class SaiApiP4MetaTable:
+    def __init__(self, id: int):
+        self.id: int = id
+        self.actions: Dict[str, SaiApiP4MetaAction] = {}
+
+
+class SaiApiP4Meta:
+    def __init__(self):
+        self.tables: List[SaiApiP4MetaTable] = []

dash-pipeline/SAI/utils/sai_spec/sai_attribute.py

@@ -31,3 +31,11 @@ def __init__(
         self.allow_null = allow_null
         self.valid_only = valid_only
         self.deprecated = deprecated
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.__dict__.update(other.__dict__)
+
+    def deprecate(self) -> bool:
+        self.deprecated = True
+        return False

dash-pipeline/SAI/utils/sai_spec/sai_common.py

@@ -6,3 +6,21 @@ class SaiCommon:
     def __init__(self, name: str, description: str):
         self.name: str = name
         self.description: str = description
+
+    def merge(self, other: "SaiCommon"):
+        """
+        Merge the other SaiCommon object into this object.
+        """
+        if not isinstance(other, type(self)):
+            raise TypeError(f"Cannot merge {type(self)} with {type(other)}")
+
+        self.description = other.description
+
+    def deprecate(self) -> bool:
+        """
+        Deprecate this object.
+        
+        If the value doesn't support deprecation marking, we don't do anything
+        but return False to keep it in the list.
+        """
+        return False

dash-pipeline/SAI/utils/sai_spec/sai_enum.py

@@ -1,6 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_enum_member import SaiEnumMember
+from . import sai_spec_utils
 
 
 class SaiEnum(SaiCommon):
@@ -11,3 +12,7 @@ class SaiEnum(SaiCommon):
     def __init__(self, name: str, description: str, members: List[SaiEnumMember] = []):
         super().__init__(name, description)
         self.members: List[SaiEnumMember] = members
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.members, other.members)

dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py

@@ -10,3 +10,7 @@ class SaiEnumMember(SaiCommon):
     def __init__(self, name: str, description: str, value: str):
         super().__init__(name, description)
         self.value: str = value
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.value = other.value