fix(evpn): fix graceful shutdown, tracer option

Signed-off-by: Saikumar, Banoth <[email protected]>

cmd/main.go

@@ -44,6 +44,7 @@ import (
 	intel_e2000_linux "github.com/opiproject/opi-evpn-bridge/pkg/LinuxVendorModule/intele2000"
 	frr "github.com/opiproject/opi-evpn-bridge/pkg/frr"
 	netlink "github.com/opiproject/opi-evpn-bridge/pkg/netlink"
+	"github.com/opiproject/opi-evpn-bridge/pkg/vendor_plugins/intel-e2000/p4runtime/p4driverapi"
 	ipu_vendor "github.com/opiproject/opi-evpn-bridge/pkg/vendor_plugins/intel-e2000/p4runtime/p4translation"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 )
@@ -73,7 +74,7 @@ var rootCmd = &cobra.Command{
 			intel_e2000_linux.Initialize()
 			frr.Initialize()
 			ipu_vendor.Initialize()
-			netlink.DeInitialize()
+
 		case "ci":
 			gen_linux.Initialize()
 			ci_linux.Initialize()
@@ -91,7 +92,6 @@ var rootCmd = &cobra.Command{
 			netlink.Initialize()
 		default:
 		}
-
 		runGrpcServer(config.GlobalConfig.GRPCPort, config.GlobalConfig.TLSFiles)
 
 	},
@@ -135,9 +135,6 @@ func setupLogger(filename string) {
 
 func cleanUp() {
 	log.Println("Defer function called")
-	if err := deleteGrdVrf(); err != nil {
-		log.Println("Failed to delete GRD vrf")
-	}
 	if err := infradb.DeleteAllResources(); err != nil {
 		log.Println("Failed to delete all the resources: ", err)
 	}
@@ -146,7 +143,10 @@ func cleanUp() {
 		gen_linux.DeInitialize()
 		intel_e2000_linux.DeInitialize()
 		frr.DeInitialize()
+		netlink.DeInitialize()
 		ipu_vendor.DeInitialize()
+		close(p4driverapi.StopCh)
+
 	case "ci":
 		gen_linux.DeInitialize()
 		ci_linux.DeInitialize()
@@ -205,12 +205,14 @@ func main() {
 
 // runGrpcServer start the grpc server for all the components
 func runGrpcServer(grpcPort uint16, tlsFiles string) {
-	tp := utils.InitTracerProvider("opi-evpn-bridge")
-	defer func() {
-		if err := tp.Shutdown(context.Background()); err != nil {
-			log.Panicf("Tracer Provider Shutdown: %v", err)
-		}
-	}()
+	if config.GlobalConfig.Tracer {
+		tp := utils.InitTracerProvider("opi-evpn-bridge")
+		defer func() {
+			if err := tp.Shutdown(context.Background()); err != nil {
+				log.Panicf("Tracer Provider Shutdown: %v", err)
+			}
+		}()
+	}
 
 	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort))
 	if err != nil {
@@ -313,16 +315,3 @@ func createGrdVrf() error {
 
 	return nil
 }
-
-// deleteGrdVrf creates the grd vrf with vni 0
-func deleteGrdVrf() error {
-	log.Printf("DeleteGrdVrf(): deleted GRD VRF object\n")
-
-	err := infradb.DeleteVrf("//network.opiproject.org/vrfs/GRD")
-	if err != nil {
-		log.Printf("CreateGrdVrf(): Error in deleting GRD VRF object %+v\n", err)
-		return err
-	}
-
-	return nil
-}

config-intel-e2000.yaml

@@ -4,6 +4,7 @@ tlsfiles:
 database: redis
 dbaddress: 127.0.0.1:6379
 buildenv: intel_e2000
+tracer: false
 subscribers:
  - name: "lvm"
    priority: 2

config.yaml

@@ -4,6 +4,7 @@ tlsfiles:
 database: redis
 dbaddress: 127.0.0.1:6379
 buildenv: ci
+tracer: false
 subscribers:
  - name: "lgm"
    priority: 1

pkg/LinuxCIModule/lci.go

@@ -226,7 +226,7 @@ func Initialize() {
 		}
 	}
 	ctx = context.Background()
-	nlink = utils.NewNetlinkWrapperWithArgs(false)
+	nlink = utils.NewNetlinkWrapperWithArgs(config.GlobalConfig.Tracer)
 }
 
 // DeInitialize function handles stops functionality

pkg/LinuxGeneralModule/lgm.go

@@ -363,7 +363,7 @@ func Initialize() {
 	brTenant = "br-tenant"
 	ipMtu = config.GlobalConfig.LinuxFrr.IPMtu
 	ctx = context.Background()
-	nlink = utils.NewNetlinkWrapperWithArgs(false)
+	nlink = utils.NewNetlinkWrapperWithArgs(config.GlobalConfig.Tracer)
 	// Set up the static configuration parts
 	_, err := nlink.LinkByName(ctx, brTenant)
 	if err != nil {

pkg/LinuxVendorModule/intele2000/intelE2000.go

@@ -391,7 +391,7 @@ func tearDownVrf(vrf *infradb.Vrf) bool {
 	Intf, err := nlink.LinkByName(ctx, vlanIntf)
 	if err != nil {
 		log.Printf("Failed to get link %v: %s\n", vlanIntf, err)
-		return true
+		return false
 	}
 	if err = nlink.LinkDel(ctx, Intf); err != nil {
 		log.Printf("Failed to delete link %v: %s\n", vlanIntf, err)
@@ -421,7 +421,7 @@ func Initialize() {
 	ipMtu = config.GlobalConfig.LinuxFrr.IPMtu
 	brTenant = "br-tenant"
 	ctx = context.Background()
-	nlink = utils.NewNetlinkWrapperWithArgs(false)
+	nlink = utils.NewNetlinkWrapperWithArgs(config.GlobalConfig.Tracer)
 }
 
 // DeInitialize function handles stops functionality

pkg/config/config.go

@@ -82,6 +82,7 @@ type Config struct {
 	Database    string             `yaml:"database"`
 	DBAddress   string             `yaml:"dbaddress"`
 	Buildenv    string             `yaml:"buildenv"`
+	Tracer      bool               `yaml:"tracer"`
 	Subscribers []SubscriberConfig `yaml:"subscribers"`
 	LinuxFrr    LinuxFrrConfig     `yaml:"linuxfrr"`
 	Netlink     NetlinkConfig      `yaml:"netlink"`

pkg/frr/frr.go

@@ -48,11 +48,24 @@ func (h *ModulefrrHandler) HandleEvent(eventType string, objectData *eventbus.Ob
 }
 
 // handlesvi handles the svi functionality
+//
+//nolint:funlen,gocognit
 func handlesvi(objectData *eventbus.ObjectData) {
 	var comp common.Component
 	svi, err := infradb.GetSvi(objectData.Name)
 	if err != nil {
 		log.Printf("GetSvi error: %s %s\n", err, objectData.Name)
+		comp.Name = frrComp
+		comp.CompStatus = common.ComponentStatusError
+		if comp.Timer == 0 {
+			comp.Timer = 2 * time.Second
+		} else {
+			comp.Timer *= 2
+		}
+		err := infradb.UpdateSviStatus(objectData.Name, objectData.ResourceVersion, objectData.NotificationID, nil, comp)
+		if err != nil {
+			log.Printf("error in updating svi status: %s\n", err)
+		}
 		return
 	}
 
@@ -127,6 +140,17 @@ func handlevrf(objectData *eventbus.ObjectData) {
 	vrf, err := infradb.GetVrf(objectData.Name)
 	if err != nil {
 		log.Printf("GetVRF error: %s %s\n", err, objectData.Name)
+		comp.Name = frrComp
+		comp.CompStatus = common.ComponentStatusError
+		if comp.Timer == 0 { // wait timer is 2 powerof natural numbers ex : 1,2,3...
+			comp.Timer = 2 * time.Second
+		} else {
+			comp.Timer *= 2
+		}
+		err := infradb.UpdateVrfStatus(objectData.Name, objectData.ResourceVersion, objectData.NotificationID, nil, comp)
+		if err != nil {
+			log.Printf("error in updating vrf status: %s\n", err)
+		}
 		return
 	}
 
@@ -256,7 +280,7 @@ func Initialize() {
 	subscribeInfradb(&config.GlobalConfig)
 
 	ctx = context.Background()
-	Frr = utils.NewFrrWrapperWithArgs("localhost", false)
+	Frr = utils.NewFrrWrapperWithArgs("localhost", config.GlobalConfig.Tracer)
 
 	// Make sure IPv4 forwarding is enabled.
 	detail, flag := run([]string{"sysctl", "-w", " net.ipv4.ip_forward=1"}, false)

pkg/infradb/infradb.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"log"
 	"sync"
+	"time"
 
 	"github.com/opiproject/opi-evpn-bridge/pkg/infradb/common"
 	"github.com/opiproject/opi-evpn-bridge/pkg/infradb/subscriberframework/eventbus"
@@ -1217,27 +1218,56 @@ func GetAllSvis() ([]*Svi, error) {
 
 // DeleteAllResources deletes all components from infradb
 func DeleteAllResources() error {
+	duration := 10 * time.Second
+	bps, _ := GetAllBPs()
+	for _, bp := range bps {
+		err := DeleteBP(bp.Name)
+		if err != nil {
+			return err
+		}
+	}
+	startTime := time.Now()
+	for {
+		b, _ := GetAllBPs()
+		if len(b) == 0 {
+			break
+		}
+		if time.Since(startTime) > duration {
+			return errors.New("failed to delete BridgePorts")
+		}
+	}
 	svis, _ := GetAllSvis()
 	for _, svi := range svis {
 		err := DeleteSvi(svi.Name)
 		if err != nil {
 			return err
 		}
 	}
-
+	startTime = time.Now()
+	for {
+		s, _ := GetAllSvis()
+		if len(s) == 0 {
+			break
+		}
+		if time.Since(startTime) > duration {
+			return errors.New("failed to delete svis")
+		}
+	}
 	vrfs, _ := GetAllVrfs()
 	for _, vrf := range vrfs {
 		err := DeleteVrf(vrf.Name)
 		if err != nil {
 			return err
 		}
 	}
-
-	bps, _ := GetAllBPs()
-	for _, bp := range bps {
-		err := DeleteBP(bp.Name)
-		if err != nil {
-			return err
+	startTime = time.Now()
+	for {
+		v, _ := GetAllVrfs()
+		if len(v) == 0 {
+			break
+		}
+		if time.Since(startTime) > duration {
+			return errors.New("failed to delete vrfs")
 		}
 	}
 	lbs, _ := GetAllLBs()
@@ -1247,6 +1277,16 @@ func DeleteAllResources() error {
 			return err
 		}
 	}
+	startTime = time.Now()
+	for {
+		l, _ := GetAllLBs()
+		if len(l) == 0 {
+			break
+		}
+		if time.Since(startTime) > duration {
+			return errors.New("failed to delete LogicalBridges")
+		}
+	}
 	return nil
 }