fix(host): migrate slave addresses and routes from physical nic to br…

…idge

pkg/hostman/hostinfo/hostbridge/hostbridge.go

@@ -37,6 +37,7 @@ import (
 )
 
 type IBridgeDriver interface {
+	MigrateSlaveConfigs(IBridgeDriver) error
 	ConfirmToConfig() (bool, error)
 	GetMac() string
 	GetVlanId() int
@@ -156,6 +157,124 @@ func (d *SBaseBridgeDriver) BringupInterface() error {
 	return nil
 }
 
+func trySetupSlaveAddressesRoutes(o IBridgeDriver, migrateAddrs [][]string, migrateRoutes []iproute2.RouteSpec) error {
+	if len(migrateAddrs) > 0 {
+		tried := 0
+		const MAX_TRIES = 4
+		errs := make([]error, 0)
+		for tried < MAX_TRIES {
+			if err := o.SetupSlaveAddresses(migrateAddrs); err != nil {
+				errs = append(errs, err)
+				log.Errorf("SetupSlaveAddresses fail: %s", err)
+				tried += 1
+				if tried >= MAX_TRIES {
+					return errors.Wrap(errors.NewAggregate(errs), "SetupSlaveAddresses")
+				} else {
+					time.Sleep(time.Duration(tried) * time.Second)
+				}
+			} else {
+				break
+			}
+		}
+	}
+	if len(migrateRoutes) > 0 {
+		tried := 0
+		const MAX_TRIES = 4
+		errs := make([]error, 0)
+		for {
+			if err := o.SetupRoutes(migrateRoutes); err != nil {
+				errs = append(errs, err)
+				log.Errorf("SetupRoutes fail: %s", err)
+				tried += 1
+				if tried >= MAX_TRIES {
+					return errors.Wrap(errors.NewAggregate(errs), "SetupRoutes")
+				} else {
+					time.Sleep(time.Duration(tried) * time.Second)
+				}
+			} else {
+				break
+			}
+		}
+	}
+	return nil
+}
+
+func (d *SBaseBridgeDriver) MigrateSlaveConfigs(o IBridgeDriver) error {
+	if d.inter != nil {
+		migrateAddrs := make([][]string, 0)
+		migrateRoutes := make([]iproute2.RouteSpec, 0)
+		{
+			currentRoutes := d.bridge.GetRouteSpecs()
+			currentSlaves := d.bridge.GetSlaveAddresses()
+			routes := d.inter.GetRouteSpecs()
+			slaveAddrs := d.inter.GetSlaveAddresses()
+
+			log.Infof("to migrate routes: %s slaveAddress: %s", jsonutils.Marshal(routes), jsonutils.Marshal(slaveAddrs))
+
+			for i := range slaveAddrs {
+				if strings.HasPrefix(slaveAddrs[i][0], "fe80:") || strings.HasPrefix(slaveAddrs[i][0], "169.254.") {
+					// skip link local address
+					continue
+				}
+				if slaveAddrs[i][0] == d.bridge.Addr {
+					continue
+				}
+				find := false
+				for j := range currentSlaves {
+					if slaveAddrs[i][0] == currentSlaves[j][0] && slaveAddrs[i][1] == currentSlaves[j][1] {
+						find = true
+						break
+					}
+				}
+				if !find {
+					// need to migrate address
+					migrateAddrs = append(migrateAddrs, slaveAddrs[i])
+				}
+			}
+
+			for i := range routes {
+				if strings.HasPrefix(routes[i].Dst.String(), "fe80:") || strings.HasPrefix(routes[i].Dst.String(), "169.254.") {
+					// skip link local routes
+					continue
+				}
+				find := false
+				for j := range currentRoutes {
+					if routes[i].Dst.String() == currentRoutes[j].Dst.String() {
+						find = true
+						break
+					}
+				}
+				if !find {
+					for j := range slaveAddrs {
+						if routes[i].Dst.String() == fmt.Sprintf("%s/%s", slaveAddrs[j][0], slaveAddrs[j][1]) {
+							find = true
+							break
+						}
+					}
+				}
+				if !find {
+					// need to migrate route
+					migrateRoutes = append(migrateRoutes, routes[i])
+				}
+			}
+		}
+		log.Infof("to migrate routes: %s slaveAddress: %s", jsonutils.Marshal(migrateRoutes), jsonutils.Marshal(migrateAddrs))
+		{
+			err := trySetupSlaveAddressesRoutes(o, migrateAddrs, migrateRoutes)
+			if err != nil {
+				return errors.Wrap(err, "trySetupSlaveAddressesRoutes")
+			}
+		}
+		{
+			err := d.inter.ClearAddrs()
+			if err != nil {
+				return errors.Wrap(err, "ClearAddrs")
+			}
+		}
+	}
+	return nil
+}
+
 func (d *SBaseBridgeDriver) ConfirmToConfig() (bool, error) {
 	exist, err := d.drv.Exists()
 	if err != nil {
@@ -334,43 +453,8 @@ func (d *SBaseBridgeDriver) Setup(o IBridgeDriver) error {
 				return errors.Wrap(err, "SetupAddresses")
 			}
 			time.Sleep(1 * time.Second)
-			if len(slaveAddrs) > 0 {
-				tried := 0
-				const MAX_TRIES = 4
-				errs := make([]error, 0)
-				for tried < MAX_TRIES {
-					if err := o.SetupSlaveAddresses(slaveAddrs); err != nil {
-						errs = append(errs, err)
-						log.Errorf("SetupSlaveAddresses fail: %s", err)
-						tried += 1
-						if tried >= MAX_TRIES {
-							return errors.Wrap(errors.NewAggregate(errs), "SetupSlaveAddresses")
-						} else {
-							time.Sleep(time.Duration(tried) * time.Second)
-						}
-					} else {
-						break
-					}
-				}
-			}
-			if len(routes) > 0 {
-				tried := 0
-				const MAX_TRIES = 4
-				errs := make([]error, 0)
-				for {
-					if err := o.SetupRoutes(routes); err != nil {
-						errs = append(errs, err)
-						log.Errorf("SetupRoutes fail: %s", err)
-						tried += 1
-						if tried >= MAX_TRIES {
-							return errors.Wrap(errors.NewAggregate(errs), "SetupRoutes")
-						} else {
-							time.Sleep(time.Duration(tried) * time.Second)
-						}
-					} else {
-						break
-					}
-				}
+			if err := trySetupSlaveAddressesRoutes(o, slaveAddrs, routes); err != nil {
+				return errors.Wrap(err, "trySetupSlaveAddressesRoutes")
 			}
 		} else {
 			if err := o.SetupAddresses(nil); err != nil {

pkg/hostman/hostinfo/hostinfohelper.go

@@ -288,7 +288,11 @@ func NewNIC(desc string) (*SNIC, error) {
 		}
 		time.Sleep(time.Second * 1)
 	} else {
-		log.Infof("Confirm to configuration!!")
+		log.Infof("Confirm to configuration!! To migrate physical interface configs")
+		err := nic.BridgeDev.MigrateSlaveConfigs(nic.BridgeDev)
+		if err != nil {
+			log.Errorf("fail to migrate configs: %s", err)
+		}
 	}
 	if err := nic.BridgeDev.PersistentConfig(); err != nil {
 		return nil, errors.Wrapf(err, "nic.BridgeDev.PersistentConfig %v", nic.BridgeDev)

pkg/util/iproute2/address_linux.go

@@ -161,3 +161,19 @@ func (address *Address) List4() ([]net.IPNet, error) {
 	}
 	return r, nil
 }
+
+func (address *Address) List6() ([]net.IPNet, error) {
+	link, ok := address.link()
+	if !ok {
+		return nil, address.Err()
+	}
+	oldAddrs, err := netlink.AddrList(link, netlink.FAMILY_V6)
+	if err != nil {
+		return nil, err
+	}
+	r := make([]net.IPNet, len(oldAddrs))
+	for i, oldAddr := range oldAddrs {
+		r[i] = *oldAddr.IPNet
+	}
+	return r, nil
+}

pkg/util/iproute2/route_linux.go

@@ -71,6 +71,29 @@ func (route *Route) List4() ([]RouteSpec, error) {
 	return rs, nil
 }
 
+func (route *Route) List6() ([]RouteSpec, error) {
+	link, ok := route.link()
+	if !ok {
+		return nil, route.Err()
+	}
+
+	rs, err := netlink.RouteList(link, netlink.FAMILY_V6)
+	if err != nil {
+		route.addErr(err, "route list")
+		return nil, route.Err()
+	}
+	for i := range rs {
+		if rs[i].Dst == nil {
+			// make the return value easier to work with
+			rs[i].Dst = &net.IPNet{
+				IP:   net.IPv6zero,
+				Mask: net.IPMask(net.IPv6zero),
+			}
+		}
+	}
+	return rs, nil
+}
+
 func (route *Route) AddByIPNet(ipnet *net.IPNet, gw net.IP) *Route {
 	r := RouteSpec{
 		Dst: ipnet,

pkg/util/netutils2/bonding.go

@@ -16,7 +16,7 @@ package netutils2
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strconv"
 	"strings"
 
@@ -44,7 +44,7 @@ func getBondingConfig(ifname string) *SBondingConfig {
 	}
 	conf := SBondingConfig{}
 	{
-		content, err := ioutil.ReadFile(modePath)
+		content, err := os.ReadFile(modePath)
 		if err != nil {
 			log.Errorf("fail to read %s", modePath)
 			return nil
@@ -56,7 +56,7 @@ func getBondingConfig(ifname string) *SBondingConfig {
 		conf.ModeName = modeName
 	}
 	{
-		content, err := ioutil.ReadFile(slavesPath)
+		content, err := os.ReadFile(slavesPath)
 		if err != nil {
 			log.Errorf("fail to read %s", modePath)
 			return nil

pkg/util/netutils2/netutils_linux.go

@@ -17,17 +17,29 @@ package netutils2
 import (
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/vishvananda/netlink"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/util/iproute2"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func (n *SNetInterface) GetAddresses() [][]string {
-	ipnets, err := iproute2.NewAddress(n.name).List4()
+	addrList := iproute2.NewAddress(n.name)
+	addrs4 := n.getAddresses(addrList.List4)
+	addrs6 := n.getAddresses(addrList.List6)
+	if len(addrs6) > 0 {
+		addrs4 = append(addrs4, addrs6...)
+	}
+	return addrs4
+}
+
+func (n *SNetInterface) getAddresses(listFunc func() ([]net.IPNet, error)) [][]string {
+	ipnets, err := listFunc()
 	if err != nil {
 		log.Errorf("list address %s: %v", n.name, err)
 		return nil
@@ -45,7 +57,26 @@ func (n *SNetInterface) GetAddresses() [][]string {
 }
 
 func (n *SNetInterface) GetRouteSpecs() []iproute2.RouteSpec {
-	routespecs, err := iproute2.NewRoute(n.name).List4()
+	routeList := iproute2.NewRoute(n.name)
+	routes4 := getRouteSpecs(routeList.List4)
+	routes6 := getRouteSpecs(routeList.List6)
+	if len(routes6) > 0 {
+		routes4 = append(routes4, routes6...)
+	}
+	return routes4
+}
+
+func (n *SNetInterface) ClearAddrs() error {
+	cmd := procutils.NewCommand("ip", "addr", "flush", "dev", n.name)
+	msg, err := cmd.Output()
+	if err != nil {
+		return errors.Wrap(err, strings.TrimSpace(string(msg)))
+	}
+	return nil
+}
+
+func getRouteSpecs(listFunc func() ([]iproute2.RouteSpec, error)) []iproute2.RouteSpec {
+	routespecs, err := listFunc()
 	if err != nil {
 		return nil
 	}

pkg/util/netutils2/netutils_others.go

@@ -33,6 +33,10 @@ func (n *SNetInterface) GetRouteSpecs() []iproute2.RouteSpec {
 	return []iproute2.RouteSpec{}
 }
 
+func (n *SNetInterface) ClearAddrs() error {
+	return nil
+}
+
 func DefaultSrcIpDev() (srcIp net.IP, ifname string, err error) {
 	err = errors.ErrNotImplemented
 	return

pkg/util/netutils2/netutils_test.go

@@ -81,6 +81,21 @@ func TestFormatMac(t *testing.T) {
 func TestNewNetInterface(t *testing.T) {
 	n := NewNetInterface("eth0")
 	t.Logf("NetInterface: %s %s %s %s", n.name, n.Addr, n.Mask.String(), n.mac)
+	addrs := n.GetAddresses()
+	t.Logf("addrs: %s", addrs)
+	slaves := n.GetSlaveAddresses()
+	t.Logf("slaves: %s", slaves)
+	routes := n.GetRouteSpecs()
+	t.Logf("routes: %s", routes)
+	for i := range routes {
+		t.Logf("route to %s", routes[i].Dst.String())
+	}
+
+	m := NewNetInterface("docker0")
+	err := m.ClearAddrs()
+	if err != nil {
+		t.Errorf("ClearAddrs fail %s", err)
+	}
 }
 
 func TestMyDefault(t *testing.T) {