From 5948c2fdfa46089cbc2eb0fcdb25a96d6301021a Mon Sep 17 00:00:00 2001 From: Muhamad Azamy Date: Wed, 18 Oct 2023 13:40:09 +0200 Subject: [PATCH] Implementn vlan:pub support - pub vlan is supported in both single and dual nic setup - delete some dead code --- cmds/modules/networkd/main.go | 4 +- pkg/network.go | 8 -- pkg/network/bridge/bridge.go | 28 ++++- pkg/network/bridge/bridge_test.go | 2 +- pkg/network/networker.go | 5 +- pkg/network/nr/container.go | 195 ------------------------------ pkg/network/public/public.go | 15 +-- 7 files changed, 37 insertions(+), 220 deletions(-) delete mode 100644 pkg/network/nr/container.go diff --git a/cmds/modules/networkd/main.go b/cmds/modules/networkd/main.go index 4d60e9630..d91c1976e 100644 --- a/cmds/modules/networkd/main.go +++ b/cmds/modules/networkd/main.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go" + "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/network/dhcp" "github.com/threefoldtech/zos/pkg/network/public" "github.com/threefoldtech/zos/pkg/network/types" @@ -98,8 +99,9 @@ func action(cli *cli.Context) error { if err != nil && err != public.ErrNoPublicConfig { return errors.Wrap(err, "failed to get node public_config") } + // EnsurePublicSetup knows how to handle a nil pub (in case of ErrNoPublicConfig) - master, err := public.EnsurePublicSetup(nodeID, pub) + master, err := public.EnsurePublicSetup(nodeID, environment.MustGet().PubVlan, pub) if err != nil { return errors.Wrap(err, "failed to setup public bridge") } diff --git a/pkg/network.go b/pkg/network.go index 75fa95058..74f3f4e70 100644 --- a/pkg/network.go +++ b/pkg/network.go @@ -14,14 +14,6 @@ import ( //go:generate mkdir -p stubs //go:generate zbusc -module network -version 0.0.1 -name network -package stubs github.com/threefoldtech/zos/pkg+Networker stubs/network_stub.go -// Member holds information about a the network namespace of a container -type Member struct { - Namespace string - IPv6 net.IP - IPv4 net.IP - YggdrasilIP net.IP -} - // ContainerNetworkConfig defines how to construct the network namespace of a container type ContainerNetworkConfig struct { IPs []string diff --git a/pkg/network/bridge/bridge.go b/pkg/network/bridge/bridge.go index 3415c8d80..e59adb98c 100644 --- a/pkg/network/bridge/bridge.go +++ b/pkg/network/bridge/bridge.go @@ -101,9 +101,9 @@ func vethName(from, to string) string { // can be directly plugged or crossed over with a veth pair // if name is provided, the name will be used in case of veth pair instead of // a generated name -func Attach(link netlink.Link, bridge *netlink.Bridge, name ...string) error { +func Attach(link netlink.Link, bridge *netlink.Bridge, vlan *uint16, name ...string) error { if link.Type() == "device" { - return AttachNic(link, bridge) + return attachNic(link, bridge, vlan) } else if link.Type() == "bridge" { linkBr := link.(*netlink.Bridge) n := vethName(link.Attrs().Name, bridge.Name) @@ -116,14 +116,14 @@ func Attach(link netlink.Link, bridge *netlink.Bridge, name ...string) error { return err } - return AttachNic(veth, linkBr) + return attachNic(veth, linkBr, vlan) } return fmt.Errorf("unsupported link type '%s'", link.Type()) } -// AttachNic attaches an interface to a bridge -func AttachNic(link netlink.Link, bridge *netlink.Bridge) error { +// attachNic attaches an interface to a bridge +func attachNic(link netlink.Link, bridge *netlink.Bridge, vlan *uint16) error { // Jan said this was fine if err := netlink.LinkSetUp(link); err != nil { return errors.Wrap(err, "could not set veth peer up") @@ -133,7 +133,23 @@ func AttachNic(link netlink.Link, bridge *netlink.Bridge) error { if err := options.Set(link.Attrs().Name, options.IPv6Disable(true)); err != nil { return errors.Wrap(err, "failed to disable ipv6 on link interface") } - return netlink.LinkSetMaster(link, bridge) + if err := netlink.LinkSetMaster(link, bridge); err != nil { + return errors.Wrapf(err, "failed to attach link %s to bridge %s", link.Attrs().Name, bridge.Name) + } + + if vlan == nil { + return nil + } + + if err := netlink.BridgeVlanDel(link, 1, true, true, false, false); err != nil { + return errors.Wrapf(err, "failed to delete default vlan tag on device '%s'", link.Attrs().Name) + } + + if err := netlink.BridgeVlanAdd(link, *vlan, true, true, false, false); err != nil { + return errors.Wrapf(err, "failed to set vlan on device '%s'", link.Attrs().Name) + } + + return nil } // List all nics attached to a bridge diff --git a/pkg/network/bridge/bridge_test.go b/pkg/network/bridge/bridge_test.go index 70515f84e..d45fa3eb7 100644 --- a/pkg/network/bridge/bridge_test.go +++ b/pkg/network/bridge/bridge_test.go @@ -67,7 +67,7 @@ func TestAttachBridge(t *testing.T) { _ = netlink.LinkDel(dummy) }() - err = AttachNic(dummy, br) + err = attachNic(dummy, br, nil) assert.NoError(t, err) } diff --git a/pkg/network/networker.go b/pkg/network/networker.go index 71f3a9b50..1963885f5 100644 --- a/pkg/network/networker.go +++ b/pkg/network/networker.go @@ -17,6 +17,7 @@ import ( "github.com/blang/semver" "github.com/threefoldtech/zos/pkg/cache" + "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/gridtypes/zos" "github.com/threefoldtech/zos/pkg/network/bootstrap" @@ -932,7 +933,7 @@ func (n *networker) Namespace(id zos.NetID) string { func (n *networker) UnsetPublicConfig() error { id := n.identity.NodeID(context.Background()) - _, err := public.EnsurePublicSetup(id, nil) + _, err := public.EnsurePublicSetup(id, environment.MustGet().PubVlan, nil) return err } @@ -953,7 +954,7 @@ func (n *networker) SetPublicConfig(cfg pkg.PublicConfig) error { } id := n.identity.NodeID(context.Background()) - _, err = public.EnsurePublicSetup(id, &cfg) + _, err = public.EnsurePublicSetup(id, environment.MustGet().PubVlan, &cfg) if err != nil { return errors.Wrap(err, "failed to apply public config") } diff --git a/pkg/network/nr/container.go b/pkg/network/nr/container.go deleted file mode 100644 index 93735b5f0..000000000 --- a/pkg/network/nr/container.go +++ /dev/null @@ -1,195 +0,0 @@ -package nr - -import ( - "fmt" - "net" - "os" - - "github.com/containernetworking/plugins/pkg/ip" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/threefoldtech/zos/pkg" - "github.com/threefoldtech/zos/pkg/network/bridge" - "github.com/threefoldtech/zos/pkg/network/ifaceutil" - "github.com/threefoldtech/zos/pkg/network/namespace" - "github.com/threefoldtech/zos/pkg/network/options" - "github.com/vishvananda/netlink" -) - -// ContainerConfig is an object used to pass the required network configuration -// for a container -type ContainerConfig struct { - ContainerID string - IPs []net.IP - PublicIP6 bool //true if the container must have a public ipv6 - IPv4Only bool // describe the state of the node, true mean it runs in ipv4 only mode -} - -// Join make a network namespace of a container join a network resource network -func (nr *NetResource) Join(cfg ContainerConfig) (join pkg.Member, err error) { - name, err := nr.BridgeName() - if err != nil { - return join, err - } - - br, err := bridge.Get(name) - if err != nil { - return join, err - } - - join.Namespace = cfg.ContainerID - netspace, err := namespace.Create(cfg.ContainerID) - if err != nil { - return join, err - } - - slog := log.With(). - Str("namespace", cfg.ContainerID). - Str("container", cfg.ContainerID). - Logger() - - defer func() { - if err != nil { - _ = namespace.Delete(netspace) - } - }() - - var hostVethName string - err = netspace.Do(func(host ns.NetNS) error { - if err := ifaceutil.SetLoUp(); err != nil { - return err - } - - slog.Info(). - Str("veth", "eth0"). - Msg("Create veth pair in net namespace") - hostVeth, containerVeth, err := ip.SetupVeth("eth0", 1500, host) - if err != nil { - return errors.Wrapf(err, "failed to create veth pair in namespace (%s)", join.Namespace) - } - - hostVethName = hostVeth.Name - - eth0, err := netlink.LinkByName(containerVeth.Name) - if err != nil { - return err - } - - for _, addr := range cfg.IPs { - slog.Info(). - Str("ip", addr.String()). - Msgf("set ip to container") - - if err := netlink.AddrAdd(eth0, &netlink.Addr{IPNet: &net.IPNet{ - IP: addr, - Mask: net.CIDRMask(24, 32), - }}); err != nil && !os.IsExist(err) { - return err - } - join.IPv4 = addr - } - - // if the node is IPv6 enabled and the user do not requires a public IPv6 - // then we create derive one to allow IPv6 traffic to go out - // if the user ask for a public IPv6, then the all config comes from SLAAC so we don't have to do anything ourself - if !cfg.IPv4Only && !cfg.PublicIP6 { - ipv6 := Convert4to6(nr.ID(), cfg.IPs[0]) - slog.Info(). - Str("ip", ipv6.String()). - Msgf("set ip to container") - - if err := netlink.AddrAdd(eth0, &netlink.Addr{IPNet: &net.IPNet{ - IP: ipv6, - Mask: net.CIDRMask(64, 128), - }}); err != nil && !os.IsExist(err) { - return err - } - join.IPv6 = ipv6 - } - - ipnet := nr.resource.Subnet - //sanity check this should be already handle by validate. - //but in case something went wrong. - if len(ipnet.IP) == 0 { - return fmt.Errorf("invalid network resource (%s): empty subnet", nr.id) - } - - ipnet.IP[len(ipnet.IP)-1] = 0x01 - - routes := []*netlink.Route{ - { - Dst: &net.IPNet{ - IP: net.ParseIP("0.0.0.0"), - Mask: net.CIDRMask(0, 32), - }, - Gw: ipnet.IP, - LinkIndex: eth0.Attrs().Index, - }, - } - - // same logic as before, we set ipv6 routes only if this is required - if !cfg.IPv4Only && !cfg.PublicIP6 { - routes = append(routes, - &netlink.Route{ - Dst: &net.IPNet{ - IP: net.ParseIP("::"), - Mask: net.CIDRMask(0, 128), - }, - Gw: net.ParseIP("fe80::1"), - LinkIndex: eth0.Attrs().Index, - }) - } - - for _, r := range routes { - slog.Info(). - Str("route", r.String()). - Msgf("set route to container") - err = netlink.RouteAdd(r) - if err != nil && !os.IsExist(err) { - return errors.Wrapf(err, "failed to set route %s on eth0", r.String()) - } - } - - return nil - }) - - if err != nil { - return join, err - } - - hostVeth, err := netlink.LinkByName(hostVethName) - if err != nil { - return join, err - } - - if err := options.Set(hostVeth.Attrs().Name, options.IPv6Disable(true)); err != nil { - return join, errors.Wrapf(err, "failed to disable ip6 on bridge %s", hostVeth.Attrs().Name) - } - - return join, bridge.AttachNic(hostVeth, br) -} - -// Leave delete a container network namespace -func (nr *NetResource) Leave(containerID string) error { - log.Info(). - Str("namespace", containerID). - Str("container", containerID). - Msg("delete container network namespace") - - namespc, err := namespace.GetByName(containerID) - if _, ok := err.(ns.NSPathNotExistErr); ok { - return nil - } else if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - defer namespc.Close() - - err = namespace.Delete(namespc) - if err != nil { - return err - } - return nil -} diff --git a/pkg/network/public/public.go b/pkg/network/public/public.go index 5ffa6f502..fbf3a93a0 100644 --- a/pkg/network/public/public.go +++ b/pkg/network/public/public.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/network/bootstrap" "github.com/threefoldtech/zos/pkg/network/bridge" @@ -95,7 +96,7 @@ func IPs() ([]net.IPNet, error) { return ips, err } -func setupPublicBridge(br *netlink.Bridge) error { +func setupPublicBridge(br *netlink.Bridge, vlan *uint16) error { exit, err := detectExitNic() if err != nil { return errors.Wrap(err, "failed to find possible exit") @@ -107,15 +108,15 @@ func setupPublicBridge(br *netlink.Bridge) error { return errors.Wrapf(err, "failed to get link '%s' by name", exit) } - return attachPublicToExit(br, exitLink) + return attachPublicToExit(br, exitLink, vlan) } -func attachPublicToExit(br *netlink.Bridge, exit netlink.Link) error { +func attachPublicToExit(br *netlink.Bridge, exit netlink.Link, vlan *uint16) error { if err := netlink.LinkSetUp(exit); err != nil { return errors.Wrapf(err, "failed to set link '%s' up", exit.Attrs().Name) } - if err := bridge.Attach(exit, br, toZosVeth); err != nil { + if err := bridge.Attach(exit, br, vlan, toZosVeth); err != nil { return errors.Wrap(err, "failed to attach exit nic to public bridge 'br-pub'") } @@ -212,7 +213,7 @@ func SetPublicExitLink(link netlink.Link) error { } } - return attachPublicToExit(br, link) + return attachPublicToExit(br, link, environment.MustGet().PubVlan) } func HasPublicSetup() bool { @@ -301,7 +302,7 @@ func GetPublicSetup() (pkg.PublicConfig, error) { // // if no nic is found zos is selected. // changes to the br-pub exit nic can then be done later with SetPublicExitLink -func EnsurePublicSetup(nodeID pkg.Identifier, inf *pkg.PublicConfig) (*netlink.Bridge, error) { +func EnsurePublicSetup(nodeID pkg.Identifier, vlan *uint16, inf *pkg.PublicConfig) (*netlink.Bridge, error) { log.Debug().Msg("ensure public setup") br, err := ensurePublicBridge() if err != nil { @@ -312,7 +313,7 @@ func EnsurePublicSetup(nodeID pkg.Identifier, inf *pkg.PublicConfig) (*netlink.B if os.IsNotExist(err) { // bridge is not initialized, wire it. log.Debug().Msg("no public bridge uplink found, setting up...") - if err := setupPublicBridge(br); err != nil { + if err := setupPublicBridge(br, vlan); err != nil { return nil, err } } else if err != nil {