Vlan support (#2084)

* Add support for vlan_filtering on zos * do go mod tidy * Implement probing with vlan Prob on configurable vlan, later on we need to also configure the privae bridge (zos) to use that configured vlan * support full vlan:priv option * also tag nic for vlan:pub * Implementn vlan:pub support - pub vlan is supported in both single and dual nic setup - delete some dead code * update docs Also disable support for dual nic for now * fix typos the configureZOS and Requires struct * Apply PR comments --------- Co-authored-by: xmonader <[email protected]>
threefoldtech · Oct 19, 2023 · 066371c · 066371c
1 parent 5745b54
commit 066371c
Show file tree

Hide file tree

Showing 18 changed files with 316 additions and 241 deletions.
diff --git a/cmds/internet/main.go b/cmds/internet/main.go
@@ -12,6 +12,7 @@ import (
 	"github.com/vishvananda/netlink"
 
 	"github.com/threefoldtech/zos/pkg/app"
+	"github.com/threefoldtech/zos/pkg/environment"
 	"github.com/threefoldtech/zos/pkg/network/bootstrap"
 	"github.com/threefoldtech/zos/pkg/network/bridge"
 	"github.com/threefoldtech/zos/pkg/network/dhcp"
@@ -112,12 +113,29 @@ func check() error {
 	return backoff.RetryNotify(f, backoff.NewExponentialBackOff(), errHandler)
 }
 
+/*
+*
+configureZOS bootstraps the private zos network (private subnet) it goes as follows:
+  - Find a physical interface that can get an IPv4 over DHCP
+  - Once interface is found, a bridge called `zos` is created, then the interface that was
+    found in previous step is attached to the zos bridge.
+  - Bridge and interface are brought UP then a DHCP daemon is started on the zos to get an IP.
+
+In case there is a priv vlan is configured (kernel param vlan:priv=<id>) it is basically the same as
+before but with the next twist:
+- During probing of the interface, probing done on that vlan
+- ZOS is added to vlan as `bridge vlan add vid <id> dev zos pvid self untagged`
+- link is added to vlan as `bridge vlan add vid <id> dev <link>`
+*/
 func configureZOS() error {
+
+	env := environment.MustGet()
+
 	f := func() error {
 		log.Info().Msg("Start network bootstrap")
 
 		ifaceConfigs, err := bootstrap.AnalyzeLinks(
-			bootstrap.RequiresIPv4,
+			bootstrap.RequiresIPv4.WithVlan(env.PrivVlan),
 			bootstrap.PhysicalFilter,
 			bootstrap.PluggedFilter)
 		if err != nil {
@@ -134,7 +152,7 @@ func configureZOS() error {
 		}
 
 		log.Info().Str("interface", zosChild).Msg("selecting interface")
-		br, err := bootstrap.CreateDefaultBridge(types.DefaultBridge)
+		br, err := bootstrap.CreateDefaultBridge(types.DefaultBridge, env.PrivVlan)
 		if err != nil {
 			return err
 		}
@@ -167,6 +185,29 @@ func configureZOS() error {
 			return errors.Wrapf(err, "could not bring %s up", zosChild)
 		}
 
+		if env.PrivVlan != nil && env.PubVlan != nil {
+			// if both priv and pub vlan are configured it means
+			// that we can remove the default tagging of vlan 1
+			// remove default
+			if err := netlink.BridgeVlanDel(link, 1, true, true, false, false); err != nil {
+				return errors.Wrapf(err, "failed to delete default vlan on device '%s'", link.Attrs().Name)
+			}
+		}
+
+		if env.PrivVlan != nil {
+			// add new vlan
+			if err := netlink.BridgeVlanAdd(link, *env.PrivVlan, false, false, false, false); err != nil {
+				return errors.Wrapf(err, "failed to set vlan on device '%s'", link.Attrs().Name)
+			}
+		}
+
+		if env.PubVlan != nil {
+			// add new vlan
+			if err := netlink.BridgeVlanAdd(link, *env.PubVlan, false, false, false, false); err != nil {
+				return errors.Wrapf(err, "failed to set vlan on device '%s'", link.Attrs().Name)
+			}
+		}
+
 		dhcpService := dhcp.NewService(types.DefaultBridge, "", zinit.Default())
 		if err := dhcpService.DestroyOlderService(); err != nil {
 			log.Error().Err(err).Msgf("failed to destory older %s service", dhcpService.Name)

diff --git a/cmds/modules/networkd/main.go b/cmds/modules/networkd/main.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
+	"github.com/threefoldtech/zos/pkg/environment"
 	"github.com/threefoldtech/zos/pkg/network/dhcp"
 	"github.com/threefoldtech/zos/pkg/network/public"
 	"github.com/threefoldtech/zos/pkg/network/types"
@@ -98,8 +99,9 @@ func action(cli *cli.Context) error {
 	if err != nil && err != public.ErrNoPublicConfig {
 		return errors.Wrap(err, "failed to get node public_config")
 	}
+
 	// EnsurePublicSetup knows how to handle a nil pub (in case of ErrNoPublicConfig)
-	master, err := public.EnsurePublicSetup(nodeID, pub)
+	master, err := public.EnsurePublicSetup(nodeID, environment.MustGet().PubVlan, pub)
 	if err != nil {
 		return errors.Wrap(err, "failed to setup public bridge")
 	}

diff --git a/docs/internals/boot.md b/docs/internals/boot.md
@@ -15,3 +15,7 @@ both `node-ready` and `boot` are not actual services, but instead they are there
 - `zos-debug`: means zos is running in debug mode
 - `zos-debug-vm`: forces zos to think it's running on a virtual machine. used mainly for development
 - `disable-gpu`: if provided GPU feature will be disabled on that node
+- `vlan:pub`: set the vlan tag of the node private subnet.
+- `vlan:priv`: sets the vlan tag of the node public subnet.
+
+For more details of `VLAN` support in zos please read more [here](network/vlans.md)
diff --git a/docs/internals/network/readme.md b/docs/internals/network/readme.md
@@ -5,4 +5,5 @@
 - [definitions of the vocabulary used in the documentation](definitions.md)
 - [Introduction to networkd, the network manager of 0-OS](introduction.md)
 - [Detail about the wireguard mesh used to interconnect 0-OS nodes](mesh.md)
-- [Documentation for farmer on how to setup the network of their farm](setup_farm_network.md)
+- [Documentation for farmer on how to setup the network of their farm](setup_farm_network.md)
+- [VLANS](vlans.md)
diff --git a/docs/internals/network/vlans.md b/docs/internals/network/vlans.md
@@ -0,0 +1,81 @@
+# VLANS
+
+ZOS support vlans by allowing the farmer to setup vlan for both private and public subnets.
+
+By default zos uses untagged traffic for both priv and public subnets (for both single or dual nic nodes). In some data centers and cloud providers, they can only provide tagged subnets.
+
+ZOS can then become VLAN aware by providing optional vlan tags during booting.
+
+## Private VLAN
+
+Setting up private vlan forces zos to tag all private traffic with the configured vlan tag. This is possible by providing the `vlan:priv` kernel command line parameter
+
+> Example `vlan:priv=302` will tag all private traffic with VLAN id `302`
+
+During boot, zos tries to find the first interface that has ipv4 (over dhcp) normally all interfaces are probed until one of them actually get an IP. If a vlan ID is set, the probing also happen on the proper vlan, then the private default bridge (called `zos`) is then setup correctly with the proper vlan
+
+```
+                ┌────────────────────────────────────┐
+                │              NODE                  │
+                │                                    │
+  vlan 302 ┌────┴──┐                                 │
+───────────┤  Nic  ├──────────┐                      │
+   tagged  └────┬──┘          │                      │
+                │        ┌────┴─────┐                │
+                │        │          │                │
+                │        │   zos    │  pvid 302      │
+                │        │   bridge ├──untagged      │
+                │        │          │                │
+                │        │          │                │
+                │        └──────────┘                │
+                │                                    │
+                │                                    │
+                │                                    │
+                └────────────────────────────────────┘
+```
+
+## Public VLAN
+
+> NOTE: Public VLAN in ZOS is **only** supported in a single nic setup. There is no support in dual nic yet
+
+Setting up private vlan forces zos to tag all private traffic with the configured vlan tag. This is possible by providing the `vlan:pub` kernel command line parameter
+
+> Example `vlan:pub=304` will tag all private traffic with VLAN id `304`
+
+zos internally create a public bridge `br-pub` that can uses a detected ingress link (usually in dual nic setup) or shares
+the same link as `zos` bridge by connecting to `br-pub` via a veth pair.
+
+Single NIC setup
+
+```
+                ┌─────────────────────────────────────────────┐
+                │                                             │
+304 tagged ┌────┴─────┐                                       │
+───────────┤   NIC    ├────────────┐                          │
+           └────┬─────┘            │                          │
+                │                  │                          │
+                │          ┌───────┴─────┐                    │
+                │          │             │                    │
+                │          │      zos    │                    │
+                │          │      bridge │                    │
+                │          │             │                    │
+                │          │             │                    │
+                │          └───────┬─────┘                    │
+                │                  │  pvid 304 untagged       │
+                │                  │                          │
+                │                  │                          │
+                │           ┌──────▼─────┐                    │
+                │           │            │                    │
+                │           │    br-pub  │                    │
+                │           │    bridge  │                    │
+                │           │            │                    │
+                │           │            │                    │
+                │           │            │                    │
+                │           └────────────┘                    │
+                │                                             │
+                └─────────────────────────────────────────────┘
+```
+
+## Dual NIC setup
+
+Right now public vlans are not supported in case of dual nic setups. So in case public network is only available on the second nic then it will always be untagged traffic. This means the `vlan:pub` flag is silently ignored
diff --git a/go.mod b/go.mod
@@ -77,6 +77,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/ethereum/go-ethereum v1.11.6 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -85,6 +86,7 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect

diff --git a/go.sum b/go.sum
@@ -208,13 +208,17 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go
@@ -5,6 +5,8 @@ import (
 	"strconv"
 	"sync"
 
+	"slices"
+
 	"github.com/pkg/errors"
 	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
 	"github.com/threefoldtech/zos/pkg"
@@ -37,6 +39,15 @@ type Environment struct {
 	GraphQL       string
 
 	ExtendedConfigURL string
+
+	// private vlan to join
+	// if set, zos will use this as its priv vlan
+	PrivVlan *uint16
+
+	// pub vlan to join
+	// if set, zos will use this as it's pub vlan
+	// only in a single nic setup
+	PubVlan *uint16
 }
 
 // RunningMode type
@@ -250,6 +261,28 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
 		env.FarmID = pkg.FarmID(id)
 	}
 
+	if vlan, found := params.GetOne("vlan:priv"); found {
+		if !slices.Contains([]string{"none", "untagged", "un"}, vlan) {
+			tag, err := strconv.ParseUint(vlan, 10, 16)
+			if err != nil {
+				return env, errors.Wrap(err, "failed to parse priv vlan value")
+			}
+			tagU16 := uint16(tag)
+			env.PrivVlan = &tagU16
+		}
+	}
+
+	if vlan, found := params.GetOne("vlan:pub"); found {
+		if !slices.Contains([]string{"none", "untagged", "un"}, vlan) {
+			tag, err := strconv.ParseUint(vlan, 10, 16)
+			if err != nil {
+				return env, errors.Wrap(err, "failed to parse pub vlan value")
+			}
+			tagU16 := uint16(tag)
+			env.PubVlan = &tagU16
+		}
+	}
+
 	// Checking if there environment variable
 	// override default settings
 

diff --git a/pkg/kernel/kernel.go b/pkg/kernel/kernel.go
@@ -38,6 +38,22 @@ func (k Params) Get(key string) ([]string, bool) {
 	return v, ok
 }
 
+// GetOne gets a single value for given key. If key is provided
+// multiple times in the cmdline, the last one is used. If key does
+// not exist, or has no associated value, a false is returned
+func (k Params) GetOne(key string) (string, bool) {
+	all, found := k.Get(key)
+	if !found {
+		return "", false
+	}
+
+	if len(all) == 0 {
+		return "", false
+	}
+
+	return all[len(all)-1], true
+}
+
 // IsDebug checks if zos-debug is set
 func (k Params) IsDebug() bool {
 	return k.Exists(Debug)

diff --git a/pkg/network.go b/pkg/network.go
@@ -14,14 +14,6 @@ import (
 //go:generate mkdir -p stubs
 //go:generate zbusc -module network -version 0.0.1 -name network -package stubs github.com/threefoldtech/zos/pkg+Networker stubs/network_stub.go
 
-// Member holds information about a the network namespace of a container
-type Member struct {
-	Namespace   string
-	IPv6        net.IP
-	IPv4        net.IP
-	YggdrasilIP net.IP
-}
-
 // ContainerNetworkConfig defines how to construct the network namespace of a container
 type ContainerNetworkConfig struct {
 	IPs         []string