From 81b1b3ac550e88f71f3854af7aaffaa380c0a1dc Mon Sep 17 00:00:00 2001 From: Sachin Tiptur Date: Mon, 12 Dec 2022 15:30:08 +0100 Subject: [PATCH 1/4] IPv6 address support in node addresses Signed-off-by: Sachin Tiptur --- cloud-controller-manager/do/cloud.go | 1 + cloud-controller-manager/do/common.go | 30 +++++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cloud-controller-manager/do/cloud.go b/cloud-controller-manager/do/cloud.go index e7618e45c..6e81e9356 100644 --- a/cloud-controller-manager/do/cloud.go +++ b/cloud-controller-manager/do/cloud.go @@ -57,6 +57,7 @@ const ( publicAccessFirewallTagsEnv string = "PUBLIC_ACCESS_FIREWALL_TAGS" regionEnv string = "REGION" doAPIRateLimitQPSEnv string = "DO_API_RATE_LIMIT_QPS" + doIPAddrFamiliesEnv string = "DO_IP_ADDR_FAMILIES" ) var version string diff --git a/cloud-controller-manager/do/common.go b/cloud-controller-manager/do/common.go index 1e0538c60..d20b60d52 100644 --- a/cloud-controller-manager/do/common.go +++ b/cloud-controller-manager/do/common.go @@ -20,6 +20,8 @@ import ( "context" "errors" "fmt" + "os" + "strings" "github.com/digitalocean/godo" v1 "k8s.io/api/core/v1" @@ -128,18 +130,28 @@ func allLoadBalancerList(ctx context.Context, client *godo.Client) ([]godo.LoadB func nodeAddresses(droplet *godo.Droplet) ([]v1.NodeAddress, error) { var addresses []v1.NodeAddress addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: droplet.Name}) + ipFamilies := os.Getenv(doIPAddrFamiliesEnv) - privateIP, err := droplet.PrivateIPv4() - if err != nil || privateIP == "" { - return nil, fmt.Errorf("could not get private ip: %v", err) - } - addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: privateIP}) + if strings.Contains(ipFamilies, "ipv4") || ipFamilies == "" { + privateIP, err := droplet.PrivateIPv4() + if err != nil || privateIP == "" { + return nil, fmt.Errorf("could not get private ip: %v", err) + } + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: privateIP}) - publicIP, err := droplet.PublicIPv4() - if err != nil || publicIP == "" { - return nil, fmt.Errorf("could not get public ip: %v", err) + publicIP, err := droplet.PublicIPv4() + if err != nil || publicIP == "" { + return nil, fmt.Errorf("could not get public ip: %v", err) + } + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIP}) + } + if strings.Contains(ipFamilies, "ipv6") { + publicIPv6, err := droplet.PublicIPv6() + if err != nil || publicIPv6 == "" { + return nil, fmt.Errorf("could not get public ipv6: %v", err) + } + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv6}) } - addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIP}) return addresses, nil } From 6b4e8db4f33a7ddc61ed57f143c50f38385ee057 Mon Sep 17 00:00:00 2001 From: Sachin Tiptur Date: Tue, 13 Dec 2022 11:18:47 +0100 Subject: [PATCH 2/4] fix order of IPs Signed-off-by: Sachin Tiptur --- cloud-controller-manager/do/cloud.go | 19 +++++++++++++++++++ cloud-controller-manager/do/common.go | 21 +++++++++++++++++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cloud-controller-manager/do/cloud.go b/cloud-controller-manager/do/cloud.go index 6e81e9356..54720d2a1 100644 --- a/cloud-controller-manager/do/cloud.go +++ b/cloud-controller-manager/do/cloud.go @@ -62,6 +62,9 @@ const ( var version string +var ipFamilyOpts = []string{"", "ipv4", "ipv6", "ipv4,ipv6", "ipv6,ipv4"} +var ipFamilies string + type tokenSource struct { AccessToken string } @@ -159,6 +162,13 @@ func newCloud() (cloudprovider.Interface, error) { addr = fmt.Sprintf("%s:%s", addrHost, addrPort) } + // var ipFamilies string + ipf, set := os.LookupEnv(doIPAddrFamiliesEnv) + ipFamilies = ipf + if set && !containsString(ipFamilyOpts, ipFamilies) { + return nil, fmt.Errorf("invalid value set for environment variable %q", doIPAddrFamiliesEnv) + } + return &cloud{ client: doClient, instances: newInstances(resources, region), @@ -281,3 +291,12 @@ func (c *cloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []strin func (c *cloud) HasClusterID() bool { return false } + +func containsString(vals []string, val string) bool { + for _, v := range vals { + if v == val { + return true + } + } + return false +} diff --git a/cloud-controller-manager/do/common.go b/cloud-controller-manager/do/common.go index d20b60d52..076570196 100644 --- a/cloud-controller-manager/do/common.go +++ b/cloud-controller-manager/do/common.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "os" "strings" "github.com/digitalocean/godo" @@ -130,9 +129,21 @@ func allLoadBalancerList(ctx context.Context, client *godo.Client) ([]godo.LoadB func nodeAddresses(droplet *godo.Droplet) ([]v1.NodeAddress, error) { var addresses []v1.NodeAddress addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: droplet.Name}) - ipFamilies := os.Getenv(doIPAddrFamiliesEnv) - if strings.Contains(ipFamilies, "ipv4") || ipFamilies == "" { + for _, i := range strings.Split(ipFamilies, ",") { + addr, err := discoverAddress(droplet, i) + if err != nil { + return nil, fmt.Errorf("could not get addresses for %s : %v", i, err) + } + addresses = append(addresses, addr...) + } + + return addresses, nil +} + +func discoverAddress(droplet *godo.Droplet, family string) ([]v1.NodeAddress, error) { + var addresses []v1.NodeAddress + if family == "ipv4" || family == "" { privateIP, err := droplet.PrivateIPv4() if err != nil || privateIP == "" { return nil, fmt.Errorf("could not get private ip: %v", err) @@ -144,13 +155,15 @@ func nodeAddresses(droplet *godo.Droplet) ([]v1.NodeAddress, error) { return nil, fmt.Errorf("could not get public ip: %v", err) } addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIP}) + return addresses, nil } - if strings.Contains(ipFamilies, "ipv6") { + if family == "ipv6" { publicIPv6, err := droplet.PublicIPv6() if err != nil || publicIPv6 == "" { return nil, fmt.Errorf("could not get public ipv6: %v", err) } addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv6}) + return addresses, nil } return addresses, nil From d066d362f7381f40d86f79a1ef1d789a021c9345 Mon Sep 17 00:00:00 2001 From: Sachin Tiptur Date: Tue, 13 Dec 2022 11:45:50 +0100 Subject: [PATCH 3/4] update unit test and docs Signed-off-by: Sachin Tiptur --- README.md | 7 ++++++- cloud-controller-manager/do/droplets_test.go | 20 +++++++++++++++++++ docs/controllers/node/examples/README.md | 4 +++- .../cloud-controller-manager.yml | 2 ++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 86471c96b..3eac7076c 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ package-hosted directory like this: ```bash cd cloud-controller-manager/cmd/digitalocean-cloud-controller-manager -REGION=fra1 DO_ACCESS_TOKEN=your_access_token go run main.go \ +REGION=fra1 DO_ACCESS_TOKEN=your_access_token DO_IP_ADDR_FAMILIES=ipv4 go run main.go \ --kubeconfig \ --leader-elect=false --v=5 --cloud-provider=digitalocean ``` @@ -79,6 +79,11 @@ You might also need to provide your DigitalOcean access token in the cloud controller to start, but in that case, you will not be able to validate integration with DigitalOcean API. +The `DO_IP_ADDR_FAMILIES` is used to configure the required IP familes and the +order in which address should be populated in nodes status. The accepted values +are one of the `{"", "ipv4", "ipv6", "ipv4,ipv6", "ipv6,ipv4"}`.IPv4 is the +default, if not set or empty. + Please note that if you use a Kubernetes cluster created on DigitalOcean, there will be a cloud controller manager running in the cluster already, so your local one will compete for API access with it. diff --git a/cloud-controller-manager/do/droplets_test.go b/cloud-controller-manager/do/droplets_test.go index d73a9b4da..d8862c584 100644 --- a/cloud-controller-manager/do/droplets_test.go +++ b/cloud-controller-manager/do/droplets_test.go @@ -118,6 +118,12 @@ func newFakeDroplet() *godo.Droplet { Type: "public", }, }, + V6: []godo.NetworkV6{ + { + IPAddress: "2a01::10", + Type: "public", + }, + }, }, Region: &godo.Region{ Name: "test-region", @@ -143,6 +149,12 @@ func newFakeShutdownDroplet() *godo.Droplet { Type: "public", }, }, + V6: []godo.NetworkV6{ + { + IPAddress: "2a01::10", + Type: "public", + }, + }, }, Region: &godo.Region{ Name: "test-region", @@ -186,6 +198,10 @@ func TestNodeAddresses(t *testing.T) { Type: v1.NodeExternalIP, Address: "99.99.99.99", }, + { + Type: "public", + Address: "2a01::10", + }, } addresses, err := instances.NodeAddresses(context.TODO(), "test-droplet") @@ -222,6 +238,10 @@ func TestNodeAddressesByProviderID(t *testing.T) { Type: v1.NodeExternalIP, Address: "99.99.99.99", }, + { + Type: "public", + Address: "2a01::10", + }, } addresses, err := instances.NodeAddressesByProviderID(context.TODO(), "digitalocean://123") diff --git a/docs/controllers/node/examples/README.md b/docs/controllers/node/examples/README.md index aaf5cc30f..c1034389b 100644 --- a/docs/controllers/node/examples/README.md +++ b/docs/controllers/node/examples/README.md @@ -68,6 +68,8 @@ status: type: InternalIP - address: 138.197.174.81 type: ExternalIP + - address: 2a03:b0c0:3:d0::e68:a001 + type: ExternalIP allocatable: cpu: "4" memory: 6012700Ki @@ -80,7 +82,7 @@ status: DigitalOcean cloud controller manager has made the cluster aware of the size of the node, in this case c-4 (4 core high CPU droplet). It has also assigned the node a failure domain which the scheduler can use for region failovers. Note also that the correct addresses were assigned to the node. The `InternalIP` now represents -the private IP of the droplet, and the `ExternalIP` is it's public IP. +the private IP of the droplet, and the `ExternalIP` is it's public IP. The order and IP families depends on the env variable `DO_IP_ADDR_FAMILIES`. ## Node clean up diff --git a/docs/example-manifests/cloud-controller-manager.yml b/docs/example-manifests/cloud-controller-manager.yml index cb82c126e..d76b456de 100644 --- a/docs/example-manifests/cloud-controller-manager.yml +++ b/docs/example-manifests/cloud-controller-manager.yml @@ -66,6 +66,8 @@ spec: secretKeyRef: name: digitalocean key: access-token + - name: DO_IP_ADDR_FAMILIES + value: ipv4,ipv6 --- apiVersion: v1 From 6b1a54ddec912880330f9e4218e9a9bc21dda2d4 Mon Sep 17 00:00:00 2001 From: Sachin Tiptur Date: Tue, 17 Jan 2023 00:04:11 +0100 Subject: [PATCH 4/4] Fixed review comments Signed-off-by: Sachin Tiptur --- README.md | 10 +++--- cloud-controller-manager/do/cloud.go | 20 +++++------ cloud-controller-manager/do/common.go | 35 +++++++++++++++----- cloud-controller-manager/do/droplets_test.go | 10 ++---- 4 files changed, 43 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 3eac7076c..8dcb68197 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ package-hosted directory like this: ```bash cd cloud-controller-manager/cmd/digitalocean-cloud-controller-manager -REGION=fra1 DO_ACCESS_TOKEN=your_access_token DO_IP_ADDR_FAMILIES=ipv4 go run main.go \ +REGION=fra1 DO_ACCESS_TOKEN=your_access_token go run main.go \ --kubeconfig \ --leader-elect=false --v=5 --cloud-provider=digitalocean ``` @@ -79,10 +79,10 @@ You might also need to provide your DigitalOcean access token in the cloud controller to start, but in that case, you will not be able to validate integration with DigitalOcean API. -The `DO_IP_ADDR_FAMILIES` is used to configure the required IP familes and the -order in which address should be populated in nodes status. The accepted values -are one of the `{"", "ipv4", "ipv6", "ipv4,ipv6", "ipv6,ipv4"}`.IPv4 is the -default, if not set or empty. +The `DO_IP_ADDR_FAMILIES` is used to configure the required IP families and the +order in which the addresses should be populated in nodes status. The accepted values +are one of the `"ipv4", "ipv6"` or a comma-separated list of multiple IP address +families. IPv4 is the default, if not set or empty. Please note that if you use a Kubernetes cluster created on DigitalOcean, there will be a cloud controller manager running in the cluster already, so your local diff --git a/cloud-controller-manager/do/cloud.go b/cloud-controller-manager/do/cloud.go index 54720d2a1..32cfedfe8 100644 --- a/cloud-controller-manager/do/cloud.go +++ b/cloud-controller-manager/do/cloud.go @@ -62,9 +62,6 @@ const ( var version string -var ipFamilyOpts = []string{"", "ipv4", "ipv6", "ipv4,ipv6", "ipv6,ipv4"} -var ipFamilies string - type tokenSource struct { AccessToken string } @@ -162,10 +159,8 @@ func newCloud() (cloudprovider.Interface, error) { addr = fmt.Sprintf("%s:%s", addrHost, addrPort) } - // var ipFamilies string ipf, set := os.LookupEnv(doIPAddrFamiliesEnv) - ipFamilies = ipf - if set && !containsString(ipFamilyOpts, ipFamilies) { + if set && !validateAndSetIPFamilies(ipf) { return nil, fmt.Errorf("invalid value set for environment variable %q", doIPAddrFamiliesEnv) } @@ -292,11 +287,14 @@ func (c *cloud) HasClusterID() bool { return false } -func containsString(vals []string, val string) bool { - for _, v := range vals { - if v == val { - return true +func validateAndSetIPFamilies(ipf string) bool { + for _, v := range strings.Split(ipf, ",") { + ipf := strings.TrimSpace(v) + if ipf == ipv4Family || ipf == ipv6Family { + ipFamilies = append(ipFamilies, IPFamily(ipf)) + } else { + return false } } - return false + return true } diff --git a/cloud-controller-manager/do/common.go b/cloud-controller-manager/do/common.go index 076570196..e4638e133 100644 --- a/cloud-controller-manager/do/common.go +++ b/cloud-controller-manager/do/common.go @@ -20,12 +20,20 @@ import ( "context" "errors" "fmt" - "strings" "github.com/digitalocean/godo" v1 "k8s.io/api/core/v1" ) +type IPFamily string + +var ipFamilies []IPFamily + +const ( + ipv4Family = "ipv4" + ipv6Family = "ipv6" +) + // apiResultsPerPage is the maximum page size that DigitalOcean's api supports. const apiResultsPerPage = 200 @@ -130,20 +138,31 @@ func nodeAddresses(droplet *godo.Droplet) ([]v1.NodeAddress, error) { var addresses []v1.NodeAddress addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: droplet.Name}) - for _, i := range strings.Split(ipFamilies, ",") { - addr, err := discoverAddress(droplet, i) + // default case when DO_IP_ADDR_FAMILIES is not set + if ipFamilies == nil { + addr, err := discoverAddress(droplet, ipv4Family) if err != nil { - return nil, fmt.Errorf("could not get addresses for %s : %v", i, err) + return nil, fmt.Errorf("could not get addresses for %s : %v", ipv4Family, err) } addresses = append(addresses, addr...) + } else { + for _, i := range ipFamilies { + addr, err := discoverAddress(droplet, i) + if err != nil { + return nil, fmt.Errorf("could not get addresses for %s : %v", i, err) + } + addresses = append(addresses, addr...) + } } return addresses, nil } -func discoverAddress(droplet *godo.Droplet, family string) ([]v1.NodeAddress, error) { +func discoverAddress(droplet *godo.Droplet, family IPFamily) ([]v1.NodeAddress, error) { var addresses []v1.NodeAddress - if family == "ipv4" || family == "" { + + switch family { + case ipv4Family: privateIP, err := droplet.PrivateIPv4() if err != nil || privateIP == "" { return nil, fmt.Errorf("could not get private ip: %v", err) @@ -156,8 +175,7 @@ func discoverAddress(droplet *godo.Droplet, family string) ([]v1.NodeAddress, er } addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIP}) return addresses, nil - } - if family == "ipv6" { + case ipv6Family: publicIPv6, err := droplet.PublicIPv6() if err != nil || publicIPv6 == "" { return nil, fmt.Errorf("could not get public ipv6: %v", err) @@ -165,6 +183,5 @@ func discoverAddress(droplet *godo.Droplet, family string) ([]v1.NodeAddress, er addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv6}) return addresses, nil } - return addresses, nil } diff --git a/cloud-controller-manager/do/droplets_test.go b/cloud-controller-manager/do/droplets_test.go index d8862c584..5c213fa72 100644 --- a/cloud-controller-manager/do/droplets_test.go +++ b/cloud-controller-manager/do/droplets_test.go @@ -199,11 +199,11 @@ func TestNodeAddresses(t *testing.T) { Address: "99.99.99.99", }, { - Type: "public", + Type: v1.NodeExternalIP, Address: "2a01::10", }, } - + ipFamilies = []IPFamily{ipv4Family, ipv6Family} addresses, err := instances.NodeAddresses(context.TODO(), "test-droplet") if !reflect.DeepEqual(addresses, expectedAddresses) { @@ -238,12 +238,8 @@ func TestNodeAddressesByProviderID(t *testing.T) { Type: v1.NodeExternalIP, Address: "99.99.99.99", }, - { - Type: "public", - Address: "2a01::10", - }, } - + ipFamilies = []IPFamily{ipv4Family} addresses, err := instances.NodeAddressesByProviderID(context.TODO(), "digitalocean://123") if !reflect.DeepEqual(addresses, expectedAddresses) {