GC registry for deferred IP removal (#25)

This patch adds mechanisms to record IPs returned and released when
using the skipDeallocate mode of the IPAM plugin. A systemd service or
cron job can be used with the -tool to allow delayed removal of these
IPs from the system if they are not reused within a certain period of
time.

This is extremely helpful in cases where a large number of pods are
created and removed in the system to avoid running into AWS rate limits.

Dockerfile

@@ -5,8 +5,8 @@ WORKDIR /go/src/github.com/lyft/cni-ipvlan-vpc-k8s/
 
 RUN go get github.com/golang/dep && \
   go install github.com/golang/dep/cmd/dep && \
-  go get -u gopkg.in/alecthomas/gometalinter.v1 && \
-  gometalinter.v1 --install
+  go get -u gopkg.in/alecthomas/gometalinter.v2 && \
+  gometalinter.v2 --install
 
 COPY . /go/src/github.com/lyft/cni-ipvlan-vpc-k8s/
 

Makefile

@@ -20,25 +20,27 @@ cache:
 
 .PHONY: lint
 lint:
-	gometalinter.v1 --disable-all \
-		--enable=golint --enable=staticcheck --enable=gosimple --enable=unused --enable=gas \
+	gometalinter.v2 --disable-all \
+		--enable=golint --enable=megacheck --enable=gas \
 		--enable=gofmt \
-		--deadline=10m --vendor ./...
+		--deadline=10m --vendor ./... \
+		--exclude="Errors unhandled.*" \
+		--enable-gc
 
 .PHONY: test
 test: dep cache lint
 ifndef GOOS
-	go test -v ./aws/... ./nl ./cmd/cni-ipvlan-vpc-k8s-tool
+	go test -v ./aws/... ./nl ./cmd/cni-ipvlan-vpc-k8s-tool ./lib/...
 else
 	@echo Tests not available when cross-compiling
 endif
 
 .PHONY: build
 build: dep cache
-	go build -o $(NAME)-ipam ./plugin/ipam/main.go
-	go build -o $(NAME)-ipvlan ./plugin/ipvlan/ipvlan.go
-	go build -o $(NAME)-unnumbered-ptp ./plugin/unnumbered-ptp/unnumbered-ptp.go
-	go build -ldflags "-X main.version=$(VERSION)" -o $(NAME)-tool ./cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go
+	go build -i -o $(NAME)-ipam ./plugin/ipam/main.go
+	go build -i -o $(NAME)-ipvlan ./plugin/ipvlan/ipvlan.go
+	go build -i -o $(NAME)-unnumbered-ptp ./plugin/unnumbered-ptp/unnumbered-ptp.go
+	go build -i -ldflags "-X main.version=$(VERSION)" -o $(NAME)-tool ./cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go
 
 	tar cvzf cni-ipvlan-vpc-k8s-$(VERSION).tar.gz $(NAME)-ipam $(NAME)-ipvlan $(NAME)-unnumbered-ptp $(NAME)-tool
 
@@ -58,7 +60,7 @@ interactive-docker: test-docker
 ci:
 	go get -u github.com/golang/dep/cmd/dep
 	go install github.com/golang/dep/cmd/dep
-	go get -u gopkg.in/alecthomas/gometalinter.v1
-	gometalinter.v1 --install
+	go get -u gopkg.in/alecthomas/gometalinter.v2
+	gometalinter.v2 --install
 
 	$(MAKE) all

aws/cache/cacheable.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"path"
 	"time"
+
+	"github.com/lyft/cni-ipvlan-vpc-k8s/lib"
 )
 
 const (
@@ -36,35 +38,10 @@ func cachePath() string {
 	return path.Join(cacheRoot, cacheProgram)
 }
 
-// JSONTime is a RFC3339 encoded time with JSON marshallers
-type JSONTime struct {
-	time.Time
-}
-
-// MarshalJSON marshals a JSONTime to an RFC3339 string
-func (j *JSONTime) MarshalJSON() ([]byte, error) {
-	return json.Marshal(j.Time.Format(time.RFC3339))
-}
-
-// UnmarshalJSON unmarshals a JSONTime to a time.Time
-func (j *JSONTime) UnmarshalJSON(js []byte) error {
-	var rawString string
-	err := json.Unmarshal(js, &rawString)
-	if err != nil {
-		return err
-	}
-	t, err := time.Parse(time.RFC3339, rawString)
-	if err != nil {
-		return err
-	}
-	j.Time = t
-	return nil
-}
-
 // Cacheable defines metadata for objects which can be cached to files as JSON
 type Cacheable struct {
-	Expires  JSONTime    `json:"_expires"`
-	Contents interface{} `json":contents"`
+	Expires  lib.JSONTime `json:"_expires"`
+	Contents interface{}  `json":contents"`
 }
 
 func ensureDirectory() error {

aws/cache/cacheable_test.go

@@ -1,44 +1,15 @@
 package cache
 
 import (
-	"encoding/json"
 	"testing"
 	"time"
 )
 
-type Foo struct {
-	TheTime JSONTime `json:"time"`
-}
-
 type Thing struct {
 	Value  string `json:"value"`
 	TValue int
 }
 
-func TestJSONTime_MarshalJSON(t *testing.T) {
-	input := Foo{JSONTime{time.Date(2017, 1, 1, 1, 1, 0, 0, time.UTC)}}
-	output, err := json.Marshal(&input)
-	if err != nil {
-		t.Error(err)
-	}
-	if string(output) != `{"time":"2017-01-01T01:01:00Z"}` {
-		t.Error(string(output))
-	}
-}
-
-func TestJSONTime_UnmarshalJSON(t *testing.T) {
-	input := []byte(`{"time":"2017-01-01T01:01:00Z"}`)
-	var foo Foo
-	err := json.Unmarshal(input, &foo)
-	if err != nil {
-		t.Error(err)
-	}
-	expected := Foo{JSONTime{time.Date(2017, 1, 1, 1, 1, 0, 0, time.UTC)}}
-	if !foo.TheTime.Time.Equal(expected.TheTime.Time) {
-		t.Errorf("Times were not equal: %v %v %v", foo.TheTime.Time, expected.TheTime.Time, foo)
-	}
-}
-
 func TestGet(t *testing.T) {
 	var d Thing
 	state := Get("key_not_exist", &d)

aws/client_test.go

@@ -25,7 +25,7 @@ func TestClientCreate(t *testing.T) {
 	}
 
 	client2, err := defaultClient.newEC2()
-	if client != client2 {
+	if client != client2 || err != nil {
 		t.Errorf("Clients returned were not identical (no caching)")
 	}
 

aws/vpc.go

@@ -101,7 +101,7 @@ func (v *vpcclient) DescribeVPCPeerCIDRs(vpcID string) ([]*net.IPNet, error) {
 	// and visible to the API, even if the CIDR is not active in
 	// one of the peered VPCs. We store all of the CIDRs in a map
 	// to de-duplicate them.
-	cidrs := make(map[string]bool, 0)
+	cidrs := make(map[string]bool)
 
 	for _, peering := range res.VpcPeeringConnections {
 		var peer *ec2.VpcPeeringConnectionVpcInfo

cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go

@@ -6,12 +6,15 @@ import (
 	"os"
 	"strings"
 	"text/tabwriter"
+	"time"
 
 	"github.com/urfave/cli"
 
-	"github.com/lyft/cni-ipvlan-vpc-k8s"
 	"github.com/lyft/cni-ipvlan-vpc-k8s/aws"
+	"github.com/lyft/cni-ipvlan-vpc-k8s/lib"
+	"github.com/lyft/cni-ipvlan-vpc-k8s/lib/freeip"
 	"github.com/lyft/cni-ipvlan-vpc-k8s/nl"
+	"github.com/lyft/cni-ipvlan-vpc-k8s/registry"
 )
 
 var version string
@@ -22,7 +25,7 @@ func filterBuild(input string) (map[string]string, error) {
 		return nil, nil
 	}
 
-	ret := make(map[string]string, 0)
+	ret := make(map[string]string)
 	tuples := strings.Split(input, ",")
 	for _, t := range tuples {
 		kv := strings.Split(t, "=")
@@ -40,7 +43,7 @@ func filterBuild(input string) (map[string]string, error) {
 }
 
 func actionNewInterface(c *cli.Context) error {
-	return cniipvlanvpck8s.LockfileRun(func() error {
+	return lib.LockfileRun(func() error {
 		filtersRaw := c.String("subnet_filter")
 		filters, err := filterBuild(filtersRaw)
 		if err != nil {
@@ -66,7 +69,7 @@ func actionNewInterface(c *cli.Context) error {
 }
 
 func actionBugs(c *cli.Context) error {
-	return cniipvlanvpck8s.LockfileRun(func() error {
+	return lib.LockfileRun(func() error {
 		w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
 		fmt.Fprintln(w, "bug\tafflicted\t")
 		for _, bug := range aws.ListBugs(aws.DefaultClient) {
@@ -78,7 +81,7 @@ func actionBugs(c *cli.Context) error {
 }
 
 func actionRemoveInterface(c *cli.Context) error {
-	return cniipvlanvpck8s.LockfileRun(func() error {
+	return lib.LockfileRun(func() error {
 		interfaces := c.Args()
 
 		if len(interfaces) <= 0 {
@@ -96,7 +99,7 @@ func actionRemoveInterface(c *cli.Context) error {
 }
 
 func actionDeallocate(c *cli.Context) error {
-	return cniipvlanvpck8s.LockfileRun(func() error {
+	return lib.LockfileRun(func() error {
 		releaseIps := c.Args()
 		for _, toRelease := range releaseIps {
 
@@ -122,7 +125,7 @@ func actionDeallocate(c *cli.Context) error {
 }
 
 func actionAllocate(c *cli.Context) error {
-	return cniipvlanvpck8s.LockfileRun(func() error {
+	return lib.LockfileRun(func() error {
 		index := c.Int("index")
 		res, err := aws.DefaultClient.AllocateIPFirstAvailableAtIndex(index)
 		if err != nil {
@@ -137,7 +140,7 @@ func actionAllocate(c *cli.Context) error {
 }
 
 func actionFreeIps(c *cli.Context) error {
-	ips, err := cniipvlanvpck8s.FindFreeIPsAtIndex(0)
+	ips, err := freeip.FindFreeIPsAtIndex(0)
 	if err != nil {
 		fmt.Println(err)
 		return err
@@ -270,6 +273,60 @@ func actionSubnets(c *cli.Context) error {
 	return nil
 }
 
+func actionRegistryList(c *cli.Context) error {
+	return lib.LockfileRun(func() error {
+
+		reg := &registry.Registry{}
+		ips, err := reg.List()
+		if err != nil {
+			return err
+		}
+		w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
+		fmt.Fprintln(w, "ip\t")
+		for _, ip := range ips {
+			fmt.Fprintf(w, "%v\t\n",
+				ip)
+		}
+		w.Flush()
+		return nil
+	})
+}
+
+func actionRegistryGc(c *cli.Context) error {
+	return lib.LockfileRun(func() error {
+
+		reg := &registry.Registry{}
+		freeAfter := c.Duration("free-after")
+		if freeAfter <= 0*time.Second {
+			fmt.Fprintf(os.Stderr,
+				"Invalid duration specified. free-after must be > 0 seconds. Got %v. Please specify with --free-minutes=[time]\n", freeAfter)
+			return fmt.Errorf("invalid duration")
+		}
+
+		// Insert free-after jitter of 15% of the period
+		freeAfter = registry.Jitter(freeAfter, 0.15)
+
+		// Invert free-after
+		freeAfter *= -1
+
+		ips, err := reg.TrackedBefore(time.Now().Add(freeAfter))
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			return err
+		}
+		for _, ip := range ips {
+			err := aws.DefaultClient.DeallocateIP(&ip)
+			if err == nil {
+				reg.ForgetIP(ip)
+			} else {
+				fmt.Fprintf(os.Stderr, "Can't deallocate %v due to %v", ip, err)
+			}
+		}
+
+		return nil
+	})
+}
+
 func main() {
 	if !aws.DefaultClient.Available() {
 		fmt.Fprintln(os.Stderr, "This command must be run from a running ec2 instance")
@@ -355,6 +412,20 @@ func main() {
 			Usage:  "Show the peered VPC CIDRs associated with current interfaces",
 			Action: actionVpcPeerCidr,
 		},
+		{
+			Name:   "registry-list",
+			Usage:  "List all known free IPs in the internal registry",
+			Action: actionRegistryList,
+		},
+		{
+			Name:   "registry-gc",
+			Usage:  "Free all IPs that have remained unused for a given time interval",
+			Action: actionRegistryGc,
+			Flags: []cli.Flag{
+				cli.DurationFlag{Name: "free-after",
+					Value: 0 * time.Second},
+			},
+		},
 	}
 	app.Version = version
 	app.Copyright = "(c) 2017-2018 Lyft Inc."

freeip.go → lib/freeip/freeip.go

@@ -1,4 +1,4 @@
-package cniipvlanvpck8s
+package freeip
 
 import (
 	"github.com/lyft/cni-ipvlan-vpc-k8s/aws"

lib/jsontime.go

@@ -0,0 +1,31 @@
+package lib
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// JSONTime is a RFC3339 encoded time with JSON marshallers
+type JSONTime struct {
+	time.Time
+}
+
+// MarshalJSON marshals a JSONTime to an RFC3339 string
+func (j *JSONTime) MarshalJSON() ([]byte, error) {
+	return json.Marshal(j.Time.Format(time.RFC3339))
+}
+
+// UnmarshalJSON unmarshals a JSONTime to a time.Time
+func (j *JSONTime) UnmarshalJSON(js []byte) error {
+	var rawString string
+	err := json.Unmarshal(js, &rawString)
+	if err != nil {
+		return err
+	}
+	t, err := time.Parse(time.RFC3339, rawString)
+	if err != nil {
+		return err
+	}
+	j.Time = t
+	return nil
+}