From 413434a7fece9236732019c6fa56c3cd329e0295 Mon Sep 17 00:00:00 2001 From: nicktate Date: Wed, 13 Nov 2019 12:28:04 -0500 Subject: [PATCH] Add support for LetsEncrypt via domain annotation * Expects root domain to already be created and validated on DigitalOcean (DO is not a registrar so we assume user has preconfigured domain) * Add domain annotation to specify either the root domain or a subdomain of your choosing to the LoadBalancer service * Automatically find or generate certificate, and attach to LoadBalancer * Automatically generate A-record for your subdomain to point to the LoadBalancer --- cloud-controller-manager/do/certificates.go | 191 ++++++ .../do/certificates_test.go | 561 +++++++++++++++++- cloud-controller-manager/do/common_test.go | 6 + cloud-controller-manager/do/loadbalancers.go | 69 +++ .../do/loadbalancers_test.go | 11 + 5 files changed, 831 insertions(+), 7 deletions(-) diff --git a/cloud-controller-manager/do/certificates.go b/cloud-controller-manager/do/certificates.go index 77786e2d5..9bd004cdc 100644 --- a/cloud-controller-manager/do/certificates.go +++ b/cloud-controller-manager/do/certificates.go @@ -16,8 +16,199 @@ limitations under the License. package do +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/digitalocean/godo" + v1 "k8s.io/api/core/v1" + "k8s.io/klog" +) + const ( // DO Certificate types certTypeLetsEncrypt = "lets_encrypt" certTypeCustom = "custom" + + // Certificate constants + certPrefix = "do-ccm-" ) + +// ensureDomain checks to see if the service contains the annDODomain annotation +// and if it does it verifies the domain exists on the users account +func (l *loadBalancers) ensureDomain(ctx context.Context, service *v1.Service) error { + domain, err := getDomain(service) + if err != nil { + return err + } + + if domain != nil { + klog.V(2).Infof("looking up root domain: %s", domain.root) + _, resp, err := l.resources.gclient.Domains.Get(ctx, domain.root) + if err != nil && resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("domain does not exist: %s", domain.root) + } else if err != nil { + return fmt.Errorf("failed to retrieve domain: %s", err) + } + } + + return nil +} + +// ensureCertificate verifies the existing certificate on the service is still a valid +// DO certificate, clearing if not. If the user also has the annDODomain annotation set, +// it verifies the certificate is valid for the domain, creating a new one if it cannot +// find a existing valid certificate in the account +func (l *loadBalancers) ensureCertificate(ctx context.Context, service *v1.Service) error { + var certificate *godo.Certificate + serviceCertID := getCertificateID(service) + + // verify the cert-id is still referencing a valid certificate + if serviceCertID != "" { + klog.V(2).Infof("looking up existing service certificate: %s", serviceCertID) + certificate, _, err := l.resources.gclient.Certificates.Get(ctx, serviceCertID) + + if err != nil { + respErr, ok := err.(*godo.ErrorResponse) + if !ok || respErr.Response.StatusCode != http.StatusNotFound { + return fmt.Errorf("failed to get DO certificate for service: %s", err) + } + } + + if certificate == nil { + klog.V(2).Infof("service certificate is no longer valid: %s", serviceCertID) + serviceCertID = "" + updateServiceAnnotation(service, annDOCertificateID, serviceCertID) + } + } + + domain, err := getDomain(service) + if err != nil { + return err + } + + // no domain annotation, no need to ensure certificate for domain + if domain == nil { + return nil + } + + // if the current cert represents the domain, save making an extra network request + // this case will arise when certificates are automatically rotated + if certificate != nil { + for _, dnsName := range certificate.DNSNames { + if dnsName == domain.full { + // we found matching certificate, break out of ensureCertificate + return nil + } + } + + // the current certificate does not match the current domain. clear it so + // we can either find the matching cert or create a new one below + klog.V(2).Infof("service certificate is not valid for domain[%s]: %s", domain.full, serviceCertID) + certificate = nil + } + + certificates, _, err := l.resources.gclient.Certificates.List(ctx, &godo.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to retrieve certificates: %s", err) + } + +findCert: + for _, cert := range certificates { + for _, dnsName := range cert.DNSNames { + if dnsName == domain.full { + klog.V(2).Infof("found existing certificate for domain[%s]: %s", domain.full, cert.ID) + certificate = &cert + break findCert + } + } + } + + if certificate == nil { + certName := getCertificateName(domain.full) + dnsNames := []string{domain.root} + + if domain.sub != "" { + dnsNames = append(dnsNames, domain.full) + } + + certificateReq := &godo.CertificateRequest{ + Name: certName, + DNSNames: dnsNames, + Type: certTypeLetsEncrypt, + } + + klog.V(2).Infof("generating new service certificate for domain: %s", domain.full) + certificate, _, err = l.resources.gclient.Certificates.Create(ctx, certificateReq) + if err != nil { + return fmt.Errorf("failed to create certificate: %s", err) + } + } + + updateServiceAnnotation(service, annDOCertificateID, certificate.ID) + return nil +} + +// ensureDomainARecord ensures that if the service has a annDODomain annotation, +// the domain has an A record for the full subdomain pointing to the loadbalancer +func (l *loadBalancers) ensureDomainARecord(ctx context.Context, service *v1.Service, lb *godo.LoadBalancer) error { + domain, err := getDomain(service) + if err != nil { + return err + } + + if domain == nil { + return nil + } + + // the do loadbalancer service ensures the root domain associated with the + // certificate has an A record pointing to the LB so we do not need to ensure + if domain.sub == "" { + klog.V(2).Infof("domain has no subdomain, no need to ensure A records: %s", domain.full) + return nil + } + + records, _, err := l.resources.gclient.Domains.Records(ctx, domain.root, &godo.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to fetch records for domain: %s", err) + } + + var domainRecord *godo.DomainRecord + for _, record := range records { + if record.Type != "A" || record.Name != domain.sub { + continue + } + + if record.Data != lb.IP { + return fmt.Errorf("domain(%s) record already in use for another ip(%s)", domain.full, record.Data) + } + + klog.V(2).Infof("found A record to loadbalancer for domain: %s", domain.full) + domainRecord = &record + break + } + + if domainRecord == nil { + domainRecordEditReq := &godo.DomainRecordEditRequest{ + Type: "A", + Name: domain.sub, + Data: lb.IP, + TTL: defaultDomainRecordTTL, + } + klog.V(2).Infof("creating new A record to loadbalance for domain: %s", domain.full) + domainRecord, _, err = l.resources.gclient.Domains.CreateRecord(ctx, domain.root, domainRecordEditReq) + if err != nil { + return fmt.Errorf("failed to create domain record: %s", err) + } + } + + return nil +} + +// getCertificateName returns a prefixed certificate so we know to cleanup +// certificate when a loadbalancer for the given domain is deleted +func getCertificateName(fullDomain string) string { + return fmt.Sprintf("%s%s", certPrefix, strings.ReplaceAll(fullDomain, ".", "-")) +} diff --git a/cloud-controller-manager/do/certificates_test.go b/cloud-controller-manager/do/certificates_test.go index b70c4e5f1..183108c1f 100644 --- a/cloud-controller-manager/do/certificates_test.go +++ b/cloud-controller-manager/do/certificates_test.go @@ -18,6 +18,7 @@ package do import ( "context" + "errors" "reflect" "testing" @@ -27,6 +28,157 @@ import ( "k8s.io/client-go/kubernetes/fake" ) +type fakeDomainService struct { + domainStore map[string]*godo.Domain + recordStore map[string]map[int]*godo.DomainRecord + + listFunc func(context.Context, *godo.ListOptions) ([]godo.Domain, *godo.Response, error) + getFunc func(context.Context, string) (*godo.Domain, *godo.Response, error) + createFunc func(context.Context, *godo.DomainCreateRequest) (*godo.Domain, *godo.Response, error) + deleteFunc func(context.Context, string) (*godo.Response, error) + + recordsFunc func(context.Context, string, *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) + recordFunc func(context.Context, string, int) (*godo.DomainRecord, *godo.Response, error) + deleteRecordFunc func(context.Context, string, int) (*godo.Response, error) + editRecordFunc func(context.Context, string, int, *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) + createRecordFunc func(context.Context, string, *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) +} + +func (f *fakeDomainService) List(ctx context.Context, opts *godo.ListOptions) ([]godo.Domain, *godo.Response, error) { + return f.listFunc(ctx, opts) +} + +func (f *fakeDomainService) Get(ctx context.Context, domain string) (*godo.Domain, *godo.Response, error) { + return f.getFunc(ctx, domain) +} + +func (f *fakeDomainService) Create(ctx context.Context, opts *godo.DomainCreateRequest) (*godo.Domain, *godo.Response, error) { + return f.createFunc(ctx, opts) +} + +func (f *fakeDomainService) Delete(ctx context.Context, domain string) (*godo.Response, error) { + return f.deleteFunc(ctx, domain) +} + +func (f *fakeDomainService) Records(ctx context.Context, domain string, opts *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) { + return f.recordsFunc(ctx, domain, opts) +} + +func (f *fakeDomainService) Record(ctx context.Context, domain string, id int) (*godo.DomainRecord, *godo.Response, error) { + return f.recordFunc(ctx, domain, id) +} + +func (f *fakeDomainService) DeleteRecord(ctx context.Context, domain string, id int) (*godo.Response, error) { + return f.deleteRecordFunc(ctx, domain, id) +} + +func (f *fakeDomainService) EditRecord(ctx context.Context, domain string, id int, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + return f.editRecordFunc(ctx, domain, id, opts) +} + +func (f *fakeDomainService) CreateRecord(ctx context.Context, domain string, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + return f.createRecordFunc(ctx, domain, opts) +} + +func newKVDomainService(domainStore map[string]*godo.Domain, recordStore map[string]map[int]*godo.DomainRecord) fakeDomainService { + return fakeDomainService{ + domainStore: domainStore, + recordStore: recordStore, + + createFunc: func(ctx context.Context, opts *godo.DomainCreateRequest) (*godo.Domain, *godo.Response, error) { + domainStore[opts.Name] = &godo.Domain{ + Name: opts.Name, + } + recordStore[opts.Name] = make(map[int]*godo.DomainRecord) + + return domainStore[opts.Name], newFakeOKResponse(), nil + }, + getFunc: func(ctx context.Context, d string) (*godo.Domain, *godo.Response, error) { + domain, ok := domainStore[d] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + return domain, newFakeOKResponse(), nil + }, + createRecordFunc: func(ctx context.Context, domain string, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + recordMap, ok := recordStore[domain] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + for _, record := range recordMap { + if record.Name == opts.Name { + return nil, newFakeBadRequestResponse(), newFakeBadRequestErrorResponse() + } + } + + id := len(recordMap) + 1 + recordMap[id] = &godo.DomainRecord{ + ID: id, + Type: opts.Type, + Name: opts.Name, + Data: opts.Data, + Priority: opts.Priority, + Port: opts.Port, + TTL: opts.TTL, + Weight: opts.Weight, + Flags: opts.Flags, + Tag: opts.Tag, + } + + return recordMap[id], newFakeOKResponse(), nil + }, + deleteRecordFunc: func(ctx context.Context, domain string, id int) (*godo.Response, error) { + recordMap, ok := recordStore[domain] + if !ok { + return newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + _, ok = recordMap[id] + if !ok { + return newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + delete(recordMap, id) + return newFakeOKResponse(), nil + }, + recordsFunc: func(ctx context.Context, domain string, opts *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) { + recordMap, ok := recordStore[domain] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + records := []godo.DomainRecord{} + + for _, r := range recordMap { + records = append(records, *r) + } + + return records, newFakeOKResponse(), nil + }, + recordFunc: func(ctx context.Context, domain string, id int) (*godo.DomainRecord, *godo.Response, error) { + recordMap, ok := recordStore[domain] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + record, ok := recordMap[id] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + return record, newFakeOKResponse(), nil + }, + } +} + +func newFakeDomainClient(fakeDomain *fakeDomainService) *godo.Client { + return newFakeClient(fakeClientOpts{ + fakeDomain: fakeDomain, + }) +} + type kvCertService struct { store map[string]*godo.Certificate reflectorMode bool @@ -47,11 +199,24 @@ func (f *kvCertService) Get(ctx context.Context, certID string) (*godo.Certifica } func (f *kvCertService) List(ctx context.Context, listOpts *godo.ListOptions) ([]godo.Certificate, *godo.Response, error) { - panic("not implemented") + certificates := []godo.Certificate{} + + for _, v := range f.store { + certificates = append(certificates, *v) + } + + return certificates, newFakeOKResponse(), nil } func (f *kvCertService) Create(ctx context.Context, crtr *godo.CertificateRequest) (*godo.Certificate, *godo.Response, error) { - panic("not implemented") + f.store[crtr.Name] = &godo.Certificate{ + ID: crtr.Name, + Name: crtr.Name, + Type: crtr.Type, + DNSNames: crtr.DNSNames, + } + + return f.store[crtr.Name], newFakeOKResponse(), nil } func (f *kvCertService) Delete(ctx context.Context, certID string) (*godo.Response, error) { @@ -75,19 +240,31 @@ func createServiceAndCert(lbID, certID, certType string) (*v1.Service, *godo.Cer } func createServiceWithCert(lbID, certID string) *v1.Service { - s := createService(lbID) - s.Annotations[annDOCertificateID] = certID + s := createService() + updateServiceAnnotation(s, annoDOLoadBalancerID, lbID) + updateServiceAnnotation(s, annDOCertificateID, certID) + return s +} + +func createServiceWithLBID(id string) *v1.Service { + s := createService() + updateServiceAnnotation(s, annoDOLoadBalancerID, id) + return s +} + +func createServiceWithDomain(domain string) *v1.Service { + s := createService() + updateServiceAnnotation(s, annDODomain, domain) return s } -func createService(lbID string) *v1.Service { +func createService() *v1.Service { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", UID: "foobar123", Annotations: map[string]string{ - annDOProtocol: "http", - annoDOLoadBalancerID: lbID, + annDOProtocol: "http", }, }, Spec: v1.ServiceSpec{ @@ -103,6 +280,376 @@ func createService(lbID string) *v1.Service { } } +func Test_ensureDomain(t *testing.T) { + tests := []struct { + name string + domainService func() *fakeDomainService + service *v1.Service + err error + }{ + { + name: "no domain annotation should continue without ensuring", + domainService: func() *fakeDomainService { + return &fakeDomainService{ + getFunc: func(ctx context.Context, domain string) (*godo.Domain, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("should not fetch domain if not annotation") + }, + } + }, + service: createServiceWithDomain(""), + err: nil, + }, + { + name: "if the root domain does not exist, it should fail to ensure domain", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(make(map[string]*godo.Domain), make(map[string]map[int]*godo.DomainRecord)) + return &domainService + }, + service: createServiceWithDomain("digitalocean.com"), + err: errors.New("domain does not exist: digitalocean.com"), + }, + { + name: "if the root domain does exist, it should successfully ensure domain", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, make(map[string]map[int]*godo.DomainRecord)) + return &domainService + }, + service: createServiceWithDomain("digitalocean.com"), + err: nil, + }, + { + name: "if passed a subdomain and the root domain does exist, it should successfully ensure domain", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, make(map[string]map[int]*godo.DomainRecord)) + return &domainService + }, + service: createServiceWithDomain("subdomain.digitalocean.com"), + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := newFakeClient(fakeClientOpts{ + fakeDomain: test.domainService(), + }) + fakeResources := newResources("", "", fakeClient) + fakeResources.kclient = fake.NewSimpleClientset() + + lb := &loadBalancers{ + resources: fakeResources, + region: "nyc1", + lbActiveTimeout: 2, + lbActiveCheckTick: 1, + } + + err := lb.ensureDomain(context.TODO(), test.service) + + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + }) + } +} + +func Test_ensureCertificate(t *testing.T) { + tests := []struct { + name string + certService func() *kvCertService + service *v1.Service + err error + expectedCertificateID string + }{ + { + name: "if the cert-id does not exist and no domain, it should clear the service cert-id", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{}, false) + return &certService + }, + service: createServiceWithCert("lb-id", "cert-id"), + err: nil, + expectedCertificateID: "", + }, + { + name: "if the cert-id exists and no domain, it should leave the service cert-id", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{ + "cert-id": { + ID: "cert-id", + Name: "cert-id", + }, + }, false) + return &certService + }, + service: createServiceWithCert("lb-id", "cert-id"), + err: nil, + expectedCertificateID: "cert-id", + }, + { + name: "if the service has a root domain, and an existing cert matching that domain exists it should update to use it", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{ + "cert-id": { + ID: "cert-id", + Name: "cert-id", + DNSNames: []string{"digitalocean.com"}, + }, + }, false) + return &certService + }, + service: createServiceWithDomain("digitalocean.com"), + err: nil, + expectedCertificateID: "cert-id", + }, + { + name: "if the service has a domain with subdomain, and an existing cert matching that subdomain exists it should update to use it", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{ + "cert-id": { + ID: "cert-id", + Name: "cert-id", + DNSNames: []string{"subdomain.digitalocean.com"}, + }, + }, false) + return &certService + }, + service: createServiceWithDomain("subdomain.digitalocean.com"), + err: nil, + expectedCertificateID: "cert-id", + }, + { + name: "if the service has a root domain, and no cert exists, it should be generated", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{ + "does-not-match": { + ID: "cert-id", + Name: "cert-id", + DNSNames: []string{"invalid.digitalocean.com"}, + }, + }, false) + return &certService + }, + service: createServiceWithDomain("digitalocean.com"), + err: nil, + // the fake cert service uses the cert name as the ID + expectedCertificateID: getCertificateName("digitalocean.com"), + }, + { + name: "if the service has a subdomain, and no cert exists, it should be generated", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{ + "does-not-match": { + ID: "cert-id", + Name: "cert-id", + DNSNames: []string{"invalid.digitalocean.com"}, + }, + }, false) + return &certService + }, + service: createServiceWithDomain("subdomain.digitalocean.com"), + err: nil, + // the fake cert service uses the cert name as the ID + expectedCertificateID: getCertificateName("subdomain.digitalocean.com"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := newFakeClient(fakeClientOpts{ + fakeCert: test.certService(), + }) + fakeResources := newResources("", "", fakeClient) + fakeResources.kclient = fake.NewSimpleClientset() + + lb := &loadBalancers{ + resources: fakeResources, + region: "nyc1", + lbActiveTimeout: 2, + lbActiveCheckTick: 1, + } + + err := lb.ensureCertificate(context.TODO(), test.service) + + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + + serviceCertID := getCertificateID(test.service) + if test.expectedCertificateID != serviceCertID { + t.Errorf("got service certificate ID: %s, want: %s", serviceCertID, test.expectedCertificateID) + } + }) + } +} + +func Test_ensureDomainARecords(t *testing.T) { + tests := []struct { + name string + domainService func() *fakeDomainService + service *v1.Service + loadbalancer *godo.LoadBalancer + expectedDomainRecord *godo.DomainRecord + err error + }{ + { + name: "if service has no domain, it should not attempt to search or create A records", + domainService: func() *fakeDomainService { + return &fakeDomainService{ + recordsFunc: func(ctx context.Context, domain string, opts *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("Should not list records if no service domain annotation exists") + }, + createRecordFunc: func(ctx context.Context, domain string, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("Should not ceate records if no service domain annotation exists") + }, + } + }, + loadbalancer: nil, + service: createServiceWithDomain(""), + err: nil, + }, + { + name: "if service has only root domain, it should not attempt to search or create A records", + domainService: func() *fakeDomainService { + return &fakeDomainService{ + recordsFunc: func(ctx context.Context, domain string, opts *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("Should not list records if domain annotation is root domain") + }, + createRecordFunc: func(ctx context.Context, domain string, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("Should not create records if domain annotation is root domain") + }, + } + }, + loadbalancer: nil, + service: createServiceWithDomain("digitalocean.com"), + err: nil, + }, + { + name: "if the A record exists but does not point to the LB IP", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": { + 1: { + Type: "A", + ID: 1, + Name: "subdomain", + Data: "not-the-lb", + }, + }, + }) + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + service: createServiceWithDomain("subdomain.digitalocean.com"), + err: errors.New("domain(subdomain.digitalocean.com) record already in use for another ip(not-the-lb)"), + }, + { + name: "if the A record exists it should not attempt to create another", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": { + 1: { + Type: "A", + ID: 1, + Name: "subdomain", + Data: "10.0.0.1", + }, + }, + }) + domainService.createRecordFunc = func(ctx context.Context, domain string, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("Should not create records if the A record already exists") + } + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + service: createServiceWithDomain("subdomain.digitalocean.com"), + err: nil, + }, + { + name: "if the A record does not exist, it should create one pointing to the LB", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": { + 1: { + Type: "A", + ID: 1, + Name: "a-different-domain", + Data: "10.0.0.1", + }, + }, + }) + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + service: createServiceWithDomain("subdomain.digitalocean.com"), + expectedDomainRecord: &godo.DomainRecord{ + // we can assume 2 because the fakeKVDomain service just auto-increments IDs from 1 + ID: 2, + Type: "A", + Name: "subdomain", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := newFakeClient(fakeClientOpts{ + fakeDomain: test.domainService(), + }) + fakeResources := newResources("", "", fakeClient) + fakeResources.kclient = fake.NewSimpleClientset() + + lb := &loadBalancers{ + resources: fakeResources, + region: "nyc1", + lbActiveTimeout: 2, + lbActiveCheckTick: 1, + } + + err := lb.ensureDomainARecord(context.TODO(), test.service, test.loadbalancer) + + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + + if test.expectedDomainRecord != nil { + expectedRecord := test.expectedDomainRecord + domain, _ := getDomain(test.service) + actualRecord, _, err := fakeClient.Domains.Record(context.TODO(), domain.root, expectedRecord.ID) + + if err != nil { + t.Error("unexpected error fetching record") + t.Logf("err: %v", err) + } + + if !reflect.DeepEqual(expectedRecord, actualRecord) { + t.Error("unexpected error") + t.Logf("expected: %v", expectedRecord) + t.Logf("actual: %v", actualRecord) + } + } + }) + } +} + func Test_LBaaSCertificateScenarios(t *testing.T) { tests := []struct { name string diff --git a/cloud-controller-manager/do/common_test.go b/cloud-controller-manager/do/common_test.go index fbefc07eb..af5d2c704 100644 --- a/cloud-controller-manager/do/common_test.go +++ b/cloud-controller-manager/do/common_test.go @@ -33,6 +33,7 @@ type fakeClientOpts struct { fakeDroplet *fakeDropletService fakeLB *fakeLBService fakeCert *kvCertService + fakeDomain *fakeDomainService } func newFakeClient(opts fakeClientOpts) *godo.Client { @@ -40,6 +41,7 @@ func newFakeClient(opts fakeClientOpts) *godo.Client { Certificates: opts.fakeCert, Droplets: opts.fakeDroplet, LoadBalancers: opts.fakeLB, + Domains: opts.fakeDomain, } } @@ -86,6 +88,10 @@ func newFakeBadRequestResponse() *godo.Response { return newFakeResponse(http.StatusBadRequest) } +func newFakeBadRequestErrorResponse() *godo.ErrorResponse { + return newFakeErrorResponse(http.StatusBadRequest) +} + func linksForPage(page int) *godo.Links { switch page { case 0, 1: diff --git a/cloud-controller-manager/do/loadbalancers.go b/cloud-controller-manager/do/loadbalancers.go index 998519ec0..b93cf4c43 100644 --- a/cloud-controller-manager/do/loadbalancers.go +++ b/cloud-controller-manager/do/loadbalancers.go @@ -162,6 +162,8 @@ const ( portProtocolTCP = "TCP" defaultSecurePort = 443 + + defaultDomainRecordTTL = 3600 ) var errLBNotFound = errors.New("loadbalancer not found") @@ -256,6 +258,18 @@ func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri patcher := newServicePatcher(l.resources.kclient, service) defer func() { err = patcher.Patch(err) }() + // If domain annotation, verify domain exists, otherwise error + err = l.ensureDomain(ctx, service) + if err != nil { + return nil, err + } + + // Ensure certificate exists and valid for domain if one exists + err = l.ensureCertificate(ctx, service) + if err != nil { + return nil, err + } + var lbRequest *godo.LoadBalancerRequest lbRequest, err = l.buildLoadBalancerRequest(ctx, service, nodes) if err != nil { @@ -290,6 +304,12 @@ func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri return nil, fmt.Errorf("load-balancer is not yet active (current status: %s)", lb.Status) } + // If a domain exists, ensure that the domain has an A record pointing to the LB IP + err = l.ensureDomainARecord(ctx, service, lb) + if err != nil { + return nil, err + } + // If a LB hostname annotation is specified, return with it instead of the IP. hostname := getHostname(service) if hostname != "" { @@ -422,6 +442,7 @@ func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa return err } + klog.V(2).Infof("deleting do loadbalancer: %s", lb.ID) resp, err := l.resources.gclient.LoadBalancers.Delete(ctx, lb.ID) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { @@ -430,6 +451,54 @@ func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa return fmt.Errorf("failed to delete load-balancer: %s", err) } + domain, err := getDomain(service) + if err != nil { + klog.Errorf("failed to fetch domain from service for cleanup: %s", err) + return nil + } + + // attempt to clean up any managed A records or certificates related to LB + if domain != nil && lb != nil { + certID := getCertificateIDFromLB(lb) + certName := getCertificateName(domain.full) + cert, resp, err := l.resources.gclient.Certificates.Get(ctx, certID) + if err != nil && resp.StatusCode != http.StatusNotFound { + klog.Errorf("failed to fetch certificate: %s", err) + } + + // if this certificate name matches the autogenerated cert format then cleanup + if cert != nil && cert.Name == certName { + klog.V(2).Infof("deleting do certificate: %s", certID) + + resp, err := l.resources.gclient.Certificates.Delete(ctx, certID) + if err != nil && resp.StatusCode != http.StatusNotFound { + klog.Errorf("failed to cleanup certificate: %s", err) + } + } + + records, _, err := l.resources.gclient.Domains.Records(ctx, domain.root, &godo.ListOptions{}) + if err != nil { + klog.Errorf("failed to fetch records for domain: %s", err) + return nil + } + + // if subdomain existed, attempt to find and clean up A record that was created + if domain.sub != "" { + for _, record := range records { + // If we find the A record corresponding to this domain and LB + if record.Type == "A" && record.Name == domain.sub && record.Data == lb.IP { + klog.V(2).Infof("deleting loadbalancer A record for domain(%s): %s", domain.root, record.Name) + resp, err = l.resources.gclient.Domains.DeleteRecord(ctx, domain.root, record.ID) + if err != nil && resp.StatusCode != http.StatusNotFound { + klog.Errorf("failed to domain A record: %s", err) + return nil + } + break + } + } + } + } + return nil } diff --git a/cloud-controller-manager/do/loadbalancers_test.go b/cloud-controller-manager/do/loadbalancers_test.go index 520185bab..b59b013c7 100644 --- a/cloud-controller-manager/do/loadbalancers_test.go +++ b/cloud-controller-manager/do/loadbalancers_test.go @@ -134,6 +134,17 @@ func createLB() *godo.LoadBalancer { } } +func createLBWithIP(ip string) *godo.LoadBalancer { + return &godo.LoadBalancer{ + // loadbalancer names are a + service.UID + // see cloudprovider.DefaultLoadBalancerName + ID: "load-balancer-id", + Name: "afoobar123", + IP: ip, + Status: lbStatusActive, + } +} + func createHTTPSLB(lbID, certID, certType string) (*godo.LoadBalancer, *godo.Certificate) { lb := &godo.LoadBalancer{ // loadbalancer names are a + service.UID