From 90507a82faf3462a2b2a017344cdf74fab62d7cf Mon Sep 17 00:00:00 2001 From: nicktate Date: Wed, 13 Nov 2019 12:28:04 -0500 Subject: [PATCH] Add support for LetsEncrypt via domain annotation * Expects root domain to already be created and validated on DigitalOcean (DO is not a registrar so we assume user has preconfigured domain) * Add domain annotation to specify either the root domain or a subdomain of your choosing to the LoadBalancer service * Automatically find or generate certificate, and attach to LoadBalancer * Automatically generate A-record for your subdomain to point to the LoadBalancer --- cloud-controller-manager/do/certificates.go | 204 ++++ .../do/certificates_test.go | 875 +++++++++++++++++- cloud-controller-manager/do/common_test.go | 6 + cloud-controller-manager/do/loadbalancers.go | 140 ++- .../do/loadbalancers_test.go | 382 ++++++-- 5 files changed, 1502 insertions(+), 105 deletions(-) diff --git a/cloud-controller-manager/do/certificates.go b/cloud-controller-manager/do/certificates.go index 77786e2d5..1e233aeba 100644 --- a/cloud-controller-manager/do/certificates.go +++ b/cloud-controller-manager/do/certificates.go @@ -16,8 +16,212 @@ limitations under the License. package do +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/digitalocean/godo" + v1 "k8s.io/api/core/v1" + "k8s.io/klog" +) + const ( // DO Certificate types certTypeLetsEncrypt = "lets_encrypt" certTypeCustom = "custom" + + // Certificate constants + certPrefix = "do-ccm-" ) + +// ensureDomain checks to see if the service contains the annDODomain annotation +// and if it does it verifies the domain exists on the users account +func (l *loadBalancers) ensureDomain(ctx context.Context, service *v1.Service) (*domain, error) { + domain, err := getDomain(service) + if err != nil { + return domain, err + } + + if domain == nil { + return nil, nil + } + + klog.V(2).Infof("Looking up root domain specified in service: %s", domain.root) + _, _, err = l.resources.gclient.Domains.Get(ctx, domain.root) + if err != nil { + return nil, fmt.Errorf("failed to retrieve root domain %s: %s", domain.root, err) + } + + return domain, nil +} + +func (l *loadBalancers) validateCertificateExistence(ctx context.Context, certificateID string) (*godo.Certificate, error) { + if certificateID == "" { + return nil, nil + } + + certificate, resp, err := l.resources.gclient.Certificates.Get(ctx, certificateID) + if err != nil && resp.StatusCode != http.StatusNotFound { + return nil, fmt.Errorf("failed to fetch certificate: %s", err) + } + + return certificate, nil +} + +// validateServiceCertificate ensures the certificate specified in the service annotation +// still exists. If it does not, then the annotation is cleared from the service. +func (l *loadBalancers) validateServiceCertificate(ctx context.Context, service *v1.Service) (*godo.Certificate, error) { + certificateID := getCertificateID(service) + klog.V(2).Infof("Looking up certificate for service %s/%s by ID %s", service.Namespace, service.Name, certificateID) + certificate, err := l.validateCertificateExistence(ctx, certificateID) + if err != nil { + return nil, err + } + + if certificate == nil { + updateServiceAnnotation(service, annDOCertificateID, "") + } + + return certificate, nil +} + +func (l *loadBalancers) ensureCertificateForDomain(ctx context.Context, serviceCertificate *godo.Certificate, domain *domain) (*godo.Certificate, error) { + if serviceCertificate != nil && isValidCertificateForDomain(serviceCertificate, domain) { + return serviceCertificate, nil + } + + serviceCertificate, err := l.findCertificateForDomain(ctx, domain) + if err != nil { + return nil, err + } + + if serviceCertificate == nil { + serviceCertificate, err = l.generateCertificateForDomain(ctx, domain) + if err != nil { + return nil, err + } + } + + return serviceCertificate, nil +} + +func isValidCertificateForDomain(certificate *godo.Certificate, domain *domain) bool { + for _, dnsName := range certificate.DNSNames { + if dnsName == domain.full { + // we found matching certificate, break out of ensureCertificate + return true + } + } + + return false +} + +func (l *loadBalancers) findCertificateForDomain(ctx context.Context, domain *domain) (*godo.Certificate, error) { + certificates, _, err := l.resources.gclient.Certificates.List(ctx, &godo.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("Failed to list certificates: %s", err) + } + + var certificate *godo.Certificate + + for _, c := range certificates { + if isValidCertificateForDomain(&c, domain) { + certificate = &c + break + } + } + + return certificate, nil +} + +func (l *loadBalancers) generateCertificateForDomain(ctx context.Context, domain *domain) (*godo.Certificate, error) { + certName := getCertificateName(domain.full) + dnsNames := []string{domain.root} + + if domain.sub != "" { + dnsNames = append(dnsNames, domain.full) + } + + certificateReq := &godo.CertificateRequest{ + Name: certName, + DNSNames: dnsNames, + Type: certTypeLetsEncrypt, + } + + klog.V(2).Infof("Generating new certificate for domain: %s", domain.full) + certificate, _, err := l.resources.gclient.Certificates.Create(ctx, certificateReq) + if err != nil { + return nil, fmt.Errorf("failed to create certificate: %s", err) + } + + return certificate, nil +} + +func findARecordForNameAndIP(records []godo.DomainRecord, name string, ip string) (*godo.DomainRecord, error) { + var record *godo.DomainRecord + + for _, r := range records { + if r.Type != "A" || r.Name != name { + continue + } + + if r.Data != ip { + return nil, fmt.Errorf("the A record(%s) is already in use with another IP(%s)", name, r.Data) + } + + record = &r + break + } + + return record, nil +} + +// ensureDomainARecords ensures that if the service has a domain annotation, +// the domain has an A record for the full subdomain pointing to the loadbalancer +func (l *loadBalancers) ensureDomainARecords(ctx context.Context, domain *domain, lb *godo.LoadBalancer) error { + records, _, err := l.resources.gclient.Domains.Records(ctx, domain.root, &godo.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to fetch records for domain(%s): %s", domain.root, err) + } + + err = l.ensureDomainARecord(ctx, records, domain.root, "@", lb.IP) + if err != nil { + return err + } + + err = l.ensureDomainARecord(ctx, records, domain.root, domain.sub, lb.IP) + if err != nil { + return err + } + + return nil +} + +func (l *loadBalancers) ensureDomainARecord(ctx context.Context, records []godo.DomainRecord, domain string, name string, ip string) error { + record, err := findARecordForNameAndIP(records, name, ip) + if err != nil { + return err + } + + if record == nil { + _, _, err = l.resources.gclient.Domains.CreateRecord(ctx, domain, &godo.DomainRecordEditRequest{ + Type: "A", + Name: name, + Data: ip, + TTL: defaultDomainRecordTTL, + }) + if err != nil { + return err + } + } + + return nil +} + +// getCertificateName returns a prefixed certificate so we know to cleanup +// certificate when a loadbalancer for the given domain is deleted +func getCertificateName(fullDomain string) string { + return fmt.Sprintf("%s%s", certPrefix, strings.ReplaceAll(fullDomain, ".", "-")) +} diff --git a/cloud-controller-manager/do/certificates_test.go b/cloud-controller-manager/do/certificates_test.go index b70c4e5f1..a8fb42a10 100644 --- a/cloud-controller-manager/do/certificates_test.go +++ b/cloud-controller-manager/do/certificates_test.go @@ -18,6 +18,7 @@ package do import ( "context" + "errors" "reflect" "testing" @@ -27,6 +28,161 @@ import ( "k8s.io/client-go/kubernetes/fake" ) +type fakeDomainService struct { + domainStore map[string]*godo.Domain + recordStore map[string]map[int]*godo.DomainRecord + + listFunc func(context.Context, *godo.ListOptions) ([]godo.Domain, *godo.Response, error) + getFunc func(context.Context, string) (*godo.Domain, *godo.Response, error) + createFunc func(context.Context, *godo.DomainCreateRequest) (*godo.Domain, *godo.Response, error) + deleteFunc func(context.Context, string) (*godo.Response, error) + + recordsFunc func(context.Context, string, *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) + recordFunc func(context.Context, string, int) (*godo.DomainRecord, *godo.Response, error) + deleteRecordFunc func(context.Context, string, int) (*godo.Response, error) + editRecordFunc func(context.Context, string, int, *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) + createRecordFunc func(context.Context, string, *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) +} + +func (f *fakeDomainService) List(ctx context.Context, opts *godo.ListOptions) ([]godo.Domain, *godo.Response, error) { + return f.listFunc(ctx, opts) +} + +func (f *fakeDomainService) Get(ctx context.Context, domain string) (*godo.Domain, *godo.Response, error) { + return f.getFunc(ctx, domain) +} + +func (f *fakeDomainService) Create(ctx context.Context, opts *godo.DomainCreateRequest) (*godo.Domain, *godo.Response, error) { + return f.createFunc(ctx, opts) +} + +func (f *fakeDomainService) Delete(ctx context.Context, domain string) (*godo.Response, error) { + return f.deleteFunc(ctx, domain) +} + +func (f *fakeDomainService) Records(ctx context.Context, domain string, opts *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) { + return f.recordsFunc(ctx, domain, opts) +} + +func (f *fakeDomainService) Record(ctx context.Context, domain string, id int) (*godo.DomainRecord, *godo.Response, error) { + return f.recordFunc(ctx, domain, id) +} + +func (f *fakeDomainService) DeleteRecord(ctx context.Context, domain string, id int) (*godo.Response, error) { + return f.deleteRecordFunc(ctx, domain, id) +} + +func (f *fakeDomainService) EditRecord(ctx context.Context, domain string, id int, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + return f.editRecordFunc(ctx, domain, id, opts) +} + +func (f *fakeDomainService) CreateRecord(ctx context.Context, domain string, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + return f.createRecordFunc(ctx, domain, opts) +} + +func newEmptyKVDomainService() fakeDomainService { + return newKVDomainService(make(map[string]*godo.Domain), make(map[string]map[int]*godo.DomainRecord)) +} + +func newKVDomainService(domainStore map[string]*godo.Domain, recordStore map[string]map[int]*godo.DomainRecord) fakeDomainService { + return fakeDomainService{ + domainStore: domainStore, + recordStore: recordStore, + + createFunc: func(ctx context.Context, opts *godo.DomainCreateRequest) (*godo.Domain, *godo.Response, error) { + domainStore[opts.Name] = &godo.Domain{ + Name: opts.Name, + } + recordStore[opts.Name] = make(map[int]*godo.DomainRecord) + + return domainStore[opts.Name], newFakeOKResponse(), nil + }, + getFunc: func(ctx context.Context, d string) (*godo.Domain, *godo.Response, error) { + domain, ok := domainStore[d] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + return domain, newFakeOKResponse(), nil + }, + createRecordFunc: func(ctx context.Context, domain string, opts *godo.DomainRecordEditRequest) (*godo.DomainRecord, *godo.Response, error) { + recordMap, ok := recordStore[domain] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + for _, record := range recordMap { + if record.Name == opts.Name { + return nil, newFakeBadRequestResponse(), newFakeBadRequestErrorResponse() + } + } + + id := len(recordMap) + 1 + recordMap[id] = &godo.DomainRecord{ + ID: id, + Type: opts.Type, + Name: opts.Name, + Data: opts.Data, + Priority: opts.Priority, + Port: opts.Port, + TTL: opts.TTL, + Weight: opts.Weight, + Flags: opts.Flags, + Tag: opts.Tag, + } + + return recordMap[id], newFakeOKResponse(), nil + }, + deleteRecordFunc: func(ctx context.Context, domain string, id int) (*godo.Response, error) { + recordMap, ok := recordStore[domain] + if !ok { + return newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + _, ok = recordMap[id] + if !ok { + return newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + delete(recordMap, id) + return newFakeOKResponse(), nil + }, + recordsFunc: func(ctx context.Context, domain string, opts *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) { + recordMap, ok := recordStore[domain] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + records := []godo.DomainRecord{} + + for _, r := range recordMap { + records = append(records, *r) + } + + return records, newFakeOKResponse(), nil + }, + recordFunc: func(ctx context.Context, domain string, id int) (*godo.DomainRecord, *godo.Response, error) { + recordMap, ok := recordStore[domain] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + record, ok := recordMap[id] + if !ok { + return nil, newFakeNotFoundResponse(), newFakeNotFoundErrorResponse() + } + + return record, newFakeOKResponse(), nil + }, + } +} + +func newFakeDomainClient(fakeDomain *fakeDomainService) *godo.Client { + return newFakeClient(fakeClientOpts{ + fakeDomain: fakeDomain, + }) +} + type kvCertService struct { store map[string]*godo.Certificate reflectorMode bool @@ -47,11 +203,24 @@ func (f *kvCertService) Get(ctx context.Context, certID string) (*godo.Certifica } func (f *kvCertService) List(ctx context.Context, listOpts *godo.ListOptions) ([]godo.Certificate, *godo.Response, error) { - panic("not implemented") + certificates := []godo.Certificate{} + + for _, v := range f.store { + certificates = append(certificates, *v) + } + + return certificates, newFakeOKResponse(), nil } func (f *kvCertService) Create(ctx context.Context, crtr *godo.CertificateRequest) (*godo.Certificate, *godo.Response, error) { - panic("not implemented") + f.store[crtr.Name] = &godo.Certificate{ + ID: crtr.Name, + Name: crtr.Name, + Type: crtr.Type, + DNSNames: crtr.DNSNames, + } + + return f.store[crtr.Name], newFakeOKResponse(), nil } func (f *kvCertService) Delete(ctx context.Context, certID string) (*godo.Response, error) { @@ -75,19 +244,31 @@ func createServiceAndCert(lbID, certID, certType string) (*v1.Service, *godo.Cer } func createServiceWithCert(lbID, certID string) *v1.Service { - s := createService(lbID) - s.Annotations[annDOCertificateID] = certID + s := createService() + updateServiceAnnotation(s, annoDOLoadBalancerID, lbID) + updateServiceAnnotation(s, annDOCertificateID, certID) + return s +} + +func createServiceWithLBID(id string) *v1.Service { + s := createService() + updateServiceAnnotation(s, annoDOLoadBalancerID, id) + return s +} + +func createServiceWithDomain(domain string) *v1.Service { + s := createService() + updateServiceAnnotation(s, annDODomain, domain) return s } -func createService(lbID string) *v1.Service { +func createService() *v1.Service { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", UID: "foobar123", Annotations: map[string]string{ - annDOProtocol: "http", - annoDOLoadBalancerID: lbID, + annDOProtocol: "http", }, }, Spec: v1.ServiceSpec{ @@ -103,6 +284,686 @@ func createService(lbID string) *v1.Service { } } +func Test_ensureDomain(t *testing.T) { + tests := []struct { + name string + domainService func() *fakeDomainService + service *v1.Service + err error + }{ + { + name: "no domain annotation should continue without ensuring", + domainService: func() *fakeDomainService { + return &fakeDomainService{ + getFunc: func(ctx context.Context, domain string) (*godo.Domain, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("should not fetch domain if not annotation") + }, + } + }, + service: createServiceWithDomain(""), + err: nil, + }, + { + name: "if the root domain does not exist, it should fail to ensure domain", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(make(map[string]*godo.Domain), make(map[string]map[int]*godo.DomainRecord)) + return &domainService + }, + service: createServiceWithDomain("digitalocean.com"), + err: errors.New("failed to retrieve root domain digitalocean.com: FAKE : 404 "), + }, + { + name: "if the root domain does exist, it should successfully ensure domain", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, make(map[string]map[int]*godo.DomainRecord)) + return &domainService + }, + service: createServiceWithDomain("digitalocean.com"), + err: nil, + }, + { + name: "if passed a subdomain and the root domain does exist, it should successfully ensure domain", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, make(map[string]map[int]*godo.DomainRecord)) + return &domainService + }, + service: createServiceWithDomain("subdomain.digitalocean.com"), + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := newFakeClient(fakeClientOpts{ + fakeDomain: test.domainService(), + }) + fakeResources := newResources("", "", fakeClient) + fakeResources.kclient = fake.NewSimpleClientset() + + lb := &loadBalancers{ + resources: fakeResources, + region: "nyc1", + lbActiveTimeout: 2, + lbActiveCheckTick: 1, + } + + _, err := lb.ensureDomain(context.TODO(), test.service) + + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + }) + } +} + +func Test_ensureCertificateForDomain(t *testing.T) { + tests := []struct { + name string + certService func() *kvCertService + serviceCert *godo.Certificate + serviceDomain *domain + expectedCert *godo.Certificate + err error + }{ + { + name: "existing certificate valid for domain", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{}, false) + return &certService + }, + serviceCert: &godo.Certificate{ + DNSNames: []string{"sample.digitalocean.com"}, + }, + serviceDomain: &domain{ + full: "sample.digitalocean.com", + }, + err: nil, + expectedCert: &godo.Certificate{ + DNSNames: []string{"sample.digitalocean.com"}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := newFakeClient(fakeClientOpts{ + fakeCert: test.certService(), + }) + fakeResources := newResources("", "", fakeClient) + fakeResources.kclient = fake.NewSimpleClientset() + + lb := &loadBalancers{ + resources: fakeResources, + region: "nyc1", + lbActiveTimeout: 2, + lbActiveCheckTick: 1, + } + + certificate, err := lb.ensureCertificateForDomain(context.TODO(), test.serviceCert, test.serviceDomain) + + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + + if !reflect.DeepEqual(certificate, test.expectedCert) { + t.Error("certificate does not match") + t.Logf("expected: %v", test.expectedCert) + t.Logf("actual: %v", certificate) + } + }) + } +} + +func Test_isValidCertificateForDomain(t *testing.T) { + tests := []struct { + name string + cert *godo.Certificate + domain *domain + isValid bool + }{ + { + name: "valid root domain with single dns entry in cert", + cert: &godo.Certificate{ + ID: "cert", + Name: "cert", + DNSNames: []string{"digitalocean.com"}, + }, + domain: &domain{ + full: "digitalocean.com", + }, + isValid: true, + }, + { + name: "valid root domain with multiple dns entries in cert", + cert: &godo.Certificate{ + ID: "cert", + Name: "cert", + DNSNames: []string{"subdomain.digitalocean.com", "digitalocean.com"}, + }, + domain: &domain{ + full: "digitalocean.com", + }, + isValid: true, + }, + { + name: "invalid subdomain with root dns entry in cert ", + cert: &godo.Certificate{ + ID: "cert", + Name: "cert", + DNSNames: []string{"digitalocean.com"}, + }, + domain: &domain{ + full: "subdomain.digitalocean.com", + }, + isValid: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := isValidCertificateForDomain(test.cert, test.domain) + + if actual != test.isValid { + t.Errorf("incorrect check of validity for certificate of domain") + t.Logf("domain %s and certificate dns %v", test.domain.full, test.cert.DNSNames) + t.Logf("expected: %t, actual: %t", test.isValid, actual) + } + }) + } +} + +func Test_findCertificateForDomain(t *testing.T) { + tests := []struct { + name string + certService func() *kvCertService + domain *domain + expectedCert *godo.Certificate + err error + }{ + { + name: "no certificate exists for domain", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{}, true) + return &certService + }, + domain: &domain{ + full: "digitalocean.com", + }, + expectedCert: nil, + err: nil, + }, + { + name: "certificate exists for domain", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{ + "cert": { + ID: "cert", + Name: "cert", + DNSNames: []string{"digitalocean.com"}, + }, + }, true) + return &certService + }, + domain: &domain{ + full: "digitalocean.com", + }, + expectedCert: &godo.Certificate{ + ID: "cert", + Name: "cert", + DNSNames: []string{"digitalocean.com"}, + }, + err: nil, + }, + { + name: "certificate exists for subdomain", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{ + "cert": { + ID: "cert", + Name: "cert", + DNSNames: []string{"digitalocean.com", "subdomain.digitalocean.com"}, + }, + }, true) + return &certService + }, + domain: &domain{ + full: "subdomain.digitalocean.com", + }, + expectedCert: &godo.Certificate{ + ID: "cert", + Name: "cert", + DNSNames: []string{"digitalocean.com", "subdomain.digitalocean.com"}, + }, + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := newFakeClient(fakeClientOpts{ + fakeCert: test.certService(), + }) + fakeResources := newResources("", "", fakeClient) + fakeResources.kclient = fake.NewSimpleClientset() + + lb := &loadBalancers{ + resources: fakeResources, + region: "nyc1", + lbActiveTimeout: 2, + lbActiveCheckTick: 1, + } + + certificate, err := lb.findCertificateForDomain(context.TODO(), test.domain) + + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + + if !reflect.DeepEqual(certificate, test.expectedCert) { + t.Error("certificate does not match") + t.Logf("expected: %v", test.expectedCert) + t.Logf("actual: %v", certificate) + } + }) + } +} + +func Test_generateCertificateForDomain(t *testing.T) { + tests := []struct { + name string + certService func() *kvCertService + domain *domain + expectedCert *godo.Certificate + err error + }{ + { + name: "root domain", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{}, false) + return &certService + }, + domain: &domain{ + full: "digitalocean.com", + root: "digitalocean.com", + }, + expectedCert: &godo.Certificate{ + ID: "do-ccm-digitalocean-com", + Name: "do-ccm-digitalocean-com", + Type: certTypeLetsEncrypt, + DNSNames: []string{"digitalocean.com"}, + }, + err: nil, + }, + { + name: "sub domain", + certService: func() *kvCertService { + certService := newKVCertService(map[string]*godo.Certificate{}, false) + return &certService + }, + domain: &domain{ + full: "subdomain.digitalocean.com", + root: "digitalocean.com", + sub: "subdomain", + }, + expectedCert: &godo.Certificate{ + ID: "do-ccm-subdomain-digitalocean-com", + Name: "do-ccm-subdomain-digitalocean-com", + Type: certTypeLetsEncrypt, + DNSNames: []string{"digitalocean.com", "subdomain.digitalocean.com"}, + }, + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := newFakeClient(fakeClientOpts{ + fakeCert: test.certService(), + }) + fakeResources := newResources("", "", fakeClient) + fakeResources.kclient = fake.NewSimpleClientset() + + lb := &loadBalancers{ + resources: fakeResources, + region: "nyc1", + lbActiveTimeout: 2, + lbActiveCheckTick: 1, + } + + certificate, err := lb.generateCertificateForDomain(context.TODO(), test.domain) + + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + + if !reflect.DeepEqual(certificate, test.expectedCert) { + t.Error("certificate does not match") + t.Logf("expected: %v", test.expectedCert) + t.Logf("actual: %v", certificate) + } + }) + } +} + +func Test_findARecordForNameAndIP(t *testing.T) { + tests := []struct { + name string + recordName string + recordIP string + records []godo.DomainRecord + err error + expectedRecord *godo.DomainRecord + }{ + { + name: "record name exists", + recordName: "@", + recordIP: "10.0.0.1", + records: []godo.DomainRecord{ + { + Type: "A", + Name: "@", + Data: "10.0.0.1", + }, + }, + err: nil, + expectedRecord: &godo.DomainRecord{ + Type: "A", + Name: "@", + Data: "10.0.0.1", + }, + }, + { + name: "record name exists but is of type TXT", + recordName: "@", + recordIP: "10.0.0.1", + records: []godo.DomainRecord{ + { + Type: "TXT", + Name: "@", + Data: "10.0.0.1", + }, + }, + err: nil, + expectedRecord: nil, + }, + { + name: "record name exists but is pointing to a different IP", + recordName: "@", + recordIP: "10.0.0.1", + records: []godo.DomainRecord{ + { + Type: "A", + Name: "@", + Data: "172.0.0.1", + }, + }, + err: errors.New("the A record(@) is already in use with another IP(172.0.0.1)"), + expectedRecord: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + record, err := findARecordForNameAndIP(test.records, test.recordName, test.recordIP) + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + + if !reflect.DeepEqual(record, test.expectedRecord) { + t.Error("record does not match") + t.Logf("expected: %v", test.expectedRecord) + t.Logf("actual: %v", record) + } + }) + } +} + +func Test_ensureDomainARecords(t *testing.T) { + tests := []struct { + name string + domainService func() *fakeDomainService + domain *domain + loadbalancer *godo.LoadBalancer + expectedRecords []godo.DomainRecord + err error + }{ + { + name: "service with root domain annotation and existing record for LB", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": { + 1: { + Type: "A", + ID: 1, + Name: "@", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + }, + }) + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + domain: &domain{ + root: "digitalocean.com", + }, + expectedRecords: []godo.DomainRecord{ + { + Type: "A", + ID: 1, + Name: "@", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + }, + err: nil, + }, + { + name: "service with root domain annotation and existing record pointing at unexpected LB", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": { + 1: { + Type: "A", + ID: 1, + Name: "@", + TTL: defaultDomainRecordTTL, + Data: "172.0.0.1", + }, + }, + }) + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + domain: &domain{ + root: "digitalocean.com", + }, + expectedRecords: []godo.DomainRecord{}, + err: errors.New("the A record(@) is already in use with another IP(172.0.0.1)"), + }, + { + name: "service with root domain annotation and no existing record", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": {}, + }) + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + domain: &domain{ + root: "digitalocean.com", + }, + expectedRecords: []godo.DomainRecord{ + { + Type: "A", + ID: 1, + Name: "@", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + }, + err: nil, + }, + { + name: "service with sub domain annotation and existing sub record for LB", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": { + 1: { + Type: "A", + ID: 1, + Name: "subdomain", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + }, + }) + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + domain: &domain{ + root: "digitalocean.com", + sub: "subdomain", + }, + expectedRecords: []godo.DomainRecord{ + { + Type: "A", + ID: 2, + Name: "@", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + { + Type: "A", + ID: 1, + Name: "subdomain", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + }, + err: nil, + }, + { + name: "service with sub domain annotation and existing sub record pointing at unexpected LB", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": { + 1: { + Type: "A", + ID: 1, + Name: "subdomain", + TTL: defaultDomainRecordTTL, + Data: "172.0.0.1", + }, + }, + }) + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + domain: &domain{ + root: "digitalocean.com", + sub: "subdomain", + }, + expectedRecords: []godo.DomainRecord{}, + err: errors.New("the A record(subdomain) is already in use with another IP(172.0.0.1)"), + }, + { + name: "service with sub domain annotation and no existing records", + domainService: func() *fakeDomainService { + domainService := newKVDomainService(map[string]*godo.Domain{ + "digitalocean.com": {}, + }, map[string]map[int]*godo.DomainRecord{ + "digitalocean.com": {}, + }) + return &domainService + }, + loadbalancer: createLBWithIP("10.0.0.1"), + domain: &domain{ + root: "digitalocean.com", + sub: "subdomain", + }, + expectedRecords: []godo.DomainRecord{ + { + Type: "A", + ID: 1, + Name: "@", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + { + Type: "A", + ID: 2, + Name: "subdomain", + TTL: defaultDomainRecordTTL, + Data: "10.0.0.1", + }, + }, + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := newFakeClient(fakeClientOpts{ + fakeDomain: test.domainService(), + }) + fakeResources := newResources("", "", fakeClient) + fakeResources.kclient = fake.NewSimpleClientset() + + lb := &loadBalancers{ + resources: fakeResources, + region: "nyc1", + lbActiveTimeout: 2, + lbActiveCheckTick: 1, + } + + err := lb.ensureDomainARecords(context.TODO(), test.domain, test.loadbalancer) + + if !reflect.DeepEqual(err, test.err) { + t.Error("unexpected error") + t.Logf("expected: %v", test.err) + t.Logf("actual: %v", err) + } + + if len(test.expectedRecords) > 0 { + for _, expected := range test.expectedRecords { + actual, _, _ := fakeClient.Domains.Record(context.TODO(), test.domain.root, expected.ID) + + if !reflect.DeepEqual(&expected, actual) { + t.Error("unexpected error") + t.Logf("expected: %v", expected) + t.Logf("expected: %v", actual) + } + } + } + }) + } +} + func Test_LBaaSCertificateScenarios(t *testing.T) { tests := []struct { name string diff --git a/cloud-controller-manager/do/common_test.go b/cloud-controller-manager/do/common_test.go index fbefc07eb..af5d2c704 100644 --- a/cloud-controller-manager/do/common_test.go +++ b/cloud-controller-manager/do/common_test.go @@ -33,6 +33,7 @@ type fakeClientOpts struct { fakeDroplet *fakeDropletService fakeLB *fakeLBService fakeCert *kvCertService + fakeDomain *fakeDomainService } func newFakeClient(opts fakeClientOpts) *godo.Client { @@ -40,6 +41,7 @@ func newFakeClient(opts fakeClientOpts) *godo.Client { Certificates: opts.fakeCert, Droplets: opts.fakeDroplet, LoadBalancers: opts.fakeLB, + Domains: opts.fakeDomain, } } @@ -86,6 +88,10 @@ func newFakeBadRequestResponse() *godo.Response { return newFakeResponse(http.StatusBadRequest) } +func newFakeBadRequestErrorResponse() *godo.ErrorResponse { + return newFakeErrorResponse(http.StatusBadRequest) +} + func linksForPage(page int) *godo.Links { switch page { case 0, 1: diff --git a/cloud-controller-manager/do/loadbalancers.go b/cloud-controller-manager/do/loadbalancers.go index 998519ec0..1fba01780 100644 --- a/cloud-controller-manager/do/loadbalancers.go +++ b/cloud-controller-manager/do/loadbalancers.go @@ -162,6 +162,8 @@ const ( portProtocolTCP = "TCP" defaultSecurePort = 443 + + defaultDomainRecordTTL = 3600 ) var errLBNotFound = errors.New("loadbalancer not found") @@ -256,6 +258,28 @@ func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri patcher := newServicePatcher(l.resources.kclient, service) defer func() { err = patcher.Patch(err) }() + serviceCertificate, err := l.validateServiceCertificate(ctx, service) + if err != nil { + return nil, err + } + + // If a domain annotation exists, verify that a corresponding DO domain exists for the root domain + var domain *domain + domain, err = l.ensureDomain(ctx, service) + if err != nil { + return nil, err + } + + // If a domain annotation exists, we validate the current serviceCertificate is valid or generate one if + // it does not exist or is not valid for the specified domain. + if domain != nil { + serviceCertificate, err = l.ensureCertificateForDomain(ctx, serviceCertificate, domain) + if err != nil { + return nil, err + } + updateServiceAnnotation(service, annDOCertificateID, serviceCertificate.ID) + } + var lbRequest *godo.LoadBalancerRequest lbRequest, err = l.buildLoadBalancerRequest(ctx, service, nodes) if err != nil { @@ -290,6 +314,16 @@ func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri return nil, fmt.Errorf("load-balancer is not yet active (current status: %s)", lb.Status) } + // If a domain exists, ensure that the domain has an A record pointing to the LB IP + if domain != nil { + err = l.ensureDomainARecords(ctx, domain, lb) + if err != nil { + return nil, err + } + + updateServiceAnnotation(service, annDOHostname, domain.full) + } + // If a LB hostname annotation is specified, return with it instead of the IP. hostname := getHostname(service) if hostname != "" { @@ -324,35 +358,25 @@ func getCertificateIDFromLB(lb *godo.LoadBalancer) string { // lets_encrypt type certificate associated with a Service that the certificate // annotation on the Service gets newly-updated certificate ID from the // Load Balancer. -func (l *loadBalancers) recordUpdatedLetsEncryptCert(ctx context.Context, service *v1.Service, lbCertID, serviceCertID string) error { - if lbCertID != "" && lbCertID != serviceCertID { - if serviceCertID != "" { - svcCert, _, err := l.resources.gclient.Certificates.Get(ctx, serviceCertID) - if err != nil { - respErr, ok := err.(*godo.ErrorResponse) - if !ok || respErr.Response.StatusCode != http.StatusNotFound { - return fmt.Errorf("failed to get DO certificate for service: %s", err) - } - } +func (l *loadBalancers) recordUpdatedLetsEncryptCert(ctx context.Context, service *v1.Service, lbCertID string) error { + certificate, err := l.validateServiceCertificate(ctx, service) + if err != nil { + return err + } - // The given certificate on the service exists, pass through so the LB is updated - if svcCert != nil { - return nil - } - } + // If the existing service certificate is valid, do not attempt to overwrite + // the existing LB certificate. We might be attempting to change the certificate. + if certificate != nil || lbCertID == "" { + return nil + } - lbCert, _, err := l.resources.gclient.Certificates.Get(ctx, lbCertID) - if err != nil { - respErr, ok := err.(*godo.ErrorResponse) - if ok && respErr.Response.StatusCode == http.StatusNotFound { - return nil - } - return fmt.Errorf("failed to get DO certificate for load-balancer: %s", err) - } + certificate, err = l.validateCertificateExistence(ctx, lbCertID) + if err != nil { + return err + } - if lbCert.Type == certTypeLetsEncrypt { - updateServiceAnnotation(service, annDOCertificateID, lbCertID) - } + if certificate != nil && certificate.Type == certTypeLetsEncrypt { + updateServiceAnnotation(service, annDOCertificateID, lbCertID) } return nil @@ -368,8 +392,7 @@ func (l *loadBalancers) updateLoadBalancer(ctx context.Context, service *v1.Serv } lbCertID := getCertificateIDFromLB(lb) - serviceCertID := getCertificateID(service) - err = l.recordUpdatedLetsEncryptCert(ctx, service, lbCertID, serviceCertID) + err = l.recordUpdatedLetsEncryptCert(ctx, service, lbCertID) if err != nil { return nil, err } @@ -422,6 +445,7 @@ func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa return err } + klog.V(2).Infof("Deleting load-balancer %s for service %s/%s", lb.ID, service.Namespace, service.Name) resp, err := l.resources.gclient.LoadBalancers.Delete(ctx, lb.ID) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { @@ -430,6 +454,66 @@ func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa return fmt.Errorf("failed to delete load-balancer: %s", err) } + domain, err := getDomain(service) + if err != nil { + klog.Errorf("Failed to parse domain from service %s/%s: %s", service.Namespace, service.Name, err) + return nil + } else if domain == nil { + return nil + } + + // Attempt to clean up any managed A records or certificates related to LB + certID := getCertificateIDFromLB(lb) + cert, resp, err := l.resources.gclient.Certificates.Get(ctx, certID) + if err != nil && resp.StatusCode != http.StatusNotFound { + // fall-through so that we attempt to clean up A records on domain later on + klog.Errorf("Failed to fetch certificate for service %s/%s: %s", service.Namespace, service.Name, err) + } + + // If this certificate name matches the autogenerated cert format then cleanup + if cert != nil && cert.Name == getCertificateName(domain.full) { + klog.V(2).Infof("Deleting certificate %s for service %s/%s", certID, service.Namespace, service.Name) + + resp, err := l.resources.gclient.Certificates.Delete(ctx, certID) + if err != nil && resp.StatusCode != http.StatusNotFound { + // fall-through so that we attempt to clean up A records on domain later on + klog.Errorf("Failed to delete certificate %s for service %s/%s during cleanup: %s", certID, service.Namespace, service.Name, err) + } + } + + records, _, err := l.resources.gclient.Domains.Records(ctx, domain.root, &godo.ListOptions{}) + if err != nil { + klog.Errorf("Failed to fetch records for domain %s of service %s/%s: %s", domain.root, service.Namespace, service.Name, err) + return nil + } + + // Attempt to clean up root domain record + record, err := findARecordForNameAndIP(records, "@", lb.IP) + if err != nil { + klog.Errorf("Failed to find A-record(%s) with IP %s for service %s/%s: %s", "@", lb.IP, service.Namespace, service.Name, err) + } + if record != nil { + klog.V(2).Infof("Deleting A-record %s with IP %s for service %s/%s", "@", lb.IP, service.Namespace, service.Name) + _, err := l.resources.gclient.Domains.DeleteRecord(ctx, domain.root, record.ID) + if err != nil { + klog.Errorf("Failed to delete A-record(%s) with IP %s for service %s/%s: %s", "@", lb.IP, service.Namespace, service.Name, err) + } + } + + if domain.sub != "" { + record, err := findARecordForNameAndIP(records, domain.sub, lb.IP) + if err != nil { + klog.Errorf("Failed to find A-record(%s) with IP %s for service %s/%s: %s", domain.sub, lb.IP, service.Namespace, service.Name, err) + } + if record != nil { + klog.V(2).Infof("Deleting A-record %s with IP %s for service %s/%s", domain.sub, lb.IP, service.Namespace, service.Name) + _, err := l.resources.gclient.Domains.DeleteRecord(ctx, domain.root, record.ID) + if err != nil { + klog.Errorf("Failed to delete A-record(%s) with IP %s for service %s/%s: %s", domain.sub, lb.IP, service.Namespace, service.Name, err) + } + } + } + return nil } diff --git a/cloud-controller-manager/do/loadbalancers_test.go b/cloud-controller-manager/do/loadbalancers_test.go index 520185bab..8a81c4622 100644 --- a/cloud-controller-manager/do/loadbalancers_test.go +++ b/cloud-controller-manager/do/loadbalancers_test.go @@ -134,6 +134,17 @@ func createLB() *godo.LoadBalancer { } } +func createLBWithIP(ip string) *godo.LoadBalancer { + return &godo.LoadBalancer{ + // loadbalancer names are a + service.UID + // see cloudprovider.DefaultLoadBalancerName + ID: "load-balancer-id", + Name: "afoobar123", + IP: ip, + Status: lbStatusActive, + } +} + func createHTTPSLB(lbID, certID, certType string) (*godo.LoadBalancer, *godo.Certificate) { lb := &godo.LoadBalancer{ // loadbalancer names are a + service.UID @@ -3593,10 +3604,9 @@ func Test_EnsureLoadBalancer(t *testing.T) { testcases := []struct { name string droplets []godo.Droplet - getFn func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) - listFn func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) - createFn func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) - updateFn func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) + fakeLBService *fakeLBService + fakeCertService *kvCertService + fakeDomainService *fakeDomainService service *v1.Service newLoadBalancerID string nodes []*v1.Node @@ -3619,17 +3629,19 @@ func Test_EnsureLoadBalancer(t *testing.T) { Name: "node-3", }, }, - getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("get should not have been invoked") - }, - listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { - return []godo.LoadBalancer{*createLB()}, newFakeOKResponse(), nil - }, - createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") - }, - updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - return createLB(), newFakeOKResponse(), nil + fakeLBService: &fakeLBService{ + getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("get should not have been invoked") + }, + listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { + return []godo.LoadBalancer{*createLB()}, newFakeOKResponse(), nil + }, + createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") + }, + updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, }, service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -3692,17 +3704,19 @@ func Test_EnsureLoadBalancer(t *testing.T) { Name: "node-3", }, }, - getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { - return createLB(), newFakeOKResponse(), nil - }, - listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("list should not have been invoked") - }, - createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") - }, - updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - return createLB(), newFakeOKResponse(), nil + fakeLBService: &fakeLBService{ + getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, + listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("list should not have been invoked") + }, + createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") + }, + updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, }, service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -3750,6 +3764,231 @@ func Test_EnsureLoadBalancer(t *testing.T) { }, err: nil, }, + { + name: "failed to ensure loadbalancer with domain annotation, root domain does not", + droplets: []godo.Droplet{ + { + ID: 100, + Name: "node-1", + }, + }, + fakeDomainService: &fakeDomainService{ + getFunc: func(context.Context, string) (*godo.Domain, *godo.Response, error) { + return nil, newFakeNotFoundResponse(), errors.New("404") + }, + }, + fakeLBService: &fakeLBService{ + getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, + listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("list should not have been invoked") + }, + createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") + }, + updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, + }, + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: "foobar123", + Annotations: map[string]string{ + annDOProtocol: "http", + annoDOLoadBalancerID: "load-balancer-id", + annDODomain: "sample.digitalocean.com", + }, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: "test", + Protocol: "TCP", + Port: int32(80), + NodePort: int32(30000), + }, + }, + }, + }, + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + }, + }, + lbStatus: nil, + err: utilerrors.NewAggregate([]error{errors.New("failed to retrieve root domain digitalocean.com: 404")}), + }, + { + name: "failed to ensure loadbalancer with domain annotation, domain exists but already has A records pointing to another LB", + droplets: []godo.Droplet{ + { + ID: 100, + Name: "node-1", + }, + }, + fakeDomainService: &fakeDomainService{ + getFunc: func(context.Context, string) (*godo.Domain, *godo.Response, error) { + return &godo.Domain{ + Name: "digitalocean.com", + }, newFakeOKResponse(), nil + }, + recordsFunc: func(context.Context, string, *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) { + return []godo.DomainRecord{ + { + Type: "A", + Name: "@", + Data: "not-the-lb", + }, + }, newFakeOKResponse(), nil + }, + }, + fakeCertService: &kvCertService{ + store: map[string]*godo.Certificate{ + "cert": { + Name: "cert", + Type: certTypeLetsEncrypt, + DNSNames: []string{"sample.digitalocean.com", "digitalocean.com"}, + }, + }, + }, + fakeLBService: &fakeLBService{ + getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, + listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("list should not have been invoked") + }, + createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") + }, + updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, + }, + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: "foobar123", + Annotations: map[string]string{ + annDOProtocol: "http", + annoDOLoadBalancerID: "load-balancer-id", + annDODomain: "sample.digitalocean.com", + }, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: "test", + Protocol: "TCP", + Port: int32(80), + NodePort: int32(30000), + }, + }, + }, + }, + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + }, + }, + lbStatus: nil, + err: utilerrors.NewAggregate([]error{errors.New("the A record(@) is already in use with another IP(not-the-lb)")}), + }, + { + name: "successfully ensure loadbalancer with domain annotation, certificate and A records exist", + droplets: []godo.Droplet{ + { + ID: 100, + Name: "node-1", + }, + }, + fakeDomainService: &fakeDomainService{ + getFunc: func(context.Context, string) (*godo.Domain, *godo.Response, error) { + return &godo.Domain{ + Name: "digitalocean.com", + }, newFakeOKResponse(), nil + }, + recordsFunc: func(context.Context, string, *godo.ListOptions) ([]godo.DomainRecord, *godo.Response, error) { + return []godo.DomainRecord{ + { + Type: "A", + Name: "@", + Data: "10.0.0.1", + }, + { + Type: "A", + Name: "sample", + Data: "10.0.0.1", + }, + }, newFakeOKResponse(), nil + }, + }, + fakeCertService: &kvCertService{ + store: map[string]*godo.Certificate{ + "cert": { + Name: "cert", + Type: certTypeLetsEncrypt, + DNSNames: []string{"sample.digitalocean.com", "digitalocean.com"}, + }, + }, + }, + fakeLBService: &fakeLBService{ + getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, + listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("list should not have been invoked") + }, + createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") + }, + updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, + }, + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: "foobar123", + Annotations: map[string]string{ + annDOProtocol: "http", + annoDOLoadBalancerID: "load-balancer-id", + annDODomain: "sample.digitalocean.com", + }, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: "test", + Protocol: "TCP", + Port: int32(80), + NodePort: int32(30000), + }, + }, + }, + }, + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + }, + }, + lbStatus: &v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + Hostname: "sample.digitalocean.com", + }, + }, + }, + err: nil, + }, { name: "successfully ensured loadbalancer by name that didn't exist", droplets: []godo.Droplet{ @@ -3766,17 +4005,19 @@ func Test_EnsureLoadBalancer(t *testing.T) { Name: "node-3", }, }, - getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("get should not have been invoked") - }, - listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { - return []godo.LoadBalancer{}, newFakeOKResponse(), nil - }, - createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - return createLB(), newFakeOKResponse(), nil - }, - updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("update should not have been invoked") + fakeLBService: &fakeLBService{ + getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("get should not have been invoked") + }, + listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { + return []godo.LoadBalancer{}, newFakeOKResponse(), nil + }, + createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return createLB(), newFakeOKResponse(), nil + }, + updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("update should not have been invoked") + }, }, service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -3839,19 +4080,21 @@ func Test_EnsureLoadBalancer(t *testing.T) { Name: "node-3", }, }, - getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeResponse(http.StatusNotFound), errors.New("LB not found") - }, - listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("list should not have been invoked") - }, - createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - lb := createLB() - lb.ID = "other-load-balancer-id" - return lb, newFakeOKResponse(), nil - }, - updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("update should not have been invoked") + fakeLBService: &fakeLBService{ + getFn: func(context.Context, string) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeResponse(http.StatusNotFound), errors.New("LB not found") + }, + listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("list should not have been invoked") + }, + createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + lb := createLB() + lb.ID = "other-load-balancer-id" + return lb, newFakeOKResponse(), nil + }, + updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("update should not have been invoked") + }, }, service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -3903,16 +4146,18 @@ func Test_EnsureLoadBalancer(t *testing.T) { { name: "failed to ensure existing load-balancer, state is non-active", droplets: []godo.Droplet{}, - listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { - return []godo.LoadBalancer{*createLB()}, newFakeOKResponse(), nil - }, - createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") - }, - updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { - lb := createLB() - lb.Status = lbStatusNew - return lb, newFakeOKResponse(), nil + fakeLBService: &fakeLBService{ + listFn: func(context.Context, *godo.ListOptions) ([]godo.LoadBalancer, *godo.Response, error) { + return []godo.LoadBalancer{*createLB()}, newFakeOKResponse(), nil + }, + createFn: func(context.Context, *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + return nil, newFakeNotOKResponse(), errors.New("create should not have been invoked") + }, + updateFn: func(ctx context.Context, lbID string, lbr *godo.LoadBalancerRequest) (*godo.LoadBalancer, *godo.Response, error) { + lb := createLB() + lb.Status = lbStatusNew + return lb, newFakeOKResponse(), nil + }, }, service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -3962,18 +4207,15 @@ func Test_EnsureLoadBalancer(t *testing.T) { return test.droplets, newFakeOKResponse(), nil }, } - fakeLB := &fakeLBService{ - getFn: test.getFn, - listFn: test.listFn, - createFn: test.createFn, - updateFn: test.updateFn, - } - certStore := make(map[string]*godo.Certificate) - fakeCert := newKVCertService(certStore, true) + /* + certStore := make(map[string]*godo.Certificate) + fakeCert := newKVCertService(certStore, true) + */ fakeClient := newFakeClient(fakeClientOpts{ fakeDroplet: fakeDroplet, - fakeLB: fakeLB, - fakeCert: &fakeCert, + fakeLB: test.fakeLBService, + fakeCert: test.fakeCertService, + fakeDomain: test.fakeDomainService, }) fakeResources := newResources("", "", fakeClient) fakeResources.kclient = fake.NewSimpleClientset()