Skip to content

Commit

Permalink
Adds initial implementation of TLSReloadTransport
Browse files Browse the repository at this point in the history
  • Loading branch information
bonzofenix committed Dec 18, 2024
1 parent 67b868d commit bf9e05e
Show file tree
Hide file tree
Showing 5 changed files with 195 additions and 35 deletions.
1 change: 0 additions & 1 deletion src/autoscaler/api/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,6 @@ func loadVcapConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader)
func configureEventGenerator(conf *Config) {
conf.EventGenerator.TLSClientCerts.CertFile = os.Getenv("CF_INSTANCE_CERT")
conf.EventGenerator.TLSClientCerts.KeyFile = os.Getenv("CF_INSTANCE_KEY")

}

func configurePolicyDb(conf *Config, vcapReader configutil.VCAPConfigurationReader) error {
Expand Down
3 changes: 3 additions & 0 deletions src/autoscaler/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ require (
github.com/jmoiron/sqlx v1.4.0
github.com/maxbrunsfeld/counterfeiter/v6 v6.9.0
github.com/ogen-go/ogen v1.8.0
github.com/onsi/ginkgo v1.16.5
github.com/onsi/ginkgo/v2 v2.22.0
github.com/onsi/gomega v1.36.0
github.com/patrickmn/go-cache v2.1.0+incompatible
Expand Down Expand Up @@ -82,6 +83,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/openzipkin/zipkin-go v0.4.3 // indirect
github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand All @@ -103,5 +105,6 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect
google.golang.org/protobuf v1.35.2 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
68 changes: 38 additions & 30 deletions src/autoscaler/helpers/httpclient.go
Original file line number Diff line number Diff line change
@@ -1,37 +1,53 @@
package helpers

import (
"encoding/base64"
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"
"time"

"code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf"
"code.cloudfoundry.org/lager/v3"
"github.com/hashicorp/go-retryablehttp"

"code.cloudfoundry.org/app-autoscaler/src/autoscaler/models"

"code.cloudfoundry.org/cfhttp/v2"
)

type TransportWithBasicAuth struct {
Username string
Password string
type TLSReloadTransport struct {
Base http.RoundTripper
logger lager.Logger
tlsCerts *models.TLSCerts
}

func (t *TransportWithBasicAuth) base() http.RoundTripper {
if t.Base != nil {
return t.Base
func (t *TLSReloadTransport) tlsClientConfig() *tls.Config {
return t.Base.(*retryablehttp.RoundTripper).Client.HTTPClient.Transport.(*http.Transport).TLSClientConfig
}

func (t *TLSReloadTransport) setTLSClientConfig(tlsConfig *tls.Config) {
t.Base.(*retryablehttp.RoundTripper).Client.HTTPClient.Transport.(*http.Transport).TLSClientConfig = tlsConfig
}

func (t *TLSReloadTransport) certificateExpiringWithin(dur time.Duration) bool {
x509Cert, err := x509.ParseCertificate(t.tlsClientConfig().Certificates[0].Certificate[0])
if err != nil {
return false
}
return http.DefaultTransport

return x509Cert.NotAfter.Sub(time.Now()) < dur
}
func (t *TLSReloadTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if t.certificateExpiringWithin(time.Hour) {
t.logger.Info("reloading-cert", lager.Data{"request": req})
tlsConfig, _ := t.tlsCerts.CreateClientConfig()
t.setTLSClientConfig(tlsConfig)
} else {
t.logger.Info("cert-not-expiring", lager.Data{"request": req})
}

func (t *TransportWithBasicAuth) RoundTrip(req *http.Request) (*http.Response, error) {
credentials := t.Username + ":" + t.Password
basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(credentials))
req.Header.Add("Authorization", basicAuth)
return t.base().RoundTrip(req)
return t.Base.RoundTrip(req)
}

func DefaultClientConfig() cf.ClientConfig {
Expand All @@ -41,22 +57,6 @@ func DefaultClientConfig() cf.ClientConfig {
}
}

func CreateHTTPClient(ba *models.BasicAuth, config cf.ClientConfig, logger lager.Logger) (*http.Client, error) {
client := cfhttp.NewClient(
cfhttp.WithDialTimeout(30*time.Second),
cfhttp.WithIdleConnTimeout(time.Duration(config.IdleConnectionTimeoutMs)*time.Millisecond),
cfhttp.WithMaxIdleConnsPerHost(config.MaxIdleConnsPerHost),
)

client = cf.RetryClient(config, client, logger)
client.Transport = &TransportWithBasicAuth{
Username: ba.Username,
Password: ba.Password,
}

return client, nil
}

func CreateHTTPSClient(tlsCerts *models.TLSCerts, config cf.ClientConfig, logger lager.Logger) (*http.Client, error) {
tlsConfig, err := tlsCerts.CreateClientConfig()
if err != nil {
Expand All @@ -70,5 +70,13 @@ func CreateHTTPSClient(tlsCerts *models.TLSCerts, config cf.ClientConfig, logger
cfhttp.WithMaxIdleConnsPerHost(config.MaxIdleConnsPerHost),
)

return cf.RetryClient(config, client, logger), nil
retryClient := cf.RetryClient(config, client, logger)

retryClient.Transport = &TLSReloadTransport{
Base: retryClient.Transport,
logger: logger,
tlsCerts: tlsCerts,
}

return retryClient, nil
}
143 changes: 143 additions & 0 deletions src/autoscaler/helpers/httpclient_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
package helpers_test

import (
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"log"
"net/http"
"os"
"time"

"code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil"
"code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers"
"code.cloudfoundry.org/app-autoscaler/src/autoscaler/models"
"code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers"
"code.cloudfoundry.org/lager/v3/lagertest"
"github.com/hashicorp/go-retryablehttp"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"github.com/onsi/gomega/ghttp"
)

var _ = Describe("HTTPClient", func() {
var (
fakeServer *ghttp.Server
client *http.Client
logger *lagertest.TestLogger
err error
)

BeforeEach(func() {
fakeServer = ghttp.NewServer()
fakeServer.RouteToHandler("GET", "/", ghttp.RespondWith(http.StatusOK, "successful"))
})

Describe("CreateHTTPSClient", func() {
var (
cfInstanceCertFile string
cfInstanceKeyFile string
cfInstanceCertContent []byte
cfInstanceKeyContent []byte
notAfter time.Time
certTmpDir string
privateKey *rsa.PrivateKey
)

JustBeforeEach(func() {
privateKey, err = rsa.GenerateKey(rand.Reader, 2048)
Expect(err).ToNot(HaveOccurred())

cfInstanceCertContent, err = testhelpers.GenerateClientCertWithPrivateKeyExpiring("org", "space", privateKey, notAfter)
certTmpDir = os.TempDir()
cfInstanceKeyContent = testhelpers.GenerateClientKeyWithPrivateKey(privateKey)

cfInstanceCertFile, err = configutil.MaterializeContentInFile(certTmpDir, "eventgenerator.crt", string(cfInstanceCertContent))
Expect(err).NotTo(HaveOccurred())

cfInstanceKeyFile, err = configutil.MaterializeContentInFile(certTmpDir, "eventgenerator.key", string(cfInstanceKeyContent))
Expect(err).NotTo(HaveOccurred())

logger = lagertest.NewTestLogger("http-client-test")

tlsCerts := &models.TLSCerts{
KeyFile: cfInstanceKeyFile,
CertFile: cfInstanceCertFile,
CACertFile: cfInstanceCertFile,
}

client, err = helpers.CreateHTTPSClient(tlsCerts, helpers.DefaultClientConfig(), logger)
Expect(err).ToNot(HaveOccurred())
})

AfterEach(func() {
os.Remove(cfInstanceCertFile)
os.Remove(cfInstanceKeyFile)
})

When("Cert cert is not within 1 hour of expiration", func() {
BeforeEach(func() {
notAfter = time.Now().Add(63 * time.Minute)
})

It("should reload the cert", func() {
Expect(client).ToNot(BeNil())
client.Get(fakeServer.URL())
Expect(logger).To(gbytes.Say("cert-not-expiring"))
})
})

When("Cert cert is within 1 hour of expiration", func() {
var cfInstanceCertFileToRotateContent []byte

BeforeEach(func() {
notAfter = time.Now().Add(59 * time.Minute)
})

It("should reload the cert", func() {
cfInstanceCertFileToRotateContent, err = testhelpers.GenerateClientCertWithPrivateKey("org", "space", privateKey)
Expect(err).ToNot(HaveOccurred())

By("Rotating with unexpired one")
_, err = configutil.MaterializeContentInFile(certTmpDir, "eventgenerator.crt", string(cfInstanceCertFileToRotateContent))
Expect(err).NotTo(HaveOccurred())

Expect(getCertFromClient(client)).To(Equal(string(cfInstanceCertContent)))
client.Get(fakeServer.URL())
Expect(logger).To(gbytes.Say("reloading-cert"))

Expect(getCertFromClient(client)).To(Equal(string(cfInstanceCertFileToRotateContent)))
})
})
})
})

func getCertFromClient(client *http.Client) string {
GinkgoHelper()
cert := client.Transport.(*helpers.TLSReloadTransport).Base.(*retryablehttp.RoundTripper).Client.HTTPClient.Transport.(*http.Transport).TLSClientConfig.Certificates[0]
return getPEM(cert)
}

func getPEM(cert tls.Certificate) string {
result := ""

for _, certBytes := range cert.Certificate {
parsedCert, err := x509.ParseCertificate(certBytes)
if err != nil {
log.Printf("Failed to parse certificate: %v", err)
continue
}

// Encode to PEM format
pemBlock := &pem.Block{
Type: "CERTIFICATE",
Bytes: parsedCert.Raw,
}
result += string(pem.EncodeToMemory(pemBlock))
}

return result
}
15 changes: 11 additions & 4 deletions src/autoscaler/testhelpers/certs.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@ import (
"code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/auth"
)

// generateClientCert generates a client certificate with the specified spaceGUID and orgGUID
// included in the organizational unit string.
func GenerateClientCertWithPrivateKey(orgGUID, spaceGUID string, privateKey *rsa.PrivateKey) ([]byte, error) {
func GenerateClientCertWithPrivateKeyExpiring(orgGUID, spaceGUID string, privateKey *rsa.PrivateKey, notAfter time.Time) ([]byte, error) {
serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
if err != nil {
return nil, err
Expand All @@ -25,13 +23,17 @@ func GenerateClientCertWithPrivateKey(orgGUID, spaceGUID string, privateKey *rsa
template := x509.Certificate{
SerialNumber: serialNumber,
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(1, 0, 0),
NotAfter: notAfter,
Subject: pkix.Name{
Organization: []string{"My Organization"},
OrganizationalUnit: []string{fmt.Sprintf("space:%s org:%s", spaceGUID, orgGUID)},
},
}

if privateKey == nil {
return nil, fmt.Errorf("private key is nil")
}

certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
if err != nil {
return nil, err
Expand All @@ -42,6 +44,11 @@ func GenerateClientCertWithPrivateKey(orgGUID, spaceGUID string, privateKey *rsa
return certPEM, nil
}

func GenerateClientCertWithPrivateKey(orgGUID, spaceGUID string, privateKey *rsa.PrivateKey) ([]byte, error) {
notAfter := time.Now().AddDate(1, 0, 0)
return GenerateClientCertWithPrivateKeyExpiring(orgGUID, spaceGUID, privateKey, notAfter)
}

func GenerateClientCert(orgGUID, spaceGUID string) ([]byte, error) {
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
Expand Down

0 comments on commit bf9e05e

Please sign in to comment.