From f19a70b9cb402b0d25c773c5150df75dd92a28ed Mon Sep 17 00:00:00 2001 From: Jan Bouska Date: Fri, 4 Oct 2024 15:04:54 +0200 Subject: [PATCH] Implement Fulcio cert rotation scenario --- .github/workflows/main.yml | 34 +++- go.mod | 3 + go.sum | 9 + test/e2e/fulcio_key_rotation_test.go | 280 +++++++++++++++++++++++++++ test/e2e/support/archive.go | 117 +++++++++++ test/e2e/support/kubernetes/cp.go | 152 +++++++++++++++ test/e2e/support/tas/cli/command.go | 8 + 7 files changed, 602 insertions(+), 1 deletion(-) create mode 100644 test/e2e/fulcio_key_rotation_test.go create mode 100644 test/e2e/support/kubernetes/cp.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5283fc147..6b5cdc1ea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,11 +89,36 @@ jobs: ${{ env.OPM }} validate v4.14/catalog/rhtas-operator docker build v4.14 -f v4.14/catalog.Dockerfile -t $CATALOG_IMG docker push $CATALOG_IMG + build-tuftool: + name: Build-tuftool + runs-on: ubuntu-20.04 + steps: + - name: Checkout tough source + uses: actions/checkout@v4 + with: + repository: "securesign/tough" + path: tough + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + - run: cd tough && cargo build --release --target-dir /tmp/tuftool + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: tuftool + path: /tmp/tuftool/release/tuftool + retention-days: 1 + if-no-files-found: error + test-kind: name: Test kind deployment runs-on: ubuntu-20.04 - needs: build-operator + needs: + - build-operator + - build-tuftool steps: - name: Checkout source uses: actions/checkout@v4 @@ -103,6 +128,13 @@ jobs: with: go-version: ${{ env.GO_VERSION }} + - name: Download tuftool + uses: actions/download-artifact@v4 + with: + name: tuftool + path: /tmp/tuftool + - run: echo "/tmp/tuftool" >> $GITHUB_PATH + - name: Log in to registry.redhat.io uses: redhat-actions/podman-login@9184318aae1ee5034fbfbacc0388acf12669171f # v1 with: diff --git a/go.mod b/go.mod index b8df9ebb8..dd3a81ce7 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -64,9 +65,11 @@ require ( github.com/letsencrypt/boulder v0.0.0-20230907030200-6d76a0f91e1e // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index 30852bee6..6b30aec9c 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -66,6 +68,9 @@ github.com/google/trillian v1.6.0 h1:jMBeDBIkINFvS2n6oV5maDqfRlxREAc6CW9QYWQ0qT4 github.com/google/trillian v1.6.0/go.mod h1:Yu3nIMITzNhhMJEHjAtp6xKiu+H/iHu2Oq5FjV2mCWI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= @@ -90,6 +95,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -97,6 +104,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= diff --git a/test/e2e/fulcio_key_rotation_test.go b/test/e2e/fulcio_key_rotation_test.go new file mode 100644 index 000000000..dbba5458e --- /dev/null +++ b/test/e2e/fulcio_key_rotation_test.go @@ -0,0 +1,280 @@ +//go:build integration + +package e2e + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/securesign/operator/api/v1alpha1" + "github.com/securesign/operator/internal/controller/common/utils" + "github.com/securesign/operator/internal/controller/common/utils/kubernetes" + "github.com/securesign/operator/internal/controller/constants" + tufAction "github.com/securesign/operator/internal/controller/tuf/actions" + "github.com/securesign/operator/test/e2e/support" + kubernetes2 "github.com/securesign/operator/test/e2e/support/kubernetes" + "github.com/securesign/operator/test/e2e/support/tas" + clients "github.com/securesign/operator/test/e2e/support/tas/cli" + "github.com/securesign/operator/test/e2e/support/tas/fulcio" + "github.com/securesign/operator/test/e2e/support/tas/securesign" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + runtimeCli "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +var _ = Describe("Fulcio cert rotation test", Ordered, func() { + cli, _ := support.CreateClient() + ctx := context.TODO() + var ( + targetImageName string + namespace *v1.Namespace + s *v1alpha1.Securesign + oldCert []byte + newCert *v1.Secret + err error + ) + + AfterEach(func() { + if CurrentSpecReport().Failed() && support.IsCIEnvironment() { + support.DumpNamespace(ctx, cli, namespace.Name) + } + }) + + BeforeAll(func() { + if _, err := exec.LookPath("tuftool"); err != nil { + Skip("tuftool command not found") + } + namespace = support.CreateTestNamespace(ctx, cli) + DeferCleanup(func() { + _ = cli.Delete(ctx, namespace) + }) + + s = &v1alpha1.Securesign{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + Name: "test", + Annotations: map[string]string{ + "rhtas.redhat.com/metrics": "false", + }, + }, + Spec: v1alpha1.SecuresignSpec{ + Rekor: v1alpha1.RekorSpec{ + ExternalAccess: v1alpha1.ExternalAccess{ + Enabled: true, + }, + RekorSearchUI: v1alpha1.RekorSearchUI{ + Enabled: utils.Pointer(true), + }, + }, + Fulcio: v1alpha1.FulcioSpec{ + ExternalAccess: v1alpha1.ExternalAccess{ + Enabled: true, + }, + Config: v1alpha1.FulcioConfig{ + OIDCIssuers: []v1alpha1.OIDCIssuer{ + { + ClientID: support.OidcClientID(), + IssuerURL: support.OidcIssuerUrl(), + Issuer: support.OidcIssuerUrl(), + Type: "email", + }, + }}, + Certificate: v1alpha1.FulcioCert{ + OrganizationName: "MyOrg", + OrganizationEmail: "my@email.org", + CommonName: "fulcio", + }, + }, + Ctlog: v1alpha1.CTlogSpec{}, + Tuf: v1alpha1.TufSpec{ + ExternalAccess: v1alpha1.ExternalAccess{ + Enabled: true, + }, + }, + Trillian: v1alpha1.TrillianSpec{Db: v1alpha1.TrillianDB{ + Create: ptr.To(true), + }}, + TimestampAuthority: &v1alpha1.TimestampAuthoritySpec{ + ExternalAccess: v1alpha1.ExternalAccess{ + Enabled: true, + }, + Signer: v1alpha1.TimestampAuthoritySigner{ + CertificateChain: v1alpha1.CertificateChain{ + RootCA: &v1alpha1.TsaCertificateAuthority{ + OrganizationName: "MyOrg", + OrganizationEmail: "my@email.org", + CommonName: "tsa.hostname", + }, + IntermediateCA: []*v1alpha1.TsaCertificateAuthority{ + { + OrganizationName: "MyOrg", + OrganizationEmail: "my@email.org", + CommonName: "tsa.hostname", + }, + }, + LeafCA: &v1alpha1.TsaCertificateAuthority{ + OrganizationName: "MyOrg", + OrganizationEmail: "my@email.org", + CommonName: "tsa.hostname", + }, + }, + }, + NTPMonitoring: v1alpha1.NTPMonitoring{ + Enabled: true, + Config: &v1alpha1.NtpMonitoringConfig{ + RequestAttempts: 3, + RequestTimeout: 5, + NumServers: 4, + ServerThreshold: 3, + MaxTimeDelta: 6, + Period: 60, + Servers: []string{"time.apple.com", "time.google.com", "time-a-b.nist.gov", "time-b-b.nist.gov", "gbg1.ntp.se"}, + }, + }, + }, + }, + } + }) + + BeforeAll(func() { + targetImageName = support.PrepareImage(ctx) + }) + + Describe("Install with autogenerated certificates", func() { + BeforeAll(func() { + Expect(cli.Create(ctx, s)).To(Succeed()) + }) + + It("All other components are running", func() { + tas.VerifyAllComponents(ctx, cli, s, true) + }) + + It("Use cosign cli", func() { + tas.VerifyByCosign(ctx, cli, s, targetImageName) + }) + }) + + Describe("Fulcio cert rotation", func() { + + It("Download fulcio cert", func() { + f := fulcio.Get(ctx, cli, namespace.Name, s.Name)() + Expect(f).ToNot(BeNil()) + oldCert, err = kubernetes.GetSecretData(cli, namespace.Name, f.Status.Certificate.CARef) + Expect(err).ToNot(HaveOccurred()) + Expect(oldCert).ToNot(BeEmpty()) + }) + + It("Update fulcio cert", func() { + secretName := "new-fulcio-cert" + newCert = fulcio.CreateSecret(namespace.Name, secretName) + Expect(cli.Create(ctx, newCert)).To(Succeed()) + + Eventually(func(g Gomega) error { + f := securesign.Get(ctx, cli, namespace.Name, s.Name)() + g.Expect(f).ToNot(BeNil()) + f.Spec.Fulcio.Certificate.PrivateKeyRef = &v1alpha1.SecretKeySelector{ + LocalObjectReference: v1alpha1.LocalObjectReference{ + Name: secretName, + }, + Key: "private", + } + + f.Spec.Fulcio.Certificate.PrivateKeyPasswordRef = &v1alpha1.SecretKeySelector{ + LocalObjectReference: v1alpha1.LocalObjectReference{ + Name: secretName, + }, + Key: "password", + } + + f.Spec.Fulcio.Certificate.CARef = &v1alpha1.SecretKeySelector{ + LocalObjectReference: v1alpha1.LocalObjectReference{ + Name: secretName, + }, + Key: "cert", + } + + f.Spec.Ctlog.RootCertificates = []v1alpha1.SecretKeySelector{ + { + LocalObjectReference: v1alpha1.LocalObjectReference{ + Name: secretName, + }, + Key: "cert", + }, + } + + return cli.Update(ctx, f) + }).Should(Succeed()) + + // wait a moment for redeploy + time.Sleep(10 * time.Second) + tas.VerifyAllComponents(ctx, cli, s, true) + }) + + It("Update TUF repository", func() { + certs, err := os.MkdirTemp(os.TempDir(), "certs") + Expect(err).ToNot(HaveOccurred()) + + Expect(os.WriteFile(certs+"/new-fulcio.cert.pem", newCert.Data["cert"], 0644)).To(Succeed()) + Expect(os.WriteFile(certs+"/fulcio_v1.crt.pem", oldCert, 0644)).To(Succeed()) + + tufRepoWorkdir, err := os.MkdirTemp(os.TempDir(), "tuf-repo") + Expect(err).ToNot(HaveOccurred()) + + tufKeys := &v1.Secret{} + Expect(os.Mkdir(filepath.Join(tufRepoWorkdir, "keys"), 0777)).To(Succeed()) + Expect(cli.Get(ctx, runtimeCli.ObjectKey{Name: "tuf-root-keys", Namespace: namespace.Name}, tufKeys)).To(Succeed()) + for k, v := range tufKeys.Data { + Expect(os.WriteFile(filepath.Join(tufRepoWorkdir, "keys", k), v, 0644)).To(Succeed()) + } + + Expect(os.Mkdir(filepath.Join(tufRepoWorkdir, "tuf-repo"), 0777)).To(Succeed()) + tufPodList := &v1.PodList{} + Expect(cli.List(ctx, tufPodList, runtimeCli.InNamespace(namespace.Name), runtimeCli.MatchingLabels{constants.LabelAppComponent: tufAction.ComponentName})).To(Succeed()) + Expect(tufPodList.Items).To(HaveLen(1)) + + Expect(kubernetes2.CopyFromPod(ctx, tufPodList.Items[0], "/var/www/html", filepath.Join(tufRepoWorkdir, "tuf-repo"))).To(Succeed()) + + Expect(clients.ExecuteInDir(certs, "tuftool", tufToolParams("fulcio_v1.crt.pem", tufRepoWorkdir, true)...)).To(Succeed()) + Expect(clients.ExecuteInDir(certs, "tuftool", tufToolParams("new-fulcio.cert.pem", tufRepoWorkdir, false)...)).To(Succeed()) + + Expect(kubernetes2.CopyToPod(ctx, config.GetConfigOrDie(), tufPodList.Items[0], filepath.Join(tufRepoWorkdir, "tuf-repo"), "/var/www/html")).To(Succeed()) + }) + + It("All other components are running", func() { + tas.VerifyAllComponents(ctx, cli, s, true) + }) + + It("Use cosign cli", func() { + tas.VerifyByCosign(ctx, cli, s, targetImageName) + newImage := support.PrepareImage(ctx) + tas.VerifyByCosign(ctx, cli, s, newImage) + }) + }) + +}) + +func tufToolParams(targetName string, workdir string, expire bool) []string { + args := []string{ + "rhtas", + "--root", workdir + "/tuf-repo/root.json", + "--key", workdir + "/keys/snapshot.pem", + "--key", workdir + "/keys/targets.pem", + "--key", workdir + "/keys/timestamp.pem", + "--set-fulcio-target", targetName, + "--fulcio-uri", "https://fulcio.localhost", + "--outdir", workdir + "/tuf-repo", + "--metadata-url", "file://" + workdir + "/tuf-repo", + } + + if expire { + args = append(args, "--fulcio-status", "Expired") + } + return args +} diff --git a/test/e2e/support/archive.go b/test/e2e/support/archive.go index a96b63d9e..3a519cd62 100644 --- a/test/e2e/support/archive.go +++ b/test/e2e/support/archive.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "os" + "path/filepath" + "strings" ) type logTarget struct { @@ -46,3 +48,118 @@ func createArchive(file *os.File, logs map[string]logTarget) error { } return nil } + +func Untar(dst string, r io.Reader) error { + + tr := tar.NewReader(r) + + for { + header, err := tr.Next() + + switch { + + // if no more files are found return + case err == io.EOF: + return nil + + // return any other error + case err != nil: + return err + + // if the header is nil, just skip it (not sure how this happens) + case header == nil: + continue + } + + // the target location where the dir/file should be created + target := filepath.Join(dst, header.Name) + + // the following switch could also be done using fi.Mode(), not sure if there + // a benefit of using one vs. the other. + // fi := header.FileInfo() + + // check the file type + switch header.Typeflag { + + // if its a dir and it doesn't exist create it + case tar.TypeDir: + if _, err := os.Stat(target); err != nil { + if err := os.MkdirAll(target, 0755); err != nil { + return err + } + } + + // if it's a file create it + case tar.TypeReg: + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + + // copy over contents + if _, err := io.Copy(f, tr); err != nil { + return err + } + + // manually close here after each file operation; defering would cause each file close + // to wait until all operations have completed. + _ = f.Close() + } + } +} + +func Tar(src string, writer io.Writer) error { + + // ensure the src actually exists before trying to tar it + if _, err := os.Stat(src); err != nil { + return fmt.Errorf("Unable to tar files - %v", err.Error()) + } + + tw := tar.NewWriter(writer) + defer func() { _ = tw.Close() }() + + // walk path + return filepath.Walk(src, func(file string, fi os.FileInfo, err error) error { + + // return on any error + if err != nil { + return err + } + + // return on non-regular files (thanks to [kumo](https://medium.com/@komuw/just-like-you-did-fbdd7df829d3) for this suggested update) + if !fi.Mode().IsRegular() { + return nil + } + + // create a new dir/file header + header, err := tar.FileInfoHeader(fi, fi.Name()) + if err != nil { + return err + } + + // update the name to correctly reflect the desired destination when untaring + header.Name = strings.TrimPrefix(strings.Replace(file, src, "", -1), string(filepath.Separator)) + + // write the header + if err := tw.WriteHeader(header); err != nil { + return err + } + + // open files for taring + f, err := os.Open(file) + if err != nil { + return err + } + + // copy file data into tar writer + if _, err := io.Copy(tw, f); err != nil { + return err + } + + // manually close here after each file operation; defering would cause each file close + // to wait until all operations have completed. + _ = f.Close() + + return nil + }) +} diff --git a/test/e2e/support/kubernetes/cp.go b/test/e2e/support/kubernetes/cp.go new file mode 100644 index 000000000..d7fdecdf8 --- /dev/null +++ b/test/e2e/support/kubernetes/cp.go @@ -0,0 +1,152 @@ +package kubernetes + +import ( + "context" + "fmt" + "io" + "log" + "os" + "strings" + _ "unsafe" + + "github.com/securesign/operator/test/e2e/support" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + controllerruntime "sigs.k8s.io/controller-runtime" +) + +func CopyToPod(ctx context.Context, config *rest.Config, pod corev1.Pod, srcPath string, destPath string) error { + client, err := v1.NewForConfig(config) + if err != nil { + return err + } + reader, writer := io.Pipe() + + go func() { + defer func() { _ = writer.Close() }() + _ = support.Tar(srcPath, writer) + }() + + //remote shell. + req := client.RESTClient(). + Post(). + Namespace(pod.Namespace). + Resource("pods"). + Name(pod.Name). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: pod.Spec.Containers[0].Name, + Command: []string{"tar", "-xf", "-", "-C", destPath}, + Stdin: true, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + log.Fatalf("error %s\n", err) + return err + } + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: reader, + Stdout: os.Stdout, + Stderr: os.Stderr, + Tty: false, + }) + if err != nil { + log.Fatalf("error %s\n", err) + return err + } + return nil +} +func CopyFromPod(ctx context.Context, pod corev1.Pod, srcPath string, destPath string) error { + reader := newRemoteTarPipe(ctx, pod, srcPath) + return support.Untar(destPath, reader) +} + +// inspired by https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/cp/cp.go +type remoteTarPipe struct { + config *rest.Config + client *v1.CoreV1Client + srcPath string + pod corev1.Pod + reader *io.PipeReader + outStream *io.PipeWriter + bytesRead uint64 + retries int + maxRetries int + ctx context.Context +} + +func newRemoteTarPipe(ctx context.Context, pod corev1.Pod, srcPath string) *remoteTarPipe { + t := new(remoteTarPipe) + t.maxRetries = 10 + t.srcPath = srcPath + t.pod = pod + t.config = controllerruntime.GetConfigOrDie() + t.client = v1.NewForConfigOrDie(t.config) + + t.initReadFrom(0) + t.ctx = ctx + return t +} + +func (t *remoteTarPipe) initReadFrom(n uint64) { + t.reader, t.outStream = io.Pipe() + options := &corev1.PodExecOptions{ + Container: t.pod.Spec.Containers[0].Name, + Command: []string{"tar", "cf", "-", "-C", t.srcPath, "."}, + Stdin: true, + Stdout: true, + Stderr: true, + TTY: false, + } + + if n > 0 { + options.Command = []string{"sh", "-c", fmt.Sprintf("%s | tail -c+%d", strings.Join(options.Command, " "), n)} + } + + req := t.client.RESTClient(). + Post(). + Namespace(t.pod.Namespace). + Resource("pods"). + Name(t.pod.Name). + SubResource("exec"). + VersionedParams(options, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(t.config, "POST", req.URL()) + if err != nil { + log.Fatalf("error %s\n", err) + } + + go func() { + defer func() { _ = t.outStream.Close() }() + _ = exec.StreamWithContext(t.ctx, remotecommand.StreamOptions{ + Stdin: os.Stdin, + Stdout: t.outStream, + Stderr: os.Stderr, + Tty: false, + }) + }() +} + +func (t *remoteTarPipe) Read(p []byte) (n int, err error) { + n, err = t.reader.Read(p) + if err != nil { + if t.retries < t.maxRetries { + t.retries++ + fmt.Printf("Resuming copy at %d bytes, retry %d/%d\n", t.bytesRead, t.retries, t.maxRetries) + t.initReadFrom(t.bytesRead + 1) + err = nil + } else { + fmt.Printf("Dropping out copy after %d retries\n", t.retries) + } + } else { + t.bytesRead += uint64(n) + } + return +} diff --git a/test/e2e/support/tas/cli/command.go b/test/e2e/support/tas/cli/command.go index ddb57117e..194d264d4 100644 --- a/test/e2e/support/tas/cli/command.go +++ b/test/e2e/support/tas/cli/command.go @@ -12,3 +12,11 @@ func Execute(command string, args ...string) error { cmd.Stdout = core.GinkgoWriter return cmd.Run() } + +func ExecuteInDir(workdir string, command string, args ...string) error { + cmd := exec.Command(command, args...) + cmd.Dir = workdir + cmd.Stderr = core.GinkgoWriter + cmd.Stdout = core.GinkgoWriter + return cmd.Run() +}