Skip to content

Commit

Permalink
feat: support s3 server side encryption (#1039)
Browse files Browse the repository at this point in the history 
* add s3 paramethers to support sse

Signed-off-by: juicer <[email protected]>

* generated manifests

Signed-off-by: juicer <[email protected]>

* update docs

Signed-off-by: juicer <[email protected]>

---------

Signed-off-by: juicer <[email protected]>
  • Loading branch information
@cw-Guo
cw-Guo authored Jan 15, 2024
1 parent a9bdd93 commit 1d594d7
Show file tree
Hide file tree
Showing 11 changed files with 165 additions and 0 deletions.
10 changes: 10 additions & 0 deletions apis/fluentd/v1alpha1/plugins/output/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,14 @@ type S3 struct {
ProxyUri *string `json:"proxyUri,omitempty"`
// Verify the SSL certificate of the endpoint.
SslVerifyPeer *bool `json:"sslVerifyPeer,omitempty"`
// the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
UseServerSideEncryption *string `json:"useServerSideEncryption,omitempty"`
// The AWS KMS enctyption algorithm.
SseCustomerAlgorithm *string `json:"sseCustomerAlgorithm,omitempty"`
// The AWS KMS key ID.
SsekmsKeyId *string `json:"ssekmsKeyId,omitempty"`
// The AWS KMS key.
SseCustomerKey *string `json:"sseCustomerKey,omitempty"`
// The AWS KMS key MD5.
SseCustomerKeyMd5 *string `json:"sseCustomerKeyMd5,omitempty"`
}
15 changes: 15 additions & 0 deletions apis/fluentd/v1alpha1/plugins/output/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,21 @@ func (o *Output) s3Plugin(parent *params.PluginStore, loader plugins.SecretLoade
if o.S3.SslVerifyPeer != nil {
parent.InsertPairs("ssl_verify_peer", fmt.Sprint(*o.S3.SslVerifyPeer))
}
if o.S3.UseServerSideEncryption != nil {
parent.InsertPairs("use_server_side_encryption", fmt.Sprint(*o.S3.UseServerSideEncryption))
}
if o.S3.SseCustomerAlgorithm != nil {
parent.InsertPairs("sse_customer_algorithm", fmt.Sprint(*o.S3.SseCustomerAlgorithm))
}
if o.S3.SsekmsKeyId != nil {
parent.InsertPairs("ssekms_key_id", fmt.Sprint(*o.S3.SsekmsKeyId))
}
if o.S3.SseCustomerKey != nil {
parent.InsertPairs("sse_customer_key", fmt.Sprint(*o.S3.SseCustomerKey))
}
if o.S3.SseCustomerKeyMd5 != nil {
parent.InsertPairs("sse_customer_key_md5", fmt.Sprint(*o.S3.SseCustomerKeyMd5))
}
return parent
}

Expand Down
15 changes: 15 additions & 0 deletions charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1793,6 +1793,18 @@ spec:
s3Region:
description: The Amazon S3 region name
type: string
sseCustomerAlgorithm:
description: The AWS KMS enctyption algorithm.
type: string
sseCustomerKey:
description: The AWS KMS key.
type: string
sseCustomerKeyMd5:
description: The AWS KMS key MD5.
type: string
ssekmsKeyId:
description: The AWS KMS key ID.
type: string
sslVerifyPeer:
description: Verify the SSL certificate of the endpoint.
type: boolean
Expand All @@ -1807,6 +1819,9 @@ spec:
timeSliceFormat:
description: This timestamp is added to each file name
type: string
useServerSideEncryption:
description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
type: string
type: object
stdout:
description: out_stdout plugin
Expand Down
15 changes: 15 additions & 0 deletions charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1793,6 +1793,18 @@ spec:
s3Region:
description: The Amazon S3 region name
type: string
sseCustomerAlgorithm:
description: The AWS KMS enctyption algorithm.
type: string
sseCustomerKey:
description: The AWS KMS key.
type: string
sseCustomerKeyMd5:
description: The AWS KMS key MD5.
type: string
ssekmsKeyId:
description: The AWS KMS key ID.
type: string
sslVerifyPeer:
description: Verify the SSL certificate of the endpoint.
type: boolean
Expand All @@ -1807,6 +1819,9 @@ spec:
timeSliceFormat:
description: This timestamp is added to each file name
type: string
useServerSideEncryption:
description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
type: string
type: object
stdout:
description: out_stdout plugin
Expand Down
15 changes: 15 additions & 0 deletions config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1793,6 +1793,18 @@ spec:
s3Region:
description: The Amazon S3 region name
type: string
sseCustomerAlgorithm:
description: The AWS KMS enctyption algorithm.
type: string
sseCustomerKey:
description: The AWS KMS key.
type: string
sseCustomerKeyMd5:
description: The AWS KMS key MD5.
type: string
ssekmsKeyId:
description: The AWS KMS key ID.
type: string
sslVerifyPeer:
description: Verify the SSL certificate of the endpoint.
type: boolean
Expand All @@ -1807,6 +1819,9 @@ spec:
timeSliceFormat:
description: This timestamp is added to each file name
type: string
useServerSideEncryption:
description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
type: string
type: object
stdout:
description: out_stdout plugin
Expand Down
15 changes: 15 additions & 0 deletions config/crd/bases/fluentd.fluent.io_outputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1793,6 +1793,18 @@ spec:
s3Region:
description: The Amazon S3 region name
type: string
sseCustomerAlgorithm:
description: The AWS KMS enctyption algorithm.
type: string
sseCustomerKey:
description: The AWS KMS key.
type: string
sseCustomerKeyMd5:
description: The AWS KMS key MD5.
type: string
ssekmsKeyId:
description: The AWS KMS key ID.
type: string
sslVerifyPeer:
description: Verify the SSL certificate of the endpoint.
type: boolean
Expand All @@ -1807,6 +1819,9 @@ spec:
timeSliceFormat:
description: This timestamp is added to each file name
type: string
useServerSideEncryption:
description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
type: string
type: object
stdout:
description: out_stdout plugin
Expand Down
5 changes: 5 additions & 0 deletions docs/plugins/fluentd/output/s3.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,8 @@ S3 defines the parameters for out_s3 output plugin
| storeAs | The compression type. | *string |
| proxyUri | The proxy URL. | *string |
| sslVerifyPeer | Verify the SSL certificate of the endpoint. | *bool |
| useServerSideEncryption | the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html | *string |
| sseCustomerAlgorithm | The AWS KMS enctyption algorithm. | *string |
| ssekmsKeyId | The AWS KMS key ID. | *string |
| sseCustomerKey | The AWS KMS key. | *string |
| sseCustomerKeyMd5 | The AWS KMS key MD5. | *string |
4 changes: 4 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,14 @@ require (
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.12.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.28.1 // indirect
Expand All @@ -72,7 +74,9 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.26.1 // indirect
k8s.io/code-generator v0.26.1 // indirect
k8s.io/component-base v0.26.1 // indirect
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
Expand Down
11 changes: 11 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
Expand Down Expand Up @@ -349,6 +350,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -502,6 +505,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
Expand All @@ -512,6 +516,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down Expand Up @@ -639,8 +644,13 @@ k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs=
k8s.io/apimachinery v0.27.4/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s=
k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ=
k8s.io/code-generator v0.26.1 h1:dusFDsnNSKlMFYhzIM0jAO1OlnTN5WYwQQ+Ai12IIlo=
k8s.io/code-generator v0.26.1/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I=
k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU=
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08=
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=
Expand All @@ -656,5 +666,6 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
30 changes: 30 additions & 0 deletions manifests/setup/fluent-operator-crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7506,6 +7506,18 @@ spec:
s3Region:
description: The Amazon S3 region name
type: string
sseCustomerAlgorithm:
description: The AWS KMS enctyption algorithm.
type: string
sseCustomerKey:
description: The AWS KMS key.
type: string
sseCustomerKeyMd5:
description: The AWS KMS key MD5.
type: string
ssekmsKeyId:
description: The AWS KMS key ID.
type: string
sslVerifyPeer:
description: Verify the SSL certificate of the endpoint.
type: boolean
Expand All @@ -7520,6 +7532,9 @@ spec:
timeSliceFormat:
description: This timestamp is added to each file name
type: string
useServerSideEncryption:
description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
type: string
type: object
stdout:
description: out_stdout plugin
Expand Down Expand Up @@ -30778,6 +30793,18 @@ spec:
s3Region:
description: The Amazon S3 region name
type: string
sseCustomerAlgorithm:
description: The AWS KMS enctyption algorithm.
type: string
sseCustomerKey:
description: The AWS KMS key.
type: string
sseCustomerKeyMd5:
description: The AWS KMS key MD5.
type: string
ssekmsKeyId:
description: The AWS KMS key ID.
type: string
sslVerifyPeer:
description: Verify the SSL certificate of the endpoint.
type: boolean
Expand All @@ -30792,6 +30819,9 @@ spec:
timeSliceFormat:
description: This timestamp is added to each file name
type: string
useServerSideEncryption:
description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
type: string
type: object
stdout:
description: out_stdout plugin
Expand Down
30 changes: 30 additions & 0 deletions manifests/setup/setup.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7506,6 +7506,18 @@ spec:
s3Region:
description: The Amazon S3 region name
type: string
sseCustomerAlgorithm:
description: The AWS KMS enctyption algorithm.
type: string
sseCustomerKey:
description: The AWS KMS key.
type: string
sseCustomerKeyMd5:
description: The AWS KMS key MD5.
type: string
ssekmsKeyId:
description: The AWS KMS key ID.
type: string
sslVerifyPeer:
description: Verify the SSL certificate of the endpoint.
type: boolean
Expand All @@ -7520,6 +7532,9 @@ spec:
timeSliceFormat:
description: This timestamp is added to each file name
type: string
useServerSideEncryption:
description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
type: string
type: object
stdout:
description: out_stdout plugin
Expand Down Expand Up @@ -30778,6 +30793,18 @@ spec:
s3Region:
description: The Amazon S3 region name
type: string
sseCustomerAlgorithm:
description: The AWS KMS enctyption algorithm.
type: string
sseCustomerKey:
description: The AWS KMS key.
type: string
sseCustomerKeyMd5:
description: The AWS KMS key MD5.
type: string
ssekmsKeyId:
description: The AWS KMS key ID.
type: string
sslVerifyPeer:
description: Verify the SSL certificate of the endpoint.
type: boolean
Expand All @@ -30792,6 +30819,9 @@ spec:
timeSliceFormat:
description: This timestamp is added to each file name
type: string
useServerSideEncryption:
description: the following parameters are for S3 kms https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
type: string
type: object
stdout:
description: out_stdout plugin
Expand Down

0 comments on commit 1d594d7

Please sign in to comment.