From f3549091db0dbf2bbfaf454264cfa1d8cb3f77b1 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 13 Sep 2023 15:57:16 +0200 Subject: [PATCH] add support for stream signing --- go.mod | 1 - go.sum | 8 - signature/auth-types.go | 68 ++++ signature/signature-errors.go | 12 + signature/signature-v4-parser.go | 7 +- signature/signature-v4.go | 82 +++- signature/streaming-signature-v4.go | 561 ++++++++++++++++++++++++++++ uploader.go | 3 - 8 files changed, 712 insertions(+), 30 deletions(-) create mode 100644 signature/auth-types.go create mode 100644 signature/streaming-signature-v4.go diff --git a/go.mod b/go.mod index 0f0b1b1f..053091a9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/SiaFoundation/gofakes3 go 1.16 require ( - github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 github.com/aws/aws-sdk-go v1.44.256 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 diff --git a/go.sum b/go.sum index 3dd42382..f8c73652 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,3 @@ -github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q= -github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY= -github.com/aws/aws-sdk-go v1.44.124/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,14 +26,11 @@ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -49,7 +43,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -72,7 +65,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= diff --git a/signature/auth-types.go b/signature/auth-types.go new file mode 100644 index 00000000..273b4352 --- /dev/null +++ b/signature/auth-types.go @@ -0,0 +1,68 @@ +package signature + +import ( + "net/http" + "net/url" + "strings" +) + +// ref: https://github.com/minio/minio/cmd/auth-handler.go + +type authType int + +// List of all supported auth types. +const ( + authTypeUnknown authType = iota + authTypeStreamingSigned + authTypeSigned + authTypeStreamingSignedTrailer + authTypeStreamingUnsignedTrailer +) + +// Verify if request has AWS Signature Version '4'. +func isRequestSignatureV4(r *http.Request) bool { + return strings.HasPrefix(r.Header.Get(headerAuth), "AWS4-HMAC-SHA256") +} + +// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation. +func isRequestSignStreamingV4(r *http.Request) bool { + return r.Header.Get("X-Amz-Content-Sha256") == streamingContentSHA256 && + r.Method == http.MethodPut +} + +// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation. +func isRequestSignStreamingTrailerV4(r *http.Request) bool { + return r.Header.Get("X-Amz-Content-Sha256") == streamingContentSHA256Trailer && + r.Method == http.MethodPut +} + +// Verify if the request has AWS Streaming Signature Version '4', with unsigned content and trailer. +func isRequestUnsignedTrailerV4(r *http.Request) bool { + return r.Header.Get("X-Amz-Content-Sha256") == "STREAMING-UNSIGNED-PAYLOAD-TRAILER" && + r.Method == http.MethodPut && strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") +} + +// Get request authentication type. +func getRequestAuthType(r *http.Request) (at authType) { + if r.URL != nil { + var err error + r.Form, err = url.ParseQuery(r.URL.RawQuery) + if err != nil { + return authTypeUnknown + } + } + if isRequestSignStreamingV4(r) { + return authTypeStreamingSigned + } else if isRequestSignStreamingTrailerV4(r) { + return authTypeStreamingSignedTrailer + } else if isRequestUnsignedTrailerV4(r) { + return authTypeStreamingUnsignedTrailer + } else if isRequestSignatureV4(r) { + return authTypeSigned + } + return authTypeUnknown +} + +func IsSupportedAuthentication(req *http.Request) bool { + return getRequestAuthType(req) != authTypeUnknown +} diff --git a/signature/signature-errors.go b/signature/signature-errors.go index 0afeda46..6910d7bf 100644 --- a/signature/signature-errors.go +++ b/signature/signature-errors.go @@ -29,11 +29,13 @@ type errorCodeMap map[ErrorCode]APIError const ( errMissingFields ErrorCode = iota errMissingCredTag + errContentSHA256Mismatch errCredMalformed errInvalidAccessKeyID errMalformedCredentialDate errInvalidRequestVersion errInvalidServiceS3 + errMissingContentLength errMissingSignHeadersTag errMissingSignTag errUnsignedHeaders @@ -59,6 +61,11 @@ var errorCodes = errorCodeMap{ Description: "Missing Credential field for this request.", HTTPStatusCode: http.StatusBadRequest, }, + errContentSHA256Mismatch: { + Code: "XAmzContentSHA256Mismatch", + Description: "The provided 'x-amz-content-sha256' header does not match what was computed.", + HTTPStatusCode: http.StatusBadRequest, + }, errCredMalformed: { Code: "AuthorizationQueryParameterserror", Description: "error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"/YYYYMMDD/REGION/SERVICE/aws4_request\".", @@ -84,6 +91,11 @@ var errorCodes = errorCodeMap{ Description: "error parsing the Credential/X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".", HTTPStatusCode: http.StatusBadRequest, }, + errMissingContentLength: { + Code: "MissingContentLength", + Description: "You must provide the Content-Length HTTP header.", + HTTPStatusCode: http.StatusLengthRequired, + }, errMissingSignHeadersTag: { Code: "InvalidArgument", Description: "Signature header missing SignedHeaders field.", diff --git a/signature/signature-v4-parser.go b/signature/signature-v4-parser.go index 713a4761..d48b76af 100644 --- a/signature/signature-v4-parser.go +++ b/signature/signature-v4-parser.go @@ -90,12 +90,7 @@ func extractFields(signElement, fieldName string) (string, ErrorCode) { // // Authorization: algorithm Credential=accessKeyID/credScope, SignedHeaders=signedHeaders, Signature=signature func parseSignV4(v4Auth string) (sv signValues, err ErrorCode) { - - if !strings.HasPrefix(v4Auth, signV4Algorithm) { - return sv, errUnsupportAlgorithm - } - - rawCred := strings.ReplaceAll(strings.TrimPrefix(v4Auth, signV4Algorithm), " ", "") + rawCred := strings.ReplaceAll(strings.TrimPrefix(v4Auth, "AWS4-HMAC-SHA256"), " ", "") authFields := strings.Split(strings.TrimSpace(rawCred), ",") if len(authFields) != 3 { return sv, errMissingFields diff --git a/signature/signature-v4.go b/signature/signature-v4.go index 157cb733..7c67f156 100644 --- a/signature/signature-v4.go +++ b/signature/signature-v4.go @@ -5,8 +5,11 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" + "errors" + "io" "net/http" "sort" + "strconv" "strings" "time" ) @@ -14,12 +17,11 @@ import ( // ref: https://github.com/minio/minio/cmd/auth-handler.go const ( - signV4Algorithm = "AWS4-HMAC-SHA256" - iso8601Format = "20060102T150405Z" - yyyymmdd = "20060102" - serviceS3 = "s3" - slashSeparator = "/" - stype = serviceS3 + iso8601Format = "20060102T150405Z" + yyyymmdd = "20060102" + serviceS3 = "s3" + slashSeparator = "/" + stype = serviceS3 headerAuth = "Authorization" headerDate = "Date" @@ -27,6 +29,8 @@ const ( amzDate = "X-Amz-Date" ) +var errSignatureMismatch = errors.New("Signature does not match") + // getCanonicalHeaders generate a list of request headers with their values func getCanonicalHeaders(signedHeaders http.Header) string { var headers []string @@ -52,6 +56,17 @@ func getCanonicalHeaders(signedHeaders http.Header) string { return buf.String() } +// getScope generate a string of a specific date, an AWS region, and a service. +func getScope(t time.Time, region string) string { + scope := strings.Join([]string{ + t.Format(yyyymmdd), + region, + string(serviceS3), + "aws4_request", + }, slashSeparator) + return scope +} + // getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names func getSignedHeaders(signedHeaders http.Header) string { var headers []string @@ -110,7 +125,7 @@ func getCanonicalRequest(extractedSignedHeaders http.Header, payload, queryStr, // getStringToSign a string based on selected query values. func getStringToSign(canonicalRequest string, t time.Time, scope string) string { - stringToSign := signV4Algorithm + "\n" + t.Format(iso8601Format) + "\n" + stringToSign := "AWS4-HMAC-SHA256\n" + t.Format(iso8601Format) + "\n" stringToSign += scope + "\n" canonicalRequestBytes := sha256.Sum256([]byte(canonicalRequest)) stringToSign += hex.EncodeToString(canonicalRequestBytes[:]) @@ -126,11 +141,7 @@ func getSigningKey(secretKey string, t time.Time, region string) []byte { return signingKey } -// V4SignVerify - Verify authorization header with calculated header in accordance with -// - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html -// -// returns nil if signature matches. -func V4SignVerify(r *http.Request) ErrorCode { +func authTypeSignedVerify(r *http.Request) ErrorCode { // Copy request. req := *r hashedPayload := getContentSha256Cksum(r) @@ -192,3 +203,50 @@ func V4SignVerify(r *http.Request) ErrorCode { // Return Error none. return ErrNone } + +func authTypeStreamingVerify(r *http.Request, authType authType) ErrorCode { + var size int64 + if sizeStr, ok := r.Header["X-Amz-Decoded-Content-Length"]; ok { + if sizeStr[0] == "" { + return errMissingContentLength + } + var err error + size, err = strconv.ParseInt(sizeStr[0], 10, 64) + if err != nil { + return errMissingContentLength + } + } + var rc io.ReadCloser + var ec ErrorCode + switch authType { + case authTypeStreamingSigned, authTypeStreamingSignedTrailer: + rc, ec = newSignV4ChunkedReader(r, authType == authTypeStreamingSignedTrailer) + case authTypeStreamingUnsignedTrailer: + return errUnsupportAlgorithm // not supported + default: + panic("can't call authTypeStreamingVerify with a non streaming auth type") + } + if ec != ErrNone { + return ec + } + r.Body = rc + r.ContentLength = size + return ErrNone +} + +// V4SignVerify - Verify authorization header with calculated header in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +// +// returns nil if signature matches. +func V4SignVerify(r *http.Request) ErrorCode { + // Make sure the authentication type is supported. + authType := getRequestAuthType(r) + switch authType { + case authTypeStreamingSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer: + return authTypeStreamingVerify(r, authType) + case authTypeSigned: + return authTypeSignedVerify(r) + default: + return errUnsupportAlgorithm + } +} diff --git a/signature/streaming-signature-v4.go b/signature/streaming-signature-v4.go new file mode 100644 index 00000000..d16eb981 --- /dev/null +++ b/signature/streaming-signature-v4.go @@ -0,0 +1,561 @@ +package signature + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package cmd This file implements helper functions to validate Streaming AWS +// Signature Version '4' authorization header. + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + "strings" + "time" +) + +// Streaming AWS Signature Version '4' constants. +const ( + streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + streamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + signV4ChunkedAlgorithm = "AWS4-HMAC-SHA256-PAYLOAD" + signV4ChunkedAlgorithmTrailer = "AWS4-HMAC-SHA256-TRAILER" + streamingContentEncoding = "aws-chunked" + awsTrailerHeader = "X-Amz-Trailer" + trailerKVSeparator = ":" +) + +// getChunkSignature - get chunk signature. +// Does not update anything in cr. +func (cr *s3ChunkedReader) getChunkSignature() string { + hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil)) + + // Calculate string to sign. + alg := signV4ChunkedAlgorithm + "\n" + stringToSign := alg + + cr.seedDate.Format(iso8601Format) + "\n" + + getScope(cr.seedDate, cr.region) + "\n" + + cr.seedSignature + "\n" + + emptySHA256 + "\n" + + hashedChunk + + // Get hmac signing key. + signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate, cr.region) + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) + + return newSignature +} + +// getTrailerChunkSignature - get trailer chunk signature. +func (cr *s3ChunkedReader) getTrailerChunkSignature() string { + hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil)) + + // Calculate string to sign. + alg := signV4ChunkedAlgorithmTrailer + "\n" + stringToSign := alg + + cr.seedDate.Format(iso8601Format) + "\n" + + getScope(cr.seedDate, cr.region) + "\n" + + cr.seedSignature + "\n" + + hashedChunk + + // Get hmac signing key. + signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate, cr.region) + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) + + return newSignature +} + +// calculateSeedSignature - Calculate seed signature in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html +// +// returns signature, error otherwise if the signature mismatches or any other +// error while parsing and validating. +func calculateSeedSignature(r *http.Request, trailers bool) (cred Credentials, signature string, region string, date time.Time, errCode ErrorCode) { + // Copy request. + req := *r + + // Save authorization header. + v4Auth := req.Header.Get(headerAuth) + + // Parse signature version '4' header. + signV4Values, errCode := parseSignV4(v4Auth) + if errCode != ErrNone { + return cred, "", "", time.Time{}, errCode + } + + // Payload streaming. + payload := streamingContentSHA256 + if trailers { + payload = streamingContentSHA256Trailer + } + + // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' + if payload != req.Header.Get(amzContentSha256) { + return cred, "", "", time.Time{}, errContentSHA256Mismatch + } + + // Extract all the signed headers along with its values. + extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r) + if errCode != ErrNone { + return cred, "", "", time.Time{}, errCode + } + + cred, _, errCode = checkKeyValid(r, signV4Values.Credential.accessKey) + if errCode != ErrNone { + return cred, "", "", time.Time{}, errCode + } + + // Verify if region is valid. + region = signV4Values.Credential.scope.region + + // Extract date, if not present throw error. + var dateStr string + if dateStr = req.Header.Get("x-amz-date"); dateStr == "" { + if dateStr = r.Header.Get("Date"); dateStr == "" { + return cred, "", "", time.Time{}, errMissingDateHeader + } + } + + // Parse date header. + var err error + date, err = time.Parse(iso8601Format, dateStr) + if err != nil { + return cred, "", "", time.Time{}, errMalformedDate + } + + // Query string. + queryStr := req.Form.Encode() + + // Get canonical request. + canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method) + + // Get string to sign from canonical request. + stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope()) + + // Get hmac signing key. + signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region) + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) + + // Verify if signature match. + if !compareSignatureV4(newSignature, signV4Values.Signature) { + return cred, "", "", time.Time{}, errSignatureDoesNotMatch + } + + // Return caculated signature. + return cred, newSignature, region, date, ErrNone +} + +// malformed encoding is generated when chunk header is wrongly formed. +var errMalformedEncoding = errors.New("malformed chunked encoding") + +// chunk is considered too big if its bigger than > 16MiB. +var errChunkTooBig = errors.New("chunk too big: choose chunk size <= 16MiB") + +// newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r +// out of HTTP "chunked" format before returning it. +// The s3ChunkedReader returns io.EOF when the final 0-length chunk is read. +// +// NewChunkedReader is not needed by normal applications. The http package +// automatically decodes chunking when reading response bodies. +func newSignV4ChunkedReader(req *http.Request, trailer bool) (io.ReadCloser, ErrorCode) { + cred, seedSignature, region, seedDate, errCode := calculateSeedSignature(req, trailer) + if errCode != ErrNone { + return nil, errCode + } + + if trailer { + // Discard anything unsigned. + req.Trailer = make(http.Header) + trailers := req.Header.Values(awsTrailerHeader) + for _, key := range trailers { + req.Trailer.Add(key, "") + } + } else { + req.Trailer = nil + } + return &s3ChunkedReader{ + trailers: req.Trailer, + reader: bufio.NewReader(req.Body), + cred: cred, + seedSignature: seedSignature, + seedDate: seedDate, + region: region, + chunkSHA256Writer: sha256.New(), + buffer: make([]byte, 64*1024), + debug: false, + }, ErrNone +} + +// Represents the overall state that is required for decoding a +// AWS Signature V4 chunked reader. +type s3ChunkedReader struct { + reader *bufio.Reader + cred Credentials + seedSignature string + seedDate time.Time + region string + trailers http.Header + + chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data. + buffer []byte + offset int + err error + debug bool // Print details on failure. Add your own if more are needed. +} + +func (cr *s3ChunkedReader) Close() (err error) { + return nil +} + +// Now, we read one chunk from the underlying reader. +// A chunk has the following format: +// +// + ";chunk-signature=" + + "\r\n" + + "\r\n" +// +// First, we read the chunk size but fail if it is larger +// than 16 MiB. We must not accept arbitrary large chunks. +// One 16 MiB is a reasonable max limit. +// +// Then we read the signature and payload data. We compute the SHA256 checksum +// of the payload and verify that it matches the expected signature value. +// +// The last chunk is *always* 0-sized. So, we must only return io.EOF if we have encountered +// a chunk with a chunk size = 0. However, this chunk still has a signature and we must +// verify it. +const maxChunkSize = 16 << 20 // 16 MiB + +// Read - implements `io.Reader`, which transparently decodes +// the incoming AWS Signature V4 streaming signature. +func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { + if cr.err != nil { + if cr.debug { + fmt.Printf("s3ChunkedReader: Returning err: %v (%T)\n", cr.err, cr.err) + } + return 0, cr.err + } + defer func() { + if err != nil && err != io.EOF { + if cr.debug { + fmt.Println("Read err:", err) + } + } + }() + // First, if there is any unread data, copy it to the client + // provided buffer. + if cr.offset > 0 { + n = copy(buf, cr.buffer[cr.offset:]) + if n == len(buf) { + cr.offset += n + return n, nil + } + cr.offset = 0 + buf = buf[n:] + } + + var size int + for { + b, err := cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b == ';' { // separating character + break + } + + // Manually deserialize the size since AWS specified + // the chunk size to be of variable width. In particular, + // a size of 16 is encoded as `10` while a size of 64 KB + // is `10000`. + switch { + case b >= '0' && b <= '9': + size = size<<4 | int(b-'0') + case b >= 'a' && b <= 'f': + size = size<<4 | int(b-('a'-10)) + case b >= 'A' && b <= 'F': + size = size<<4 | int(b-('A'-10)) + default: + cr.err = errMalformedEncoding + return n, cr.err + } + if size > maxChunkSize { + cr.err = errChunkTooBig + return n, cr.err + } + } + + // Now, we read the signature of the following payload and expect: + // chunk-signature=" + + "\r\n" + // + // The signature is 64 bytes long (hex-encoded SHA256 hash) and + // starts with a 16 byte header: len("chunk-signature=") + 64 == 80. + var signature [80]byte + _, err = io.ReadFull(cr.reader, signature[:]) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if !bytes.HasPrefix(signature[:], []byte("chunk-signature=")) { + cr.err = errMalformedEncoding + return n, cr.err + } + b, err := cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b != '\r' { + cr.err = errMalformedEncoding + return n, cr.err + } + b, err = cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b != '\n' { + cr.err = errMalformedEncoding + return n, cr.err + } + + if cap(cr.buffer) < size { + cr.buffer = make([]byte, size) + } else { + cr.buffer = cr.buffer[:size] + } + + // Now, we read the payload and compute its SHA-256 hash. + _, err = io.ReadFull(cr.reader, cr.buffer) + if err == io.EOF && size != 0 { + err = io.ErrUnexpectedEOF + } + if err != nil && err != io.EOF { + cr.err = err + return n, cr.err + } + + // Once we have read the entire chunk successfully, we verify + // that the received signature matches our computed signature. + cr.chunkSHA256Writer.Write(cr.buffer) + newSignature := cr.getChunkSignature() + if !compareSignatureV4(string(signature[16:]), newSignature) { + cr.err = errSignatureMismatch + return n, cr.err + } + cr.seedSignature = newSignature + cr.chunkSHA256Writer.Reset() + + // If the chunk size is zero we return io.EOF. As specified by AWS, + // only the last chunk is zero-sized. + if len(cr.buffer) == 0 { + if cr.debug { + fmt.Println("EOF. Reading Trailers:", cr.trailers) + } + if cr.trailers != nil { + err = cr.readTrailers() + if cr.debug { + fmt.Println("trailers returned:", err, "now:", cr.trailers) + } + if err != nil { + cr.err = err + return 0, err + } + } + cr.err = io.EOF + return n, cr.err + } + + b, err = cr.reader.ReadByte() + if b != '\r' || err != nil { + if cr.debug { + fmt.Printf("want %q, got %q\n", "\r", string(b)) + } + cr.err = errMalformedEncoding + return n, cr.err + } + b, err = cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b != '\n' { + if cr.debug { + fmt.Printf("want %q, got %q\n", "\r", string(b)) + } + cr.err = errMalformedEncoding + return n, cr.err + } + + cr.offset = copy(buf, cr.buffer) + n += cr.offset + return n, err +} + +// readTrailers will read all trailers and populate cr.trailers with actual values. +func (cr *s3ChunkedReader) readTrailers() error { + if cr.debug { + fmt.Printf("pre trailer sig: %s\n", cr.seedSignature) + } + var valueBuffer bytes.Buffer + // Read value + for { + v, err := cr.reader.ReadByte() + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if v != '\r' { + valueBuffer.WriteByte(v) + continue + } + // End of buffer, do not add to value. + v, err = cr.reader.ReadByte() + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if v != '\n' { + return errMalformedEncoding + } + break + } + + // Read signature + var signatureBuffer bytes.Buffer + for { + v, err := cr.reader.ReadByte() + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if v != '\r' { + signatureBuffer.WriteByte(v) + continue + } + var tmp [3]byte + _, err = io.ReadFull(cr.reader, tmp[:]) + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if string(tmp[:]) != "\n\r\n" { + if cr.debug { + fmt.Printf("signature, want %q, got %q", "\n\r\n", string(tmp[:])) + } + return errMalformedEncoding + } + // No need to write final newlines to buffer. + break + } + + // Verify signature. + sig := signatureBuffer.Bytes() + if !bytes.HasPrefix(sig, []byte("x-amz-trailer-signature:")) { + if cr.debug { + fmt.Printf("prefix, want prefix %q, got %q", "x-amz-trailer-signature:", string(sig)) + } + return errMalformedEncoding + } + + // TODO: It seems like we may have to be prepared to rewrite and sort trailing headers: + // https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html + + // Any value must end with a newline. + // Not all clients send that. + trailerRaw := valueBuffer.Bytes() + if len(trailerRaw) > 0 && trailerRaw[len(trailerRaw)-1] != '\n' { + valueBuffer.Write([]byte{'\n'}) + } + sig = sig[len("x-amz-trailer-signature:"):] + sig = bytes.TrimSpace(sig) + cr.chunkSHA256Writer.Write(valueBuffer.Bytes()) + wantSig := cr.getTrailerChunkSignature() + if !compareSignatureV4(string(sig), wantSig) { + if cr.debug { + fmt.Printf("signature, want: %q, got %q\nSignature buffer: %q\n", wantSig, string(sig), valueBuffer.String()) + } + return errSignatureMismatch + } + + // Parse trailers. + wantTrailers := make(map[string]struct{}, len(cr.trailers)) + for k := range cr.trailers { + wantTrailers[strings.ToLower(k)] = struct{}{} + } + input := bufio.NewScanner(bytes.NewReader(valueBuffer.Bytes())) + for input.Scan() { + line := strings.TrimSpace(input.Text()) + if line == "" { + continue + } + // Find first separator. + idx := strings.IndexByte(line, trailerKVSeparator[0]) + if idx <= 0 || idx >= len(line) { + if cr.debug { + fmt.Printf("index, ':' not found in %q\n", line) + } + return errMalformedEncoding + } + key := line[:idx] + value := line[idx+1:] + if _, ok := wantTrailers[key]; !ok { + if cr.debug { + fmt.Printf("%q not found in %q\n", key, cr.trailers) + } + return errMalformedEncoding + } + cr.trailers.Set(key, value) + delete(wantTrailers, key) + } + + // Check if we got all we want. + if len(wantTrailers) > 0 { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/uploader.go b/uploader.go index c10f7eb6..48b3af26 100644 --- a/uploader.go +++ b/uploader.go @@ -368,9 +368,6 @@ func (u *uploader) AbortMultipartUpload(bucket, object string, id UploadID) erro } func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) { - if partNumber > MaxUploadPartNumber { - return "", ErrInvalidPart - } body, err := io.ReadAll(input) if err != nil { return "", err