Upstream versity/versitygw state of the touched files before aaf73219cd

s3api/controllers/bucket-list.go

@@ -0,0 +1,78 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"github.com/gofiber/fiber/v3"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3response"
+)
+
+func (c S3ApiController) ListBuckets(ctx fiber.Ctx) (*Response, error) {
+	cToken := ctx.Query("continuation-token")
+	prefix := ctx.Query("prefix")
+	maxBucketsStr := ctx.Query("max-buckets")
+	acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
+	isRoot, _ := utils.ContextKeyIsRoot.Get(ctx).(bool)
+
+	if err := auth.VerifyListAllMyBucketsAccess(ctx, c.iam, isRoot, acct); err != nil {
+		return &Response{
+			MetaOpts: &MetaOptions{},
+		}, err
+	}
+
+	owner, listAll := acct.Access, acct.Role == auth.RoleAdmin
+	// A backend that fixes bucket ownership leaves no per-caller subset to
+	// narrow the listing to, so every caller lists every bucket, the way an
+	// AWS account's users do. What they may then do with one stays an IAM
+	// policy decision, made per request.
+	if fixedOwner, fixed := auth.ResolveFixedBucketOwner(c.iam); fixed {
+		owner, listAll = fixedOwner.Access, true
+	}
+
+	region, ok := utils.ContextKeyRegion.Get(ctx).(string)
+	if !ok {
+		region = defaultRegion
+	}
+
+	maxBuckets, err := utils.ParseMaxLimiter(maxBucketsStr, utils.LimiterTypeMaxBuckets)
+	if err != nil {
+		return &Response{
+			MetaOpts: &MetaOptions{},
+		}, err
+	}
+
+	// IsAdmin is the backends' "return every bucket, unfiltered" flag.
+	res, err := c.be.ListBuckets(ctx.RequestCtx(),
+		s3response.ListBucketsInput{
+			Owner:             owner,
+			IsAdmin:           listAll,
+			MaxBuckets:        maxBuckets,
+			ContinuationToken: cToken,
+			Prefix:            prefix,
+		})
+	if err != nil {
+		return &Response{}, err
+	}
+
+	for i := range res.Buckets.Bucket {
+		res.Buckets.Bucket[i].BucketRegion = region
+	}
+
+	return &Response{
+		Data: res,
+	}, nil
+}

s3api/controllers/bucket-list_test.go

@@ -0,0 +1,153 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+func TestS3ApiController_ListBuckets(t *testing.T) {
+	validRes := s3response.ListAllMyBucketsResult{
+		Owner: s3response.CanonicalUser{
+			ID: "root",
+		},
+		Buckets: s3response.ListAllMyBucketsList{
+			Bucket: []s3response.ListAllMyBucketsEntry{
+				{Name: "test"},
+			},
+		},
+	}
+
+	tests := []struct {
+		name   string
+		input  testInput
+		output testOutput
+	}{
+		{
+			name: "negative max buckets",
+			input: testInput{
+				locals: defaultLocals,
+				queries: map[string]string{
+					"max-buckets": "-1",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					MetaOpts: &MetaOptions{},
+				},
+				err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "-1"),
+			},
+		},
+		{
+			name: "max buckets exceeds upper limit",
+			input: testInput{
+				locals: defaultLocals,
+				queries: map[string]string{
+					"max-buckets": "10001",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					MetaOpts: &MetaOptions{},
+				},
+				err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "10001"),
+			},
+		},
+		{
+			name: "max buckets 0",
+			input: testInput{
+				locals: defaultLocals,
+				queries: map[string]string{
+					"max-buckets": "0",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					MetaOpts: &MetaOptions{},
+				},
+				err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "0"),
+			},
+		},
+		{
+			name: "invalid max buckets",
+			input: testInput{
+				locals: defaultLocals,
+				queries: map[string]string{
+					"max-buckets": "bla",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					MetaOpts: &MetaOptions{},
+				},
+				err: s3err.GetInvalidArgMaxLimiter("max-buckets", "bla"),
+			},
+		},
+		{
+			name: "backend returns error",
+			input: testInput{
+				locals: defaultLocals,
+				beErr:  s3err.GetAPIError(s3err.ErrNoSuchBucket),
+				beRes:  s3response.ListAllMyBucketsResult{},
+			},
+			output: testOutput{
+				response: &Response{},
+				err:      s3err.GetAPIError(s3err.ErrNoSuchBucket),
+			},
+		},
+		{
+			name: "successful response",
+			input: testInput{
+				locals: defaultLocals,
+				beRes:  validRes,
+				queries: map[string]string{
+					"max-buckets": "3",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					Data: validRes,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			be := &BackendMock{
+				ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+					return tt.input.beRes.(s3response.ListAllMyBucketsResult), tt.input.beErr
+				},
+			}
+
+			ctrl := S3ApiController{
+				be: be,
+			}
+
+			testController(
+				t,
+				ctrl.ListBuckets,
+				tt.output.response,
+				tt.output.err,
+				ctxInputs{
+					locals:  tt.input.locals,
+					queries: tt.input.queries,
+				})
+		})
+	}
+}

s3api/utils/utils.go

@@ -472,6 +472,27 @@ func ParseCalculatedChecksumHeaders(ctx fiber.Ctx) (ChecksumValues, error) {
 	return checksums, nil
 }
 
+// HasPayloadIntegrityCheck reports whether an upload carries an integrity
+// check of its body: a Content-MD5 header, an x-amz-checksum-* header, or an
+// x-amz-checksum-* trailer. A payload hash, signed or not, and chunk
+// signatures don't count as one.
+func HasPayloadIntegrityCheck(ctx fiber.Ctx) bool {
+	if len(ctx.Request().Header.Peek("Content-Md5")) != 0 {
+		return true
+	}
+	if strings.HasPrefix(strings.ToLower(ctx.Get("X-Amz-Trailer")), "x-amz-checksum-") {
+		return true
+	}
+	for key := range ctx.Request().Header.All() {
+		switch k := string(key); {
+		case k == "X-Amz-Checksum-Type", k == "X-Amz-Checksum-Algorithm":
+		case strings.HasPrefix(k, "X-Amz-Checksum-"):
+			return true
+		}
+	}
+	return false
+}
+
 // ParseCalculatedChecksumFields parses and validates object POST checksum fields
 func ParseCalculatedChecksumFields(fields map[string]string) (ChecksumValues, error) {
 	checksums := ChecksumValues{}
@@ -1087,6 +1108,25 @@ func DetectResourceType(ctx fiber.Ctx) s3err.ResourceType {
 	return s3err.ResourceTypeObject
 }
 
+// SetRegionMismatchHeader reports the region the request should have been
+// signed for in the x-amz-bucket-region response header, when err is a signing
+// region mismatch. HEAD requests carry no response body, so the header is the
+// only channel through which a client can discover the gateway region and
+// retry against it, which is how the aws sdks recover from the mismatch.
+func SetRegionMismatchHeader(ctx fiber.Ctx, err error) {
+	rerr, ok := err.(s3err.RegionMismatchError)
+	if !ok {
+		return
+	}
+
+	region := rerr.ExpectedRegion()
+	if region == "" {
+		return
+	}
+
+	ctx.Response().Header.Set("x-amz-bucket-region", region)
+}
+
 // ValidateLocationConstraint checks a CreateBucket location constraint. The
 // global endpoint serves us-east-1 and takes no constraint; any other
 // region is a region specific endpoint and requires the constraint to name it.
@@ -1112,3 +1152,56 @@ func ValidateLocationConstraint(constraint *string, region string) error {
 
 	return nil
 }
+
+// AwsChunkedEncoding is the coding a client announces when it frames a body in
+// aws-chunked, as the SDKs do to carry a trailing checksum.
+const AwsChunkedEncoding = "aws-chunked"
+
+// HasAwsChunkedEncoding reports whether a Content-Encoding value carries the
+// aws-chunked token.
+func HasAwsChunkedEncoding(contentEncoding string) bool {
+	for _, coding := range strings.Split(contentEncoding, ",") {
+		if strings.EqualFold(strings.TrimSpace(coding), AwsChunkedEncoding) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ParseContentEncoding returns the Content-Encoding to store for a request,
+// dropping the aws-chunked token unless the request declares a non streaming
+// payload type in x-amz-content-sha256.
+//
+// S3 keys this on the declaration alone. A streaming payload type frames the
+// body in aws-chunked, and a request that declares no payload type - a
+// presigned URL, signed as UNSIGNED-PAYLOAD and carrying no header - may frame
+// it too, so both drop the token. A declared hex digest or UNSIGNED-PAYLOAD
+// says the body is sent as-is, so there the token is a coding the client chose
+// and is stored as sent.
+//
+// S3 applies this to the Content-Encoding header itself, before any API sees
+// it, so every controller that reads the header inherits it - including
+// CopyObject and CreateMultipartUpload, which carry no body to frame. POSTObject
+// takes its value from a form field and ignores the header, so it stores the
+// token as sent and must not call this.
+func ParseContentEncoding(ctx fiber.Ctx) string {
+	contentEncoding := ctx.Get("Content-Encoding")
+	payloadType := ctx.Get("X-Amz-Content-Sha256")
+	if payloadType != "" && !IsStreamingPayload(payloadType) {
+		return contentEncoding
+	}
+
+	codings := strings.Split(contentEncoding, ",")
+	kept := make([]string, 0, len(codings))
+	for _, coding := range codings {
+		trimmed := strings.TrimSpace(coding)
+		if trimmed == "" || strings.EqualFold(trimmed, AwsChunkedEncoding) {
+			continue
+		}
+
+		kept = append(kept, trimmed)
+	}
+
+	return strings.Join(kept, ",")
+}