Upstream versity/versitygw state of the touched files before aaf73219cd

s3api/controllers/bucket-list.go

@@ -0,0 +1,78 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"github.com/gofiber/fiber/v3"
+	"github.com/versity/versitygw/auth"
+	"github.com/versity/versitygw/s3api/utils"
+	"github.com/versity/versitygw/s3response"
+)
+
+func (c S3ApiController) ListBuckets(ctx fiber.Ctx) (*Response, error) {
+	cToken := ctx.Query("continuation-token")
+	prefix := ctx.Query("prefix")
+	maxBucketsStr := ctx.Query("max-buckets")
+	acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
+	isRoot, _ := utils.ContextKeyIsRoot.Get(ctx).(bool)
+
+	if err := auth.VerifyListAllMyBucketsAccess(ctx, c.iam, isRoot, acct); err != nil {
+		return &Response{
+			MetaOpts: &MetaOptions{},
+		}, err
+	}
+
+	owner, listAll := acct.Access, acct.Role == auth.RoleAdmin
+	// A backend that fixes bucket ownership leaves no per-caller subset to
+	// narrow the listing to, so every caller lists every bucket, the way an
+	// AWS account's users do. What they may then do with one stays an IAM
+	// policy decision, made per request.
+	if fixedOwner, fixed := auth.ResolveFixedBucketOwner(c.iam); fixed {
+		owner, listAll = fixedOwner.Access, true
+	}
+
+	region, ok := utils.ContextKeyRegion.Get(ctx).(string)
+	if !ok {
+		region = defaultRegion
+	}
+
+	maxBuckets, err := utils.ParseMaxLimiter(maxBucketsStr, utils.LimiterTypeMaxBuckets)
+	if err != nil {
+		return &Response{
+			MetaOpts: &MetaOptions{},
+		}, err
+	}
+
+	// IsAdmin is the backends' "return every bucket, unfiltered" flag.
+	res, err := c.be.ListBuckets(ctx.RequestCtx(),
+		s3response.ListBucketsInput{
+			Owner:             owner,
+			IsAdmin:           listAll,
+			MaxBuckets:        maxBuckets,
+			ContinuationToken: cToken,
+			Prefix:            prefix,
+		})
+	if err != nil {
+		return &Response{}, err
+	}
+
+	for i := range res.Buckets.Bucket {
+		res.Buckets.Bucket[i].BucketRegion = region
+	}
+
+	return &Response{
+		Data: res,
+	}, nil
+}

s3api/controllers/bucket-list_test.go

@@ -0,0 +1,153 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+func TestS3ApiController_ListBuckets(t *testing.T) {
+	validRes := s3response.ListAllMyBucketsResult{
+		Owner: s3response.CanonicalUser{
+			ID: "root",
+		},
+		Buckets: s3response.ListAllMyBucketsList{
+			Bucket: []s3response.ListAllMyBucketsEntry{
+				{Name: "test"},
+			},
+		},
+	}
+
+	tests := []struct {
+		name   string
+		input  testInput
+		output testOutput
+	}{
+		{
+			name: "negative max buckets",
+			input: testInput{
+				locals: defaultLocals,
+				queries: map[string]string{
+					"max-buckets": "-1",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					MetaOpts: &MetaOptions{},
+				},
+				err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "-1"),
+			},
+		},
+		{
+			name: "max buckets exceeds upper limit",
+			input: testInput{
+				locals: defaultLocals,
+				queries: map[string]string{
+					"max-buckets": "10001",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					MetaOpts: &MetaOptions{},
+				},
+				err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "10001"),
+			},
+		},
+		{
+			name: "max buckets 0",
+			input: testInput{
+				locals: defaultLocals,
+				queries: map[string]string{
+					"max-buckets": "0",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					MetaOpts: &MetaOptions{},
+				},
+				err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "0"),
+			},
+		},
+		{
+			name: "invalid max buckets",
+			input: testInput{
+				locals: defaultLocals,
+				queries: map[string]string{
+					"max-buckets": "bla",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					MetaOpts: &MetaOptions{},
+				},
+				err: s3err.GetInvalidArgMaxLimiter("max-buckets", "bla"),
+			},
+		},
+		{
+			name: "backend returns error",
+			input: testInput{
+				locals: defaultLocals,
+				beErr:  s3err.GetAPIError(s3err.ErrNoSuchBucket),
+				beRes:  s3response.ListAllMyBucketsResult{},
+			},
+			output: testOutput{
+				response: &Response{},
+				err:      s3err.GetAPIError(s3err.ErrNoSuchBucket),
+			},
+		},
+		{
+			name: "successful response",
+			input: testInput{
+				locals: defaultLocals,
+				beRes:  validRes,
+				queries: map[string]string{
+					"max-buckets": "3",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					Data: validRes,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			be := &BackendMock{
+				ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
+					return tt.input.beRes.(s3response.ListAllMyBucketsResult), tt.input.beErr
+				},
+			}
+
+			ctrl := S3ApiController{
+				be: be,
+			}
+
+			testController(
+				t,
+				ctrl.ListBuckets,
+				tt.output.response,
+				tt.output.err,
+				ctxInputs{
+					locals:  tt.input.locals,
+					queries: tt.input.queries,
+				})
+		})
+	}
+}

s3api/utils/utils.go

@@ -472,6 +472,27 @@ func ParseCalculatedChecksumHeaders(ctx fiber.Ctx) (ChecksumValues, error) {
 	return checksums, nil
 }
 
+// HasPayloadIntegrityCheck reports whether an upload carries an integrity
+// check of its body: a Content-MD5 header, an x-amz-checksum-* header, or an
+// x-amz-checksum-* trailer. A payload hash, signed or not, and chunk
+// signatures don't count as one.
+func HasPayloadIntegrityCheck(ctx fiber.Ctx) bool {
+	if len(ctx.Request().Header.Peek("Content-Md5")) != 0 {
+		return true
+	}
+	if strings.HasPrefix(strings.ToLower(ctx.Get("X-Amz-Trailer")), "x-amz-checksum-") {
+		return true
+	}
+	for key := range ctx.Request().Header.All() {
+		switch k := string(key); {
+		case k == "X-Amz-Checksum-Type", k == "X-Amz-Checksum-Algorithm":
+		case strings.HasPrefix(k, "X-Amz-Checksum-"):
+			return true
+		}
+	}
+	return false
+}
+
 // ParseCalculatedChecksumFields parses and validates object POST checksum fields
 func ParseCalculatedChecksumFields(fields map[string]string) (ChecksumValues, error) {
 	checksums := ChecksumValues{}
@@ -1087,6 +1108,25 @@ func DetectResourceType(ctx fiber.Ctx) s3err.ResourceType {
 	return s3err.ResourceTypeObject
 }
 
+// SetRegionMismatchHeader reports the region the request should have been
+// signed for in the x-amz-bucket-region response header, when err is a signing
+// region mismatch. HEAD requests carry no response body, so the header is the
+// only channel through which a client can discover the gateway region and
+// retry against it, which is how the aws sdks recover from the mismatch.
+func SetRegionMismatchHeader(ctx fiber.Ctx, err error) {
+	rerr, ok := err.(s3err.RegionMismatchError)
+	if !ok {
+		return
+	}
+
+	region := rerr.ExpectedRegion()
+	if region == "" {
+		return
+	}
+
+	ctx.Response().Header.Set("x-amz-bucket-region", region)
+}
+
 // ValidateLocationConstraint checks a CreateBucket location constraint. The
 // global endpoint serves us-east-1 and takes no constraint; any other
 // region is a region specific endpoint and requires the constraint to name it.
@@ -1112,3 +1152,56 @@ func ValidateLocationConstraint(constraint *string, region string) error {
 
 	return nil
 }
+
+// AwsChunkedEncoding is the coding a client announces when it frames a body in
+// aws-chunked, as the SDKs do to carry a trailing checksum.
+const AwsChunkedEncoding = "aws-chunked"
+
+// HasAwsChunkedEncoding reports whether a Content-Encoding value carries the
+// aws-chunked token.
+func HasAwsChunkedEncoding(contentEncoding string) bool {
+	for _, coding := range strings.Split(contentEncoding, ",") {
+		if strings.EqualFold(strings.TrimSpace(coding), AwsChunkedEncoding) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ParseContentEncoding returns the Content-Encoding to store for a request,
+// dropping the aws-chunked token unless the request declares a non streaming
+// payload type in x-amz-content-sha256.
+//
+// S3 keys this on the declaration alone. A streaming payload type frames the
+// body in aws-chunked, and a request that declares no payload type - a
+// presigned URL, signed as UNSIGNED-PAYLOAD and carrying no header - may frame
+// it too, so both drop the token. A declared hex digest or UNSIGNED-PAYLOAD
+// says the body is sent as-is, so there the token is a coding the client chose
+// and is stored as sent.
+//
+// S3 applies this to the Content-Encoding header itself, before any API sees
+// it, so every controller that reads the header inherits it - including
+// CopyObject and CreateMultipartUpload, which carry no body to frame. POSTObject
+// takes its value from a form field and ignores the header, so it stores the
+// token as sent and must not call this.
+func ParseContentEncoding(ctx fiber.Ctx) string {
+	contentEncoding := ctx.Get("Content-Encoding")
+	payloadType := ctx.Get("X-Amz-Content-Sha256")
+	if payloadType != "" && !IsStreamingPayload(payloadType) {
+		return contentEncoding
+	}
+
+	codings := strings.Split(contentEncoding, ",")
+	kept := make([]string, 0, len(codings))
+	for _, coding := range codings {
+		trimmed := strings.TrimSpace(coding)
+		if trimmed == "" || strings.EqualFold(trimmed, AwsChunkedEncoding) {
+			continue
+		}
+
+		kept = append(kept, trimmed)
+	}
+
+	return strings.Join(kept, ",")
+}

tests/integration/ListBuckets.go

@@ -0,0 +1,435 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
+)
+
+func ListBuckets_as_user(s *S3Conf) error {
+	testName := "ListBuckets_as_user"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
+		for range 6 {
+			bckt := getBucketName()
+
+			err := setup(s, bckt)
+			if err != nil {
+				return err
+			}
+
+			buckets = append(buckets, types.Bucket{
+				Name:         &bckt,
+				BucketRegion: &s.awsRegion,
+			})
+		}
+
+		testuser := getUser("user")
+
+		err := createUsers(s, []user{testuser})
+		if err != nil {
+			return err
+		}
+
+		bckts := []string{}
+		for i := range 3 {
+			bckts = append(bckts, *buckets[i].Name)
+		}
+
+		err = changeBucketsOwner(s, bckts, testuser.access)
+		if err != nil {
+			return err
+		}
+
+		userClient := s.getUserClient(testuser)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := userClient.ListBuckets(ctx, &s3.ListBucketsInput{})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if getString(out.Owner.ID) != testuser.access {
+			return fmt.Errorf("expected buckets owner to be %v, instead got %v",
+				testuser.access, getString(out.Owner.ID))
+		}
+		if !compareBuckets(out.Buckets, buckets[:3]) {
+			return fmt.Errorf("expected list buckets result to be %v, instead got %v",
+				buckets[:3], out.Buckets)
+		}
+
+		for _, elem := range buckets[1:] {
+			err = teardown(s, *elem.Name)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func ListBuckets_as_admin(s *S3Conf) error {
+	testName := "ListBuckets_as_admin"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		testuser, adminUser := getUser("user"), getUser("admin")
+
+		err := createUsers(s, []user{testuser, adminUser})
+		if err != nil {
+			return err
+		}
+
+		adminClient := s.getUserClient(adminUser)
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		initOut, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ignore := make([]string, 0, len(initOut.Buckets))
+		for _, b := range initOut.Buckets {
+			if *b.Name != bucket {
+				ignore = append(ignore, *b.Name)
+			}
+		}
+
+		buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
+		for range 6 {
+			bckt := getBucketName()
+
+			err := setup(s, bckt)
+			if err != nil {
+				return err
+			}
+
+			buckets = append(buckets, types.Bucket{
+				Name:         &bckt,
+				BucketRegion: &s.awsRegion,
+			})
+		}
+
+		bckts := []string{}
+		for i := range 3 {
+			bckts = append(bckts, *buckets[i].Name)
+		}
+
+		err = changeBucketsOwner(s, bckts, testuser.access)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if getString(out.Owner.ID) != adminUser.access {
+			return fmt.Errorf("expected buckets owner to be %v, instead got %v",
+				adminUser.access, getString(out.Owner.ID))
+		}
+		if !compareBuckets(out.Buckets, buckets, ignore...) {
+			return fmt.Errorf("expected list buckets result to be %v, instead got %v",
+				sprintBuckets(buckets), sprintBuckets(out.Buckets))
+		}
+
+		for _, elem := range buckets[1:] {
+			err = teardown(s, *elem.Name)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func ListBuckets_with_prefix(s *S3Conf) error {
+	testName := "ListBuckets_with_prefix"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		prefix := "my-prefix-"
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		prefixInitOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
+			Prefix: &prefix,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ignore := make([]string, 0, len(prefixInitOut.Buckets))
+		for _, b := range prefixInitOut.Buckets {
+			ignore = append(ignore, *b.Name)
+		}
+
+		allBuckets, prefixedBuckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}, []types.Bucket{}
+		for i := range 5 {
+			bckt := getBucketName()
+			if i%2 == 0 {
+				bckt = prefix + bckt
+			}
+
+			err := setup(s, bckt)
+			if err != nil {
+				return err
+			}
+
+			allBuckets = append(allBuckets, types.Bucket{
+				Name:         &bckt,
+				BucketRegion: &s.awsRegion,
+			})
+
+			if i%2 == 0 {
+				prefixedBuckets = append(prefixedBuckets, types.Bucket{
+					Name:         &bckt,
+					BucketRegion: &s.awsRegion,
+				})
+			}
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
+			Prefix: &prefix,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if getString(out.Owner.ID) != s.awsID {
+			return fmt.Errorf("expected owner to be %v, instead got %v",
+				s.awsID, getString(out.Owner.ID))
+		}
+		if getString(out.Prefix) != prefix {
+			return fmt.Errorf("expected prefix to be %v, instead got %v",
+				prefix, getString(out.Prefix))
+		}
+		if !compareBuckets(out.Buckets, prefixedBuckets, ignore...) {
+			return fmt.Errorf("expected list buckets result to be %v, instead got %v",
+				prefixedBuckets, out.Buckets)
+		}
+
+		for _, elem := range allBuckets[1:] {
+			err = teardown(s, *elem.Name)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func ListBuckets_invalid_max_buckets(s *S3Conf) error {
+	testName := "ListBuckets_invalid_max_buckets"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		listBuckets := func(maxBuckets int32) error {
+			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+			_, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
+				MaxBuckets: &maxBuckets,
+			})
+			cancel()
+			return err
+		}
+
+		invMaxBuckets := int32(-3)
+		err := listBuckets(invMaxBuckets)
+		if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, fmt.Sprint(invMaxBuckets))); err != nil {
+			return err
+		}
+
+		invMaxBuckets = 2000000
+		err = listBuckets(invMaxBuckets)
+		if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, fmt.Sprint(invMaxBuckets))); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func ListBuckets_truncated(s *S3Conf) error {
+	testName := "ListBuckets_truncated"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		newBuckets := []types.Bucket{}
+		for range 5 {
+			bckt := getBucketName()
+
+			err := setup(s, bckt)
+			if err != nil {
+				return err
+			}
+
+			newBuckets = append(newBuckets, types.Bucket{
+				Name:         &bckt,
+				BucketRegion: &s.awsRegion,
+			})
+		}
+
+		buckets := append(initOut.Buckets, newBuckets...)
+		sort.Slice(buckets, func(i, j int) bool {
+			return *buckets[i].Name < *buckets[j].Name
+		})
+
+		maxBuckets := int32(3)
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
+			MaxBuckets: &maxBuckets,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if getString(out.Owner.ID) != s.awsID {
+			return fmt.Errorf("expected owner to be %v, instead got %v",
+				s.awsID, getString(out.Owner.ID))
+		}
+		if !compareBuckets(out.Buckets, buckets[:maxBuckets]) {
+			return fmt.Errorf("expected list buckets result to be %v, instead got %v",
+				sprintBuckets(buckets[:maxBuckets]), sprintBuckets(out.Buckets))
+		}
+		if getString(out.ContinuationToken) != getString(buckets[maxBuckets-1].Name) {
+			return fmt.Errorf("expected ContinuationToken to be %v, instead got %v",
+				getString(buckets[maxBuckets-1].Name), getString(out.ContinuationToken))
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err = s3client.ListBuckets(ctx, &s3.ListBucketsInput{
+			ContinuationToken: out.ContinuationToken,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if !compareBuckets(out.Buckets, buckets[maxBuckets:]) {
+			return fmt.Errorf("expected list buckets result to be %v, instead got %v",
+				sprintBuckets(buckets[maxBuckets:]), sprintBuckets(out.Buckets))
+		}
+		if out.ContinuationToken != nil {
+			return fmt.Errorf("expected nil continuation token, instead got %v",
+				*out.ContinuationToken)
+		}
+		if out.Prefix != nil {
+			return fmt.Errorf("expected nil prefix, instead got %v", *out.Prefix)
+		}
+
+		for _, elem := range newBuckets {
+			err = teardown(s, *elem.Name)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func ListBuckets_empty_success(s *S3Conf) error {
+	testName := "ListBuckets_empty_success"
+	return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if len(out.Buckets) > 0 {
+			return fmt.Errorf("expected list buckets result to be %v, instead got %v",
+				[]types.Bucket{}, sprintBuckets(out.Buckets))
+		}
+
+		return nil
+	})
+}
+
+func ListBuckets_success(s *S3Conf) error {
+	testName := "ListBuckets_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ignore := make([]string, 0, len(initOut.Buckets))
+		for _, b := range initOut.Buckets {
+			if *b.Name != bucket {
+				ignore = append(ignore, *b.Name)
+			}
+		}
+
+		buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
+		for range 5 {
+			bckt := getBucketName()
+
+			err := setup(s, bckt)
+			if err != nil {
+				return err
+			}
+
+			buckets = append(buckets, types.Bucket{
+				Name:         &bckt,
+				BucketRegion: &s.awsRegion,
+			})
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if getString(out.Owner.ID) != s.awsID {
+			return fmt.Errorf("expected owner to be %v, instead got %v",
+				s.awsID, getString(out.Owner.ID))
+		}
+		if !compareBuckets(out.Buckets, buckets, ignore...) {
+			return fmt.Errorf("expected list buckets result to be %v, instead got %v",
+				sprintBuckets(buckets), sprintBuckets(out.Buckets))
+		}
+
+		for _, elem := range buckets[1:] {
+			err = teardown(s, *elem.Name)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}

tests/integration/group-tests.go

@@ -546,10 +546,12 @@ func TestListMultipartUploads(ts *TestState) {
 	ts.Run(ListMultipartUploads_empty_result)
 	ts.Run(ListMultipartUploads_invalid_max_uploads)
 	ts.Run(ListMultipartUploads_max_uploads)
+	ts.Run(ListMultipartUploads_next_markers_not_truncated)
 	ts.Run(ListMultipartUploads_exceeding_max_uploads)
 	ts.Run(ListMultipartUploads_ignore_upload_id_marker)
 	ts.Run(ListMultipartUploads_invalid_uploadId_marker)
 	ts.Run(ListMultipartUploads_keyMarker_not_from_list)
+	ts.Run(ListMultipartUploads_upload_id_marker_with_key_marker)
 	ts.Run(ListMultipartUploads_delimiter_truncated)
 	ts.Run(ListMultipartUploads_prefix)
 	ts.Run(ListMultipartUploads_both_delimiter_and_prefix)
@@ -2191,9 +2193,11 @@ func TestRouter(ts *TestState) {
 	ts.Run(RouterGetUploadsWithKey)
 	ts.Run(RouterCopySourceNotAllowed)
 	ts.Run(RouterListVersionsWithKey)
+	ts.Run(RouterUploadsWithUploadId)
 	ts.Run(RouterObjectAnnotationNotImplemented)
 	ts.Run(RouterRenameObjectNotImplemented)
 	ts.Run(RouterObjectTorrentMethodNotAllowed)
+	ts.Run(RouterInvalidURLEscape)
 }
 
 func TestPostObject(ts *TestState) {
@@ -3282,10 +3286,12 @@ func GetIntTests() IntTests {
 		"ListMultipartUploads_empty_result":                                                ListMultipartUploads_empty_result,
 		"ListMultipartUploads_invalid_max_uploads":                                         ListMultipartUploads_invalid_max_uploads,
 		"ListMultipartUploads_max_uploads":                                                 ListMultipartUploads_max_uploads,
+		"ListMultipartUploads_next_markers_not_truncated":                                  ListMultipartUploads_next_markers_not_truncated,
 		"ListMultipartUploads_exceeding_max_uploads":                                       ListMultipartUploads_exceeding_max_uploads,
 		"ListMultipartUploads_ignore_upload_id_marker":                                     ListMultipartUploads_ignore_upload_id_marker,
 		"ListMultipartUploads_invalid_uploadId_marker":                                     ListMultipartUploads_invalid_uploadId_marker,
 		"ListMultipartUploads_keyMarker_not_from_list":                                     ListMultipartUploads_keyMarker_not_from_list,
+		"ListMultipartUploads_upload_id_marker_with_key_marker":                            ListMultipartUploads_upload_id_marker_with_key_marker,
 		"ListMultipartUploads_delimiter_truncated":                                         ListMultipartUploads_delimiter_truncated,
 		"ListMultipartUploads_prefix":                                                      ListMultipartUploads_prefix,
 		"ListMultipartUploads_both_delimiter_and_prefix":                                   ListMultipartUploads_both_delimiter_and_prefix,
@@ -3794,9 +3800,11 @@ func GetIntTests() IntTests {
 		"RouterGetUploadsWithKey":                                                          RouterGetUploadsWithKey,
 		"RouterCopySourceNotAllowed":                                                       RouterCopySourceNotAllowed,
 		"RouterListVersionsWithKey":                                                        RouterListVersionsWithKey,
+		"RouterUploadsWithUploadId":                                                        RouterUploadsWithUploadId,
 		"RouterObjectAnnotationNotImplemented":                                             RouterObjectAnnotationNotImplemented,
 		"RouterRenameObjectNotImplemented":                                                 RouterRenameObjectNotImplemented,
 		"RouterObjectTorrentMethodNotAllowed":                                              RouterObjectTorrentMethodNotAllowed,
+		"RouterInvalidURLEscape":                                                           RouterInvalidURLEscape,
 		"UnsignedStreaminPayloadTrailer_malformed_trailer":                                 UnsignedStreaminPayloadTrailer_malformed_trailer,
 		"UnsignedStreamingPayloadTrailer_aborted_connection":                               UnsignedStreamingPayloadTrailer_aborted_connection,
 		"UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length":               UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length,

tests/test_rest_list_buckets.sh

@@ -0,0 +1,385 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+load ./bats-support/load
+load ./bats-assert/load
+
+source ./tests/commands/list_buckets.sh
+source ./tests/drivers/create_bucket/create_bucket_rest.sh
+source ./tests/drivers/list_buckets/list_buckets_rest.sh
+source ./tests/drivers/user.sh
+source ./tests/logger.sh
+source ./tests/setup.sh
+
+export RUN_USERS=true
+
+# tags: openssl,malformed-message
+@test "REST - empty message" {
+  test_file="test_file"
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1249"
+  fi
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket_and_file_v2 "$bucket_name" "$test_file"
+  assert_success
+
+  echo -en "\r\n" > "$TEST_FILE_FOLDER/empty.txt"
+  run send_via_openssl_with_timeout "$TEST_FILE_FOLDER/empty.txt"
+  assert_success
+}
+
+# tags: malformed-message,openssl
+@test "REST - deformed message" {
+  test_file="test_file"
+  echo -en "abcdefg\r\n\r\n" > "$TEST_FILE_FOLDER/deformed.txt"
+  run send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/deformed.txt" 400 "BadRequest" "An error occurred when parsing the HTTP request."
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,Authorization,invalid-header
+@test "REST - invalid authorization scheme" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1705"
+  fi
+  run list_buckets_check_authorization_scheme_error
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,Authorization,invalid-header
+@test "REST - very invalid credential string" {
+  run send_rest_go_command_expect_error "400" "AuthorizationHeaderMalformed" "the Credential is mal-formed" "-incorrectCredential" "Credentials"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,Authorization,invalid-header
+@test "REST - nonexistent key ID" {
+  run send_rest_go_command_expect_error "403" "InvalidAccessKeyId" "does not exist" "-awsAccessKeyId" "dummy"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,Authorization,x-amz-date,invalid-header
+@test "REST - invalid year/month/day" {
+  run send_rest_go_command_expect_error "400" "AuthorizationHeaderMalformed" "incorrect date format" "-invalidYearMonthDay"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,Authorization,x-amz-date,invalid-header
+@test "REST - incorrect year/month/day" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1514"
+  fi
+  run list_buckets_check_request_time_too_skewed_error
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,Authorization,invalid-header
+@test "REST - invalid region" {
+  run send_rest_go_command_expect_error "400" "AuthorizationHeaderMalformed" "us-eest-1" "-awsRegion" "us-eest-1"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,Authorization,invalid-header
+@test "REST - invalid service name" {
+  run send_rest_go_command_expect_error "400" "AuthorizationHeaderMalformed" "incorrect service" "-serviceName" "s2"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,Authorization,invalid-header
+@test "REST - incorrect signature" {
+  run send_rest_go_command_expect_error "403" "SignatureDoesNotMatch" "does not match" "-incorrectSignature"
+  assert_success
+}
+
+# tags: openssl,ListBuckets,required-headers,host,invalid-header
+@test "REST - missing host parameter" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1530"
+  fi
+  run send_openssl_go_command "400" "-missingHostParam"
+  assert_success
+}
+
+# tags: curl,ListBuckets,minimal-request
+@test "test_rest_list_buckets" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  run list_check_buckets_rest "$bucket_name"
+  assert_success
+}
+
+# tags: curl,ListBuckets,max-buckets,continuation-token
+@test "REST - list buckets - continuation token isn't bucket name" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1399"
+  fi
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  # shellcheck disable=SC2153
+  run get_bucket_name "$BUCKET_TWO_NAME"
+  assert_success
+  bucket_two_name="$output"
+
+  run setup_buckets_v2 "$bucket_name" "$bucket_two_name"
+  assert_success
+
+  run check_continuation_token
+  assert_success
+}
+
+# tags: curl,ListBuckets,max-buckets,continuation-token
+@test "REST - list buckets - success (multiple pages)" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run get_bucket_name "$BUCKET_TWO_NAME"
+  assert_success
+  bucket_two_name="$output"
+
+  run setup_buckets_v2 "$bucket_name" "$bucket_two_name"
+  assert_success
+
+  run check_for_buckets_with_multiple_pages "$bucket_name" "$bucket_two_name"
+  assert_success
+}
+
+# tags: curl,ListBuckets,prefix
+@test "REST - list buckets w/prefix" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run get_bucket_name "$BUCKET_TWO_NAME"
+  assert_success
+  bucket_two_name="$output"
+
+  run setup_buckets_v2 "$bucket_name" "$bucket_two_name"
+  assert_success
+
+  run list_check_buckets_rest "$bucket_name" "$bucket_two_name"
+  assert_success
+
+  run list_check_buckets_rest_with_prefix "$bucket_name"
+  assert_success
+
+  run list_check_buckets_rest_with_prefix "$bucket_two_name"
+  assert_success
+}
+
+# tags: curl,ListBuckets,minimal-request,user
+@test "REST - ListBuckets - correct buckets show up" {
+  if [ "$SKIP_USERS_TESTS" == "true" ]; then
+    skip "skip versitygw-specific users tests"
+  fi
+  if [ "$DIRECT" == "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1704"
+  fi
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket_and_user "$bucket_name" "$USERNAME_ONE" "$PASSWORD_ONE" "user"
+  assert_success
+  username=${lines[${#lines[@]}-2]}
+  password=${lines[${#lines[@]}-1]}
+
+  run get_bucket_name "$BUCKET_TWO_NAME"
+  assert_success
+  bucket_two_name="$output"
+
+  run setup_bucket "$bucket_two_name"
+  assert_success
+
+  run change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$bucket_two_name" "$username"
+  assert_success
+
+  log 5 "username: $username, password: $password"
+  run list_check_buckets_user "$username" "$password" "$bucket_two_name"
+  assert_success
+}
+
+# tags: curl,invalid-method
+@test "REST - service route - invalid POST route" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1810"
+  fi
+  run get_file_name
+  assert_success
+  file_name=$output
+
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket_and_add_file "$bucket_name" "$file_name"
+  assert_success
+
+  run send_rest_go_command_expect_error_with_specific_arg_names_values "405" "MethodNotAllowed" "is not allowed" 4 "Method" "POST" "ResourceType" "SERVICE" "-method" "POST"
+  assert_success
+}
+
+# tags: curl,invalid-method
+@test "REST - service route - invalid method" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1846"
+  fi
+  run send_rest_go_command_expect_error "400" "BadRequest" "An error occurred when parsing the HTTP request" "-method" "GETS"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,x-amz-date
+@test "REST - ListBuckets - error Content-Type is application/xml" {
+  run send_rest_go_command_check_header_key_and_value "403" "Content-Type" "application/xml" "-method" "GET" "-omitDate"
+  assert_success
+}
+
+# tags: curl,ListBuckets,bucket-region,invalid-query
+@test "REST - ListBuckets - invalid bucket-region query" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1923"
+  fi
+  local invalid_region="abc"
+
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  local bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  run send_rest_go_command_expect_error_with_specific_arg_name_value "400" "InvalidArgument" "Argument value $invalid_region is not a valid AWS Region" \
+   "ArgumentName" "bucket-region" "-query" "bucket-region=$invalid_region"
+  assert_success
+}
+
+# tags: curl,ListBuckets,bucket-region
+@test "REST - ListBuckets - incorrect bucket region" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1930"
+  fi
+  local test_region="us-east-1"
+  if [ "$AWS_REGION" == "us-east-1" ]; then
+    test_region="us-west-1"
+  fi
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  local params=()
+  if [ "$DIRECT" == "true" ]; then
+    params+=("-url" "https://s3.$test_region.amazonaws.com" "-awsRegion" "$test_region")
+  fi
+
+  run list_buckets_bucket_not_in_list "$bucket_name" "$test_region" "${params[@]}"
+  assert_success
+}
+
+# tags: curl,ListBuckets,bucket-region
+@test "REST - ListBuckets - correct bucket region" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  run send_rest_go_command_callback "200" "check_bucket_and_region" "-query" "bucket-region=$AWS_REGION" "--" "$bucket_name" "$AWS_REGION"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,x-amz-content-sha256
+@test "REST - ListBuckets - missing sha256 hash" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  run send_rest_go_command_expect_error "400" "InvalidRequest" "Missing required header for this request: x-amz-content-sha256" "-omitSHA256Hash"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,x-amz-content-sha256
+@test "REST - ListBuckets - invalid hash type" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  run send_rest_go_command_expect_error "400" "InvalidArgument" "x-amz-content-sha256 must be" "-customSHA256Hash" "ABCDEFG"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,x-amz-content-sha256
+@test "REST - ListBuckets - non-matching hash type" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  run send_rest_go_command_expect_error "400" "InvalidRequest" "The value of x-amz-content-sha256 header is invalid" "-customSHA256Hash" "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,x-amz-date
+@test "REST - ListBuckets - omit date" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1934"
+  fi
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  run send_rest_go_command_expect_error "403" "AccessDenied" "AWS authentication requires a valid Date or x-amz-date header" "-omitDate"
+  assert_success
+}
+
+# tags: curl,ListBuckets,required-headers,x-amz-date
+@test "REST - ListBuckets - invalid date" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1934"
+  fi
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run setup_bucket "$bucket_name"
+  assert_success
+
+  run send_rest_go_command_expect_error "403" "AccessDenied" "AWS authentication requires a valid Date or x-amz-date header" "-customDate" "ABCDEFG"
+  assert_success
+}