From 2bb327ec69340005f465b19ff22cffb2c943c8cb Mon Sep 17 00:00:00 2001 From: upstream Date: Fri, 2 Oct 2026 16:07:01 +0200 Subject: [PATCH] Upstream versity/versitygw state of the touched files before aaf73219cd --- s3api/controllers/bucket-list.go | 78 ++ s3api/controllers/bucket-list_test.go | 153 +++ s3api/utils/utils.go | 93 ++ s3api/utils/utils_test.go | 1651 +++++++++++++++++++++++++ tests/integration/ListBuckets.go | 435 +++++++ tests/integration/group-tests.go | 8 + tests/test_rest_list_buckets.sh | 385 ++++++ 7 files changed, 2803 insertions(+) create mode 100644 s3api/controllers/bucket-list.go create mode 100644 s3api/controllers/bucket-list_test.go create mode 100644 s3api/utils/utils_test.go create mode 100644 tests/integration/ListBuckets.go create mode 100644 tests/test_rest_list_buckets.sh diff --git a/s3api/controllers/bucket-list.go b/s3api/controllers/bucket-list.go new file mode 100644 index 0000000..c85f77c --- /dev/null +++ b/s3api/controllers/bucket-list.go @@ -0,0 +1,78 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) ListBuckets(ctx fiber.Ctx) (*Response, error) { + cToken := ctx.Query("continuation-token") + prefix := ctx.Query("prefix") + maxBucketsStr := ctx.Query("max-buckets") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot, _ := utils.ContextKeyIsRoot.Get(ctx).(bool) + + if err := auth.VerifyListAllMyBucketsAccess(ctx, c.iam, isRoot, acct); err != nil { + return &Response{ + MetaOpts: &MetaOptions{}, + }, err + } + + owner, listAll := acct.Access, acct.Role == auth.RoleAdmin + // A backend that fixes bucket ownership leaves no per-caller subset to + // narrow the listing to, so every caller lists every bucket, the way an + // AWS account's users do. What they may then do with one stays an IAM + // policy decision, made per request. + if fixedOwner, fixed := auth.ResolveFixedBucketOwner(c.iam); fixed { + owner, listAll = fixedOwner.Access, true + } + + region, ok := utils.ContextKeyRegion.Get(ctx).(string) + if !ok { + region = defaultRegion + } + + maxBuckets, err := utils.ParseMaxLimiter(maxBucketsStr, utils.LimiterTypeMaxBuckets) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{}, + }, err + } + + // IsAdmin is the backends' "return every bucket, unfiltered" flag. + res, err := c.be.ListBuckets(ctx.RequestCtx(), + s3response.ListBucketsInput{ + Owner: owner, + IsAdmin: listAll, + MaxBuckets: maxBuckets, + ContinuationToken: cToken, + Prefix: prefix, + }) + if err != nil { + return &Response{}, err + } + + for i := range res.Buckets.Bucket { + res.Buckets.Bucket[i].BucketRegion = region + } + + return &Response{ + Data: res, + }, nil +} diff --git a/s3api/controllers/bucket-list_test.go b/s3api/controllers/bucket-list_test.go new file mode 100644 index 0000000..bfa16f6 --- /dev/null +++ b/s3api/controllers/bucket-list_test.go @@ -0,0 +1,153 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "testing" + + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_ListBuckets(t *testing.T) { + validRes := s3response.ListAllMyBucketsResult{ + Owner: s3response.CanonicalUser{ + ID: "root", + }, + Buckets: s3response.ListAllMyBucketsList{ + Bucket: []s3response.ListAllMyBucketsEntry{ + {Name: "test"}, + }, + }, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "negative max buckets", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-buckets": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "-1"), + }, + }, + { + name: "max buckets exceeds upper limit", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-buckets": "10001", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "10001"), + }, + }, + { + name: "max buckets 0", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-buckets": "0", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "0"), + }, + }, + { + name: "invalid max buckets", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-buckets": "bla", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetInvalidArgMaxLimiter("max-buckets", "bla"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + beRes: s3response.ListAllMyBucketsResult{}, + }, + output: testOutput{ + response: &Response{}, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + queries: map[string]string{ + "max-buckets": "3", + }, + }, + output: testOutput{ + response: &Response{ + Data: validRes, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { + return tt.input.beRes.(s3response.ListAllMyBucketsResult), tt.input.beErr + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListBuckets, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + queries: tt.input.queries, + }) + }) + } +} diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index f0b42bd..16e7bfa 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -472,6 +472,27 @@ func ParseCalculatedChecksumHeaders(ctx fiber.Ctx) (ChecksumValues, error) { return checksums, nil } +// HasPayloadIntegrityCheck reports whether an upload carries an integrity +// check of its body: a Content-MD5 header, an x-amz-checksum-* header, or an +// x-amz-checksum-* trailer. A payload hash, signed or not, and chunk +// signatures don't count as one. +func HasPayloadIntegrityCheck(ctx fiber.Ctx) bool { + if len(ctx.Request().Header.Peek("Content-Md5")) != 0 { + return true + } + if strings.HasPrefix(strings.ToLower(ctx.Get("X-Amz-Trailer")), "x-amz-checksum-") { + return true + } + for key := range ctx.Request().Header.All() { + switch k := string(key); { + case k == "X-Amz-Checksum-Type", k == "X-Amz-Checksum-Algorithm": + case strings.HasPrefix(k, "X-Amz-Checksum-"): + return true + } + } + return false +} + // ParseCalculatedChecksumFields parses and validates object POST checksum fields func ParseCalculatedChecksumFields(fields map[string]string) (ChecksumValues, error) { checksums := ChecksumValues{} @@ -1087,6 +1108,25 @@ func DetectResourceType(ctx fiber.Ctx) s3err.ResourceType { return s3err.ResourceTypeObject } +// SetRegionMismatchHeader reports the region the request should have been +// signed for in the x-amz-bucket-region response header, when err is a signing +// region mismatch. HEAD requests carry no response body, so the header is the +// only channel through which a client can discover the gateway region and +// retry against it, which is how the aws sdks recover from the mismatch. +func SetRegionMismatchHeader(ctx fiber.Ctx, err error) { + rerr, ok := err.(s3err.RegionMismatchError) + if !ok { + return + } + + region := rerr.ExpectedRegion() + if region == "" { + return + } + + ctx.Response().Header.Set("x-amz-bucket-region", region) +} + // ValidateLocationConstraint checks a CreateBucket location constraint. The // global endpoint serves us-east-1 and takes no constraint; any other // region is a region specific endpoint and requires the constraint to name it. @@ -1112,3 +1152,56 @@ func ValidateLocationConstraint(constraint *string, region string) error { return nil } + +// AwsChunkedEncoding is the coding a client announces when it frames a body in +// aws-chunked, as the SDKs do to carry a trailing checksum. +const AwsChunkedEncoding = "aws-chunked" + +// HasAwsChunkedEncoding reports whether a Content-Encoding value carries the +// aws-chunked token. +func HasAwsChunkedEncoding(contentEncoding string) bool { + for _, coding := range strings.Split(contentEncoding, ",") { + if strings.EqualFold(strings.TrimSpace(coding), AwsChunkedEncoding) { + return true + } + } + + return false +} + +// ParseContentEncoding returns the Content-Encoding to store for a request, +// dropping the aws-chunked token unless the request declares a non streaming +// payload type in x-amz-content-sha256. +// +// S3 keys this on the declaration alone. A streaming payload type frames the +// body in aws-chunked, and a request that declares no payload type - a +// presigned URL, signed as UNSIGNED-PAYLOAD and carrying no header - may frame +// it too, so both drop the token. A declared hex digest or UNSIGNED-PAYLOAD +// says the body is sent as-is, so there the token is a coding the client chose +// and is stored as sent. +// +// S3 applies this to the Content-Encoding header itself, before any API sees +// it, so every controller that reads the header inherits it - including +// CopyObject and CreateMultipartUpload, which carry no body to frame. POSTObject +// takes its value from a form field and ignores the header, so it stores the +// token as sent and must not call this. +func ParseContentEncoding(ctx fiber.Ctx) string { + contentEncoding := ctx.Get("Content-Encoding") + payloadType := ctx.Get("X-Amz-Content-Sha256") + if payloadType != "" && !IsStreamingPayload(payloadType) { + return contentEncoding + } + + codings := strings.Split(contentEncoding, ",") + kept := make([]string, 0, len(codings)) + for _, coding := range codings { + trimmed := strings.TrimSpace(coding) + if trimmed == "" || strings.EqualFold(trimmed, AwsChunkedEncoding) { + continue + } + + kept = append(kept, trimmed) + } + + return strings.Join(kept, ",") +} diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go new file mode 100644 index 0000000..217d4bc --- /dev/null +++ b/s3api/utils/utils_test.go @@ -0,0 +1,1651 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "bufio" + "bytes" + "encoding/xml" + "errors" + "math/rand" + "net/http" + "net/url" + "reflect" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v3" + "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +// a helper method to construct a raw http request with the given http request headers +// to further parse with fasthttp.Request.Read and return fasthttp.RequestHeader +func createHeadersFromRawRequest(t *testing.T, hdrs [][2]string) *fasthttp.RequestHeader { + t.Helper() + + var b strings.Builder + b.WriteString("PUT / HTTP/1.1\r\n") + b.WriteString("Host: example.com\r\n") + for _, kv := range hdrs { + b.WriteString(kv[0]) + b.WriteString(": ") + b.WriteString(kv[1]) + b.WriteString("\r\n") + } + b.WriteString("\r\n") + + var req fasthttp.Request + if err := req.Read(bufio.NewReader(bytes.NewReader([]byte(b.String())))); err != nil { + t.Fatalf("failed to parse raw request: %v", err) + } + return &req.Header +} + +func TestGetUserMetaData(t *testing.T) { + tests := []struct { + name string + hdrs [][2]string + want map[string]string + wantErr error + }{ + { + name: "no metadata headers", + hdrs: [][2]string{ + {"Content-Type", "application/json"}, + }, + want: map[string]string{}, + }, + { + name: "single metadata header", + hdrs: [][2]string{ + {"x-amz-meta-foo", "bar"}, + }, + want: map[string]string{"foo": "bar"}, + }, + { + name: "multiple metadata headers", + hdrs: [][2]string{ + {"x-amz-meta-foo", "bar"}, + {"x-amz-meta-baz", "qux"}, + }, + want: map[string]string{"foo": "bar", "baz": "qux"}, + }, + { + name: "case-insensitive prefix and key lowercasing", + hdrs: [][2]string{ + {"X-Amz-Meta-TestKey", "Value"}, + }, + want: map[string]string{"testkey": "Value"}, + }, + { + name: "ignores non-metadata headers", + hdrs: [][2]string{ + {"authorization", "token"}, + {"x-amz-meta-foo", "bar"}, + }, + want: map[string]string{"foo": "bar"}, + }, + { + name: "metadata size exceeds limit (single header)", + hdrs: [][2]string{ + {"x-amz-meta-big", strings.Repeat("a", maxMetadataSize+1)}, + }, + wantErr: s3err.GetMetadataTooLargeErr(2052, maxMetadataSize), + }, + { + name: "metadata cumulative size exceeds limit (multiple headers)", + hdrs: [][2]string{ + {"x-amz-meta-a", strings.Repeat("a", maxMetadataSize/2)}, + {"x-amz-meta-b", strings.Repeat("b", maxMetadataSize/2+10)}, + }, + wantErr: s3err.GetMetadataTooLargeErr(2060, maxMetadataSize), + }, + { + name: "duplicate keys combined", + hdrs: [][2]string{ + {"x-amz-meta-Foo", "first"}, + {"x-amz-meta-foo", "second"}, + }, + want: map[string]string{"foo": "first,second"}, + }, + { + name: "duplicate same value keys combined", + hdrs: [][2]string{ + {"x-amz-meta-Foo", "value"}, + {"x-amz-meta-foo", "value"}, + }, + want: map[string]string{"foo": "value,value"}, + }, + { + name: "mixed keys", + hdrs: [][2]string{ + {"x-amz-meta-Foo", "value2"}, + {"x-amz-meta-fOo", "value1"}, + {"x-amz-meta-foO", "value3"}, + {"x-amz-meta-bar", "baz"}, + {"x-amz-meta-quxx", "efg"}, + {"x-amz-meta-abc", "value"}, + {"x-amz-meta-Abc", "value"}, + {"x-amz-meta-aBc", "value"}, + {"x-amz-meta-abC", "value"}, + {"x-amz-meta-ABC", "value"}, + }, + want: map[string]string{ + "foo": "value2,value1,value3", + "bar": "baz", + "quxx": "efg", + "abc": "value,value,value,value,value", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := createHeadersFromRawRequest(t, tt.hdrs) + + got, err := GetUserMetaData(h) + assert.Equal(t, tt.wantErr, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestIsValidBucketName(t *testing.T) { + type args struct { + bucket string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "IsValidBucketName-short-name", + args: args{ + bucket: "a", + }, + want: false, + }, + { + name: "IsValidBucketName-start-with-hyphen", + args: args{ + bucket: "-bucket", + }, + want: false, + }, + { + name: "IsValidBucketName-start-with-dot", + args: args{ + bucket: ".bucket", + }, + want: false, + }, + { + name: "IsValidBucketName-contain-invalid-character", + args: args{ + bucket: "my@bucket", + }, + want: false, + }, + { + name: "IsValidBucketName-end-with-hyphen", + args: args{ + bucket: "bucket-", + }, + want: false, + }, + { + name: "IsValidBucketName-end-with-dot", + args: args{ + bucket: "bucket.", + }, + want: false, + }, + { + name: "IsValidBucketName-valid-bucket-name", + args: args{ + bucket: "my-bucket", + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidBucketName(tt.args.bucket); got != tt.want { + t.Errorf("IsValidBucketName() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSetBucketNameValidationStrict(t *testing.T) { + SetBucketNameValidationStrict(true) + t.Cleanup(func() { + SetBucketNameValidationStrict(true) + }) + + invalidBucket := "Invalid_Bucket" + if IsValidBucketName(invalidBucket) { + t.Fatalf("expected %q to be invalid with strict validation", invalidBucket) + } + + SetBucketNameValidationStrict(false) + if !IsValidBucketName(invalidBucket) { + t.Fatalf("expected %q to be accepted when strict validation disabled", invalidBucket) + } + + SetBucketNameValidationStrict(true) + if IsValidBucketName(invalidBucket) { + t.Fatalf("expected %q to be invalid after re-enabling strict validation", invalidBucket) + } +} + +func TestParseMaxLimiter(t *testing.T) { + type args struct { + str string + lt LimiterType + } + type expected struct { + err error + res int32 + } + tests := []struct { + name string + args args + expected expected + }{ + { + name: "empty_string", + args: args{ + str: "", + lt: LimiterTypeMaxKeys, + }, + expected: expected{ + err: nil, + res: 1000, + }, + }, + { + name: "empty_max-buckets", + args: args{ + str: "", + lt: LimiterTypeMaxBuckets, + }, + expected: expected{ + err: nil, + res: 10000, + }, + }, + { + name: "invalid_max-parts", + args: args{ + str: "bla", + lt: LimiterTypeMaxParts, + }, + expected: expected{ + err: s3err.GetInvalidArgMaxLimiter(string(LimiterTypeMaxParts), "bla"), + res: 0, + }, + }, + { + name: "invalid_max-uploads", + args: args{ + str: "invalid", + lt: LimiterTypeMaxUploads, + }, + expected: expected{ + err: s3err.GetInvalidArgMaxLimiter(string(LimiterTypeMaxUploads), "invalid"), + res: 0, + }, + }, + { + name: "invalid_max-buckets", + args: args{ + str: "invalid", + lt: LimiterTypeMaxBuckets, + }, + expected: expected{ + err: s3err.GetInvalidArgMaxLimiter(string(LimiterTypeMaxBuckets), "invalid"), + res: 0, + }, + }, + { + name: "invalid_versions_max-keys", + args: args{ + str: "invalid", + lt: LimiterTypeMaxKeys, + }, + expected: expected{ + err: s3err.GetInvalidArgMaxLimiter(string(LimiterTypeMaxKeys), "invalid"), + res: 0, + }, + }, + { + name: "negative_max-keys", + args: args{ + str: "-5", + lt: LimiterTypeMaxKeys, + }, + expected: expected{ + err: s3err.GetInvalidArgNegativeMaxLimiter(string(LimiterTypeMaxKeys), "-5"), + res: 0, + }, + }, + { + name: "negative_part-number-marker", + args: args{ + str: "-5", + lt: LimiterTypePartNumberMarker, + }, + expected: expected{ + err: s3err.GetInvalidArgNegativeMaxLimiter(string(LimiterTypePartNumberMarker), "-5"), + res: 0, + }, + }, + { + name: "negative_max-buckets", + args: args{ + str: "-12", + lt: LimiterTypeMaxBuckets, + }, + expected: expected{ + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "-12"), + res: 0, + }, + }, + { + name: "negative_versions_max-keys", + args: args{ + str: "-12", + lt: LimiterTypeVersionsMaxKeys, + }, + expected: expected{ + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgNegativeMaxKeys, "-12"), + res: 0, + }, + }, + { + name: "greater_than_10000_max-buckets", + args: args{ + str: "25000", + lt: LimiterTypeMaxBuckets, + }, + expected: expected{ + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, "25000"), + res: 0, + }, + }, + { + name: "greater_than_1000_max-buckets", + args: args{ + str: "1300", + lt: LimiterTypeMaxBuckets, + }, + expected: expected{ + err: nil, + res: 1300, + }, + }, + { + name: "greater_than_1000", + args: args{ + str: "25000", + lt: LimiterTypeMaxParts, + }, + expected: expected{ + err: nil, + res: 1000, + }, + }, + { + name: "success", + args: args{ + str: "23", + lt: LimiterTypeMaxUploads, + }, + expected: expected{ + err: nil, + res: 23, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseMaxLimiter(tt.args.str, tt.args.lt) + assert.Equal(t, tt.expected.err, err) + assert.Equal(t, tt.expected.res, got) + }) + } +} + +func TestFilterObjectAttributes(t *testing.T) { + type args struct { + attrs map[s3response.ObjectAttributes]struct{} + output s3response.GetObjectAttributesResponse + } + + etag, objSize := "etag", int64(3222) + delMarker := true + + tests := []struct { + name string + args args + want s3response.GetObjectAttributesResponse + }{ + { + name: "keep only ETag", + args: args{ + attrs: map[s3response.ObjectAttributes]struct{}{ + s3response.ObjectAttributesEtag: {}, + }, + output: s3response.GetObjectAttributesResponse{ + ObjectSize: &objSize, + ETag: &etag, + }, + }, + want: s3response.GetObjectAttributesResponse{ETag: &etag}, + }, + { + name: "keep multiple props", + args: args{ + attrs: map[s3response.ObjectAttributes]struct{}{ + s3response.ObjectAttributesEtag: {}, + s3response.ObjectAttributesObjectSize: {}, + s3response.ObjectAttributesStorageClass: {}, + }, + output: s3response.GetObjectAttributesResponse{ + ObjectSize: &objSize, + ETag: &etag, + ObjectParts: &s3response.ObjectParts{}, + VersionId: &etag, + }, + }, + want: s3response.GetObjectAttributesResponse{ + ETag: &etag, + ObjectSize: &objSize, + }, + }, + { + name: "make sure LastModified, DeleteMarker and VersionId are removed", + args: args{ + attrs: map[s3response.ObjectAttributes]struct{}{ + s3response.ObjectAttributesEtag: {}, + }, + output: s3response.GetObjectAttributesResponse{ + ObjectSize: &objSize, + ETag: &etag, + ObjectParts: &s3response.ObjectParts{}, + VersionId: &etag, + LastModified: backend.GetTimePtr(time.Now()), + DeleteMarker: &delMarker, + }, + }, + want: s3response.GetObjectAttributesResponse{ + ETag: &etag, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := FilterObjectAttributes(tt.args.attrs, tt.args.output); !reflect.DeepEqual(got, tt.want) { + t.Errorf("FilterObjectAttributes() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsValidOwnership(t *testing.T) { + type args struct { + val types.ObjectOwnership + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "valid-BucketOwnerEnforced", + args: args{ + val: types.ObjectOwnershipBucketOwnerEnforced, + }, + want: true, + }, + { + name: "valid-BucketOwnerPreferred", + args: args{ + val: types.ObjectOwnershipBucketOwnerPreferred, + }, + want: true, + }, + { + name: "valid-ObjectWriter", + args: args{ + val: types.ObjectOwnershipObjectWriter, + }, + want: true, + }, + { + name: "invalid_value", + args: args{ + val: types.ObjectOwnership("invalid_value"), + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidOwnership(tt.args.val); got != tt.want { + t.Errorf("IsValidOwnership() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsChecksumAlgorithmValid(t *testing.T) { + type args struct { + alg types.ChecksumAlgorithm + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "empty", + args: args{ + alg: "", + }, + wantErr: false, + }, + { + name: "crc32", + args: args{ + alg: types.ChecksumAlgorithmCrc32, + }, + wantErr: false, + }, + { + name: "crc32c", + args: args{ + alg: types.ChecksumAlgorithmCrc32c, + }, + wantErr: false, + }, + { + name: "sha1", + args: args{ + alg: types.ChecksumAlgorithmSha1, + }, + wantErr: false, + }, + { + name: "sha256", + args: args{ + alg: types.ChecksumAlgorithmSha256, + }, + wantErr: false, + }, + { + name: "crc64nvme", + args: args{ + alg: types.ChecksumAlgorithmCrc64nvme, + }, + wantErr: false, + }, + { + name: "sha512", + args: args{ + alg: types.ChecksumAlgorithmSha512, + }, + wantErr: false, + }, + { + name: "md5", + args: args{ + alg: types.ChecksumAlgorithmMd5, + }, + wantErr: false, + }, + { + name: "xxhash64", + args: args{ + alg: types.ChecksumAlgorithmXxhash64, + }, + wantErr: false, + }, + { + name: "xxhash3", + args: args{ + alg: types.ChecksumAlgorithmXxhash3, + }, + wantErr: false, + }, + { + name: "xxhash128", + args: args{ + alg: types.ChecksumAlgorithmXxhash128, + }, + wantErr: false, + }, + { + name: "invalid", + args: args{ + alg: types.ChecksumAlgorithm("invalid_checksum_algorithm"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := IsChecksumAlgorithmValid(tt.args.alg); (err != nil) != tt.wantErr { + t.Errorf("IsChecksumAlgorithmValid() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestIsValidChecksum(t *testing.T) { + type args struct { + checksum string + algorithm types.ChecksumAlgorithm + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "invalid-base64", + args: args{ + checksum: "invalid_base64_string", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: false, + }, + { + name: "invalid-crc32", + args: args{ + checksum: "YXNkZmFzZGZhc2Rm", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: false, + }, + { + name: "valid-crc32", + args: args{ + checksum: "ww2FVQ==", + algorithm: types.ChecksumAlgorithmCrc32, + }, + want: true, + }, + { + name: "invalid-crc32c", + args: args{ + checksum: "Zmdoa2doZmtnZmhr", + algorithm: types.ChecksumAlgorithmCrc32c, + }, + want: false, + }, + { + name: "valid-crc32c", + args: args{ + checksum: "DOsb4w==", + algorithm: types.ChecksumAlgorithmCrc32c, + }, + want: true, + }, + { + name: "invalid-sha1", + args: args{ + checksum: "YXNkZmFzZGZhc2RmYXNkZnNhZGZzYWRm", + algorithm: types.ChecksumAlgorithmSha1, + }, + want: false, + }, + { + name: "valid-sha1", + args: args{ + checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=", + algorithm: types.ChecksumAlgorithmSha1, + }, + want: true, + }, + { + name: "invalid-sha256", + args: args{ + checksum: "Zmdoa2doZmtnZmhrYXNkZmFzZGZhc2RmZHNmYXNkZg==", + algorithm: types.ChecksumAlgorithmSha256, + }, + want: false, + }, + { + name: "valid-sha256", + args: args{ + checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=", + algorithm: types.ChecksumAlgorithmSha256, + }, + want: true, + }, + { + name: "invalid-crc64nvme", + args: args{ + checksum: "ww2FVQ==", + algorithm: types.ChecksumAlgorithmCrc64nvme, + }, + want: false, + }, + { + name: "valid-crc64nvme", + args: args{ + checksum: "AAAAAAAAAAA=", + algorithm: types.ChecksumAlgorithmCrc64nvme, + }, + want: true, + }, + { + name: "invalid-sha512", + args: args{ + checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=", + algorithm: types.ChecksumAlgorithmSha512, + }, + want: false, + }, + { + name: "valid-sha512", + args: args{ + checksum: "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==", + algorithm: types.ChecksumAlgorithmSha512, + }, + want: true, + }, + { + name: "invalid-md5", + args: args{ + checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=", + algorithm: types.ChecksumAlgorithmMd5, + }, + want: false, + }, + { + name: "valid-md5", + args: args{ + checksum: "1B2M2Y8AsgTpgAmY7PhCfg==", + algorithm: types.ChecksumAlgorithmMd5, + }, + want: true, + }, + { + name: "invalid-xxhash64", + args: args{ + checksum: "1B2M2Y8AsgTpgAmY7PhCfg==", + algorithm: types.ChecksumAlgorithmXxhash64, + }, + want: false, + }, + { + name: "valid-xxhash64", + args: args{ + checksum: "70bbN1HY6Zk=", + algorithm: types.ChecksumAlgorithmXxhash64, + }, + want: true, + }, + { + name: "invalid-xxhash3", + args: args{ + checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=", + algorithm: types.ChecksumAlgorithmXxhash3, + }, + want: false, + }, + { + name: "valid-xxhash3", + args: args{ + checksum: "LQaABTjTlMI=", + algorithm: types.ChecksumAlgorithmXxhash3, + }, + want: true, + }, + { + name: "invalid-xxhash128", + args: args{ + checksum: "70bbN1HY6Zk=", + algorithm: types.ChecksumAlgorithmXxhash128, + }, + want: false, + }, + { + name: "valid-xxhash128", + args: args{ + checksum: "maoG0wFHmNhgAcMkRo1Jfw==", + algorithm: types.ChecksumAlgorithmXxhash128, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidChecksum(tt.args.checksum, tt.args.algorithm); got != tt.want { + t.Errorf("IsValidChecksum() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExtractMetadataFromFields(t *testing.T) { + tests := []struct { + name string + fields map[string]string + want map[string]string + wantErr error + }{ + { + name: "extracts metadata only", + fields: map[string]string{ + "x-amz-meta-owner": "alice", + "x-amz-meta-env": "prod", + "key": "uploads/report.pdf", + }, + want: map[string]string{ + "owner": "alice", + "env": "prod", + }, + }, + { + name: "returns empty map when no metadata fields exist", + fields: map[string]string{ + "key": "uploads/report.pdf", + "acl": "private", + "file": "ignored", + }, + want: map[string]string{}, + }, + { + name: "allows empty metadata value", + fields: map[string]string{ + "x-amz-meta-owner": "", + }, + want: map[string]string{ + "owner": "", + }, + }, + { + name: "metadata too large", + fields: map[string]string{ + "x-amz-meta-big": strings.Repeat("a", maxMetadataSize-len("big")+1), + }, + wantErr: s3err.GetMetadataTooLargeErr(2049, maxMetadataSize), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ExtractMetadataFromFields(tt.fields) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected error %v, got %v", tt.wantErr, err) + } + if err != nil { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("unexpected metadata: got %v want %v", got, tt.want) + } + }) + } +} + +func TestParseCalculatedChecksumFields(t *testing.T) { + tests := []struct { + name string + fields map[string]string + want ChecksumValues + wantErr error + }{ + { + name: "empty fields", + fields: map[string]string{ + "key": "uploads/file.txt", + }, + want: ChecksumValues{}, + }, + { + name: "single valid checksum field", + fields: map[string]string{ + "x-amz-checksum-crc32": "ww2FVQ==", + }, + want: ChecksumValues{ + types.ChecksumAlgorithmCrc32: "ww2FVQ==", + }, + }, + { + name: "ignores algorithm and type helper fields", + fields: map[string]string{ + "x-amz-checksum-algorithm": "CRC32", + "x-amz-checksum-type": "FULL_OBJECT", + "x-amz-checksum-crc32": "ww2FVQ==", + }, + want: ChecksumValues{ + types.ChecksumAlgorithmCrc32: "ww2FVQ==", + }, + }, + { + name: "invalid checksum field name", + fields: map[string]string{ + "x-amz-checksum-madeup": "abc", + }, + wantErr: s3err.GetAPIError(s3err.ErrInvalidChecksumHeader), + }, + { + name: "multiple checksum fields", + fields: map[string]string{ + "x-amz-checksum-crc32": "ww2FVQ==", + "x-amz-checksum-sha256": "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=", + }, + wantErr: s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders), + }, + { + name: "invalid checksum value", + fields: map[string]string{ + "x-amz-checksum-crc32": "invalid_base64_string", + }, + wantErr: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseCalculatedChecksumFields(tt.fields) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected error %v, got %v", tt.wantErr, err) + } + if err != nil { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("unexpected checksums: got %v want %v", got, tt.want) + } + }) + } +} + +func TestIsChecksumTypeValid(t *testing.T) { + type args struct { + t types.ChecksumType + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "valid_FULL_OBJECT", + args: args{ + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "valid_COMPOSITE", + args: args{ + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "invalid", + args: args{ + t: types.ChecksumType("invalid_checksum_type"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := IsChecksumTypeValid(tt.args.t); (err != nil) != tt.wantErr { + t.Errorf("IsChecksumTypeValid() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_checkChecksumTypeAndAlgo(t *testing.T) { + type args struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "full_object-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-crc32c", + args: args{ + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-sha1", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-sha256", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-crc64nvme", + args: args{ + algo: types.ChecksumAlgorithmCrc64nvme, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "composite-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-crc32c", + args: args{ + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-sha1", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-sha256", + args: args{ + algo: types.ChecksumAlgorithmSha256, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-crc64nvme", + args: args{ + algo: types.ChecksumAlgorithmCrc64nvme, + t: types.ChecksumTypeComposite, + }, + wantErr: true, + }, + { + name: "full_object-sha512", + args: args{ + algo: types.ChecksumAlgorithmSha512, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-md5", + args: args{ + algo: types.ChecksumAlgorithmMd5, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-xxhash64", + args: args{ + algo: types.ChecksumAlgorithmXxhash64, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-xxhash3", + args: args{ + algo: types.ChecksumAlgorithmXxhash3, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-xxhash128", + args: args{ + algo: types.ChecksumAlgorithmXxhash128, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "composite-sha512", + args: args{ + algo: types.ChecksumAlgorithmSha512, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-md5", + args: args{ + algo: types.ChecksumAlgorithmMd5, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-xxhash64", + args: args{ + algo: types.ChecksumAlgorithmXxhash64, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-xxhash3", + args: args{ + algo: types.ChecksumAlgorithmXxhash3, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-xxhash128", + args: args{ + algo: types.ChecksumAlgorithmXxhash128, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-empty", + args: args{ + t: types.ChecksumTypeComposite, + }, + wantErr: true, + }, + { + name: "full_object-empty", + args: args{ + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := checkChecksumTypeAndAlgo(tt.args.algo, tt.args.t); (err != nil) != tt.wantErr { + t.Errorf("checkChecksumTypeAndAlgo() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestParseTagging(t *testing.T) { + genRandStr := func(lgth int) string { + b := make([]byte, lgth) + for i := range b { + b[i] = byte(rand.Intn(95) + 32) // 126 - 32 + 1 = 95 printable characters + } + return string(b) + } + getTagSet := func(lgth int) s3response.Tagging { + res := s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{}, + }, + } + + for range lgth { + res.TagSet.Tags = append(res.TagSet.Tags, s3response.Tag{ + Key: genRandStr(10), + Value: genRandStr(20), + }) + } + + return res + } + type args struct { + data s3response.Tagging + overrideXML []byte + limit TagLimit + } + tests := []struct { + name string + args args + want map[string]string + wantErr error + }{ + { + name: "valid tags within limit", + args: args{ + data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + {Key: "key1", Value: "value1"}, + {Key: "key2", Value: "value2"}, + }, + }, + }, + limit: TagLimitObject, + }, + want: map[string]string{"key1": "value1", "key2": "value2"}, + wantErr: nil, + }, + { + name: "malformed XML", + args: args{ + overrideXML: []byte("invalid xml"), + limit: TagLimitObject, + }, + want: nil, + wantErr: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + { + name: "valid tags without namespace", + args: args{ + overrideXML: []byte(`key1value1key2value2`), + limit: TagLimitObject, + }, + want: map[string]string{"key1": "value1", "key2": "value2"}, + wantErr: nil, + }, + { + name: "valid tags with namespace", + args: args{ + overrideXML: []byte(`key1value1key2value2`), + limit: TagLimitObject, + }, + want: map[string]string{"key1": "value1", "key2": "value2"}, + wantErr: nil, + }, + { + name: "exceeds bucket tag limit", + args: args{ + data: getTagSet(51), + limit: TagLimitBucket, + }, + want: nil, + wantErr: s3err.GetAPIError(s3err.ErrBucketTaggingLimited), + }, + { + name: "exceeds object tag limit", + args: args{ + data: getTagSet(11), + limit: TagLimitObject, + }, + want: nil, + wantErr: s3err.GetAPIError(s3err.ErrObjectTaggingLimited), + }, + { + name: "invalid 0 length tag key", + args: args{ + data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{{Key: "", Value: "value1"}}, + }, + }, + limit: TagLimitObject, + }, + want: nil, + wantErr: s3err.GetAPIError(s3err.ErrInvalidTagKey), + }, + { + name: "invalid long tag key", + args: args{ + data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{{Key: genRandStr(130), Value: "value1"}}, + }, + }, + limit: TagLimitObject, + }, + want: nil, + wantErr: s3err.GetAPIError(s3err.ErrInvalidTagKey), + }, + { + name: "invalid long tag value", + args: args{ + data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{{Key: "key", Value: genRandStr(257)}}, + }, + }, + limit: TagLimitBucket, + }, + want: nil, + wantErr: s3err.GetAPIError(s3err.ErrInvalidTagValue), + }, + { + name: "duplicate tag key", + args: args{ + data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + {Key: "key", Value: "value1"}, + {Key: "key", Value: "value2"}, + }, + }, + }, + limit: TagLimitObject, + }, + want: nil, + wantErr: s3err.GetAPIError(s3err.ErrDuplicateTagKey), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var data []byte + if tt.args.overrideXML != nil { + data = tt.args.overrideXML + } else { + var err error + data, err = xml.Marshal(tt.args.data) + if err != nil { + t.Fatalf("error marshalling input: %v", err) + } + } + got, err := ParseTagging(data, tt.args.limit) + + if !errors.Is(err, tt.wantErr) { + t.Errorf("expected error %v, got %v", tt.wantErr, err) + } + if err == nil && !reflect.DeepEqual(got, tt.want) { + t.Errorf("expected result %v, got %v", tt.want, got) + } + }) + } +} + +func TestConvertTaggingXMLToQueryString(t *testing.T) { + tests := []struct { + name string + data s3response.Tagging + rawXML []byte + want string + wantErr error + }{ + { + name: "success", + data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + {Key: "project", Value: "versity gw"}, + {Key: "team", Value: "storage"}, + }, + }, + }, + want: "project=versity+gw&team=storage", + }, + { + name: "parse error", + rawXML: []byte("not xml"), + wantErr: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var data []byte + if tt.rawXML != nil { + data = tt.rawXML + } else { + var err error + data, err = xml.Marshal(tt.data) + if err != nil { + t.Fatalf("error marshalling input: %v", err) + } + } + + got, err := ConvertTaggingXMLToQueryString(data) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected error %v, got %v", tt.wantErr, err) + } + if err != nil { + return + } + + if got != tt.want { + t.Fatalf("unexpected tagging string: got %q want %q", got, tt.want) + } + + values, err := url.ParseQuery(got) + if err != nil { + t.Fatalf("parse query: %v", err) + } + for _, tag := range tt.data.TagSet.Tags { + if values.Get(tag.Key) != tag.Value { + t.Fatalf("unexpected query value for %q: got %q want %q", tag.Key, values.Get(tag.Key), tag.Value) + } + } + }) + } +} + +func TestValidateCopySource(t *testing.T) { + tests := []struct { + name string + copysource string + err error + }{ + // invalid encoding + {"invalid encoding 1", "%", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%")}, + {"invalid encoding 2", "%2", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%2")}, + {"invalid encoding 3", "%G1", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%G1")}, + {"invalid encoding 4", "%1Z", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%1Z")}, + {"invalid encoding 5", "%0H", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%0H")}, + {"invalid encoding 6", "%XY", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%XY")}, + {"invalid encoding 7", "%E", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%E")}, + {"invalid encoding 8", "hello%", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "hello%")}, + {"invalid encoding 9", "%%", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%%")}, + {"invalid encoding 10", "%2Gmore", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%2Gmore")}, + {"invalid encoding 11", "100%%sure", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "100%%sure")}, + {"invalid encoding 12", "%#00", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%#00")}, + {"invalid encoding 13", "%0%0", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%0%0")}, + {"invalid encoding 14", "%?versionId=id", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceEncoding, "%?versionId=id")}, + // invalid bucket name + {"invalid bucket name 1", "168.200.1.255/obj/foo", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "168.200.1.255/obj/foo")}, + {"invalid bucket name 2", "/0000:0db8:85a3:0000:0000:8a2e:0370:7224/smth", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "/0000:0db8:85a3:0000:0000:8a2e:0370:7224/smth")}, + {"invalid bucket name 3", "", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "")}, + {"invalid bucket name 4", "//obj/foo", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "//obj/foo")}, + {"invalid bucket name 5", "//obj/foo?versionId=id", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "//obj/foo?versionId=id")}, + // invalid object name + {"invalid object name 1", "bucket/../foo", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, "../foo")}, + {"invalid object name 2", "bucket/", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, "")}, + {"invalid object name 3", "bucket", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, "")}, + {"invalid object name 4", "bucket/../foo/dir/../../../", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, "../foo/dir/../../../")}, + {"invalid object name 5", "bucket/.?versionId=smth", s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceObject, ".")}, + // success + {"no error 1", "bucket/object", nil}, + {"no error 2", "bucket/object/key", nil}, + {"no error 3", "bucket/4*&(*&(89765))", nil}, + {"no error 4", "bucket/foo/../bar", nil}, + {"no error 5", "bucket/foo/bar/baz?versionId=01BX5ZZKBKACTAV9WEVGEMMVRZ", nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateCopySource(tt.copysource) + assert.Equal(t, tt.err, err) + }) + } +} + +func TestHasAwsChunkedEncoding(t *testing.T) { + tests := []struct { + name string + contentEncoding string + want bool + }{ + {"empty", "", false}, + {"only aws-chunked", "aws-chunked", true}, + {"uppercase", "AWS-CHUNKED", true}, + {"padded", " aws-chunked ", true}, + {"alongside another coding", "aws-chunked, gzip", true}, + {"other coding only", "gzip", false}, + {"token is a prefix of another coding", "aws-chunked-custom", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, HasAwsChunkedEncoding(tt.contentEncoding)) + }) + } +} + +func TestParseContentEncoding(t *testing.T) { + tests := []struct { + name string + contentSha256 string + contentEncoding string + want string + }{ + // streaming: the token frames the request, so S3 drops it + {"streaming, only aws-chunked", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "aws-chunked", ""}, + {"streaming, aws-chunked first", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "aws-chunked,gzip", "gzip"}, + {"streaming, aws-chunked last", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", "gzip,aws-chunked", "gzip"}, + {"streaming, aws-chunked in the middle", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER", "deflate,aws-chunked,gzip", "deflate,gzip"}, + {"streaming, spaces around codings", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "aws-chunked, gzip", "gzip"}, + {"streaming, uppercase", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "AWS-CHUNKED,gzip", "gzip"}, + {"streaming, repeated", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "aws-chunked,aws-chunked", ""}, + {"streaming, empty coding dropped", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "gzip,,aws-chunked", "gzip"}, + {"streaming, token is a prefix of another coding", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "aws-chunked-custom", "aws-chunked-custom"}, + {"streaming, no aws-chunked", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "gzip", "gzip"}, + {"streaming, no content encoding", "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "", ""}, + // no payload type: a presigned request is signed as UNSIGNED-PAYLOAD and + // may still frame its body, so S3 drops the token + {"no payload type, only aws-chunked", "", "aws-chunked", ""}, + {"no payload type, aws-chunked first", "", "aws-chunked,gzip", "gzip"}, + {"no payload type, aws-chunked last", "", "gzip,aws-chunked", "gzip"}, + {"no payload type, uppercase", "", "AWS-CHUNKED", ""}, + {"no payload type, token is a prefix of another coding", "", "aws-chunked-custom", "aws-chunked-custom"}, + {"no payload type, no aws-chunked", "", "gzip", "gzip"}, + // a declared non streaming payload type: the token is a value the client + // chose, and S3 keeps it + {"hex payload keeps aws-chunked", "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", "aws-chunked", "aws-chunked"}, + {"hex payload keeps other codings", "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", "aws-chunked,gzip", "aws-chunked,gzip"}, + {"unsigned payload keeps aws-chunked", "UNSIGNED-PAYLOAD", "aws-chunked", "aws-chunked"}, + {"ecdsa streaming is not chunk decoded, so the value is kept", "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD", "aws-chunked", "aws-chunked"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + headers := http.Header{} + if tt.contentSha256 != "" { + headers.Set("X-Amz-Content-Sha256", tt.contentSha256) + } + if tt.contentEncoding != "" { + headers.Set("Content-Encoding", tt.contentEncoding) + } + + ctx := fiberCtxFromURL(t, http.MethodPut, "http://localhost/bucket/object", headers) + assert.Equal(t, tt.want, ParseContentEncoding(ctx)) + }) + } +} + +func TestSetRegionMismatchHeader(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + { + "signature v4 header region mismatch", + s3err.MalformedAuth.IncorrectRegion("eu-west-1", "us-east-1"), + "eu-west-1", + }, + { + "presigned url region mismatch", + s3err.QueryAuthErrors.IncorrectRegion("eu-west-1", "us-east-1"), + "eu-west-1", + }, + { + "post object region mismatch", + s3err.PostAuth.IncorrectRegion("creds", "eu-west-1", "us-east-1"), + "eu-west-1", + }, + // only the region mismatch carries a region, the rest of the errors + // must not report one + { + "malformed auth error without a region", + s3err.MalformedAuth.MissingSignature(), + "", + }, + { + "presigned url error without a region", + s3err.QueryAuthErrors.IncorrectService("", "iam"), + "", + }, + { + "invalid argument error without a region", + s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySourceBucket, "bucket"), + "", + }, + { + "unrelated s3 error", + s3err.GetAPIError(s3err.ErrNoSuchBucket), + "", + }, + { + "non s3 error", + errors.New("internal"), + "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := fiber.New().AcquireCtx(&fasthttp.RequestCtx{}) + + SetRegionMismatchHeader(ctx, tt.err) + + assert.Equal(t, tt.want, + string(ctx.Response().Header.Peek("x-amz-bucket-region"))) + }) + } +} diff --git a/tests/integration/ListBuckets.go b/tests/integration/ListBuckets.go new file mode 100644 index 0000000..178fa61 --- /dev/null +++ b/tests/integration/ListBuckets.go @@ -0,0 +1,435 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + "fmt" + "sort" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +func ListBuckets_as_user(s *S3Conf) error { + testName := "ListBuckets_as_user" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}} + for range 6 { + bckt := getBucketName() + + err := setup(s, bckt) + if err != nil { + return err + } + + buckets = append(buckets, types.Bucket{ + Name: &bckt, + BucketRegion: &s.awsRegion, + }) + } + + testuser := getUser("user") + + err := createUsers(s, []user{testuser}) + if err != nil { + return err + } + + bckts := []string{} + for i := range 3 { + bckts = append(bckts, *buckets[i].Name) + } + + err = changeBucketsOwner(s, bckts, testuser.access) + if err != nil { + return err + } + + userClient := s.getUserClient(testuser) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := userClient.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + if getString(out.Owner.ID) != testuser.access { + return fmt.Errorf("expected buckets owner to be %v, instead got %v", + testuser.access, getString(out.Owner.ID)) + } + if !compareBuckets(out.Buckets, buckets[:3]) { + return fmt.Errorf("expected list buckets result to be %v, instead got %v", + buckets[:3], out.Buckets) + } + + for _, elem := range buckets[1:] { + err = teardown(s, *elem.Name) + if err != nil { + return err + } + } + + return nil + }) +} + +func ListBuckets_as_admin(s *S3Conf) error { + testName := "ListBuckets_as_admin" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser, adminUser := getUser("user"), getUser("admin") + + err := createUsers(s, []user{testuser, adminUser}) + if err != nil { + return err + } + + adminClient := s.getUserClient(adminUser) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + initOut, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + ignore := make([]string, 0, len(initOut.Buckets)) + for _, b := range initOut.Buckets { + if *b.Name != bucket { + ignore = append(ignore, *b.Name) + } + } + + buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}} + for range 6 { + bckt := getBucketName() + + err := setup(s, bckt) + if err != nil { + return err + } + + buckets = append(buckets, types.Bucket{ + Name: &bckt, + BucketRegion: &s.awsRegion, + }) + } + + bckts := []string{} + for i := range 3 { + bckts = append(bckts, *buckets[i].Name) + } + + err = changeBucketsOwner(s, bckts, testuser.access) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + if getString(out.Owner.ID) != adminUser.access { + return fmt.Errorf("expected buckets owner to be %v, instead got %v", + adminUser.access, getString(out.Owner.ID)) + } + if !compareBuckets(out.Buckets, buckets, ignore...) { + return fmt.Errorf("expected list buckets result to be %v, instead got %v", + sprintBuckets(buckets), sprintBuckets(out.Buckets)) + } + + for _, elem := range buckets[1:] { + err = teardown(s, *elem.Name) + if err != nil { + return err + } + } + + return nil + }) +} + +func ListBuckets_with_prefix(s *S3Conf) error { + testName := "ListBuckets_with_prefix" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + prefix := "my-prefix-" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + prefixInitOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{ + Prefix: &prefix, + }) + cancel() + if err != nil { + return err + } + + ignore := make([]string, 0, len(prefixInitOut.Buckets)) + for _, b := range prefixInitOut.Buckets { + ignore = append(ignore, *b.Name) + } + + allBuckets, prefixedBuckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}, []types.Bucket{} + for i := range 5 { + bckt := getBucketName() + if i%2 == 0 { + bckt = prefix + bckt + } + + err := setup(s, bckt) + if err != nil { + return err + } + + allBuckets = append(allBuckets, types.Bucket{ + Name: &bckt, + BucketRegion: &s.awsRegion, + }) + + if i%2 == 0 { + prefixedBuckets = append(prefixedBuckets, types.Bucket{ + Name: &bckt, + BucketRegion: &s.awsRegion, + }) + } + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{ + Prefix: &prefix, + }) + cancel() + if err != nil { + return err + } + + if getString(out.Owner.ID) != s.awsID { + return fmt.Errorf("expected owner to be %v, instead got %v", + s.awsID, getString(out.Owner.ID)) + } + if getString(out.Prefix) != prefix { + return fmt.Errorf("expected prefix to be %v, instead got %v", + prefix, getString(out.Prefix)) + } + if !compareBuckets(out.Buckets, prefixedBuckets, ignore...) { + return fmt.Errorf("expected list buckets result to be %v, instead got %v", + prefixedBuckets, out.Buckets) + } + + for _, elem := range allBuckets[1:] { + err = teardown(s, *elem.Name) + if err != nil { + return err + } + } + + return nil + }) +} + +func ListBuckets_invalid_max_buckets(s *S3Conf) error { + testName := "ListBuckets_invalid_max_buckets" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + listBuckets := func(maxBuckets int32) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{ + MaxBuckets: &maxBuckets, + }) + cancel() + return err + } + + invMaxBuckets := int32(-3) + err := listBuckets(invMaxBuckets) + if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, fmt.Sprint(invMaxBuckets))); err != nil { + return err + } + + invMaxBuckets = 2000000 + err = listBuckets(invMaxBuckets) + if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgMaxBuckets, fmt.Sprint(invMaxBuckets))); err != nil { + return err + } + + return nil + }) +} + +func ListBuckets_truncated(s *S3Conf) error { + testName := "ListBuckets_truncated" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + newBuckets := []types.Bucket{} + for range 5 { + bckt := getBucketName() + + err := setup(s, bckt) + if err != nil { + return err + } + + newBuckets = append(newBuckets, types.Bucket{ + Name: &bckt, + BucketRegion: &s.awsRegion, + }) + } + + buckets := append(initOut.Buckets, newBuckets...) + sort.Slice(buckets, func(i, j int) bool { + return *buckets[i].Name < *buckets[j].Name + }) + + maxBuckets := int32(3) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{ + MaxBuckets: &maxBuckets, + }) + cancel() + if err != nil { + return err + } + + if getString(out.Owner.ID) != s.awsID { + return fmt.Errorf("expected owner to be %v, instead got %v", + s.awsID, getString(out.Owner.ID)) + } + if !compareBuckets(out.Buckets, buckets[:maxBuckets]) { + return fmt.Errorf("expected list buckets result to be %v, instead got %v", + sprintBuckets(buckets[:maxBuckets]), sprintBuckets(out.Buckets)) + } + if getString(out.ContinuationToken) != getString(buckets[maxBuckets-1].Name) { + return fmt.Errorf("expected ContinuationToken to be %v, instead got %v", + getString(buckets[maxBuckets-1].Name), getString(out.ContinuationToken)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.ListBuckets(ctx, &s3.ListBucketsInput{ + ContinuationToken: out.ContinuationToken, + }) + cancel() + if err != nil { + return err + } + + if !compareBuckets(out.Buckets, buckets[maxBuckets:]) { + return fmt.Errorf("expected list buckets result to be %v, instead got %v", + sprintBuckets(buckets[maxBuckets:]), sprintBuckets(out.Buckets)) + } + if out.ContinuationToken != nil { + return fmt.Errorf("expected nil continuation token, instead got %v", + *out.ContinuationToken) + } + if out.Prefix != nil { + return fmt.Errorf("expected nil prefix, instead got %v", *out.Prefix) + } + + for _, elem := range newBuckets { + err = teardown(s, *elem.Name) + if err != nil { + return err + } + } + + return nil + }) +} + +func ListBuckets_empty_success(s *S3Conf) error { + testName := "ListBuckets_empty_success" + return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + if len(out.Buckets) > 0 { + return fmt.Errorf("expected list buckets result to be %v, instead got %v", + []types.Bucket{}, sprintBuckets(out.Buckets)) + } + + return nil + }) +} + +func ListBuckets_success(s *S3Conf) error { + testName := "ListBuckets_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + ignore := make([]string, 0, len(initOut.Buckets)) + for _, b := range initOut.Buckets { + if *b.Name != bucket { + ignore = append(ignore, *b.Name) + } + } + + buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}} + for range 5 { + bckt := getBucketName() + + err := setup(s, bckt) + if err != nil { + return err + } + + buckets = append(buckets, types.Bucket{ + Name: &bckt, + BucketRegion: &s.awsRegion, + }) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + if getString(out.Owner.ID) != s.awsID { + return fmt.Errorf("expected owner to be %v, instead got %v", + s.awsID, getString(out.Owner.ID)) + } + if !compareBuckets(out.Buckets, buckets, ignore...) { + return fmt.Errorf("expected list buckets result to be %v, instead got %v", + sprintBuckets(buckets), sprintBuckets(out.Buckets)) + } + + for _, elem := range buckets[1:] { + err = teardown(s, *elem.Name) + if err != nil { + return err + } + } + + return nil + }) +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 7dd4b5e..fb99680 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -546,10 +546,12 @@ func TestListMultipartUploads(ts *TestState) { ts.Run(ListMultipartUploads_empty_result) ts.Run(ListMultipartUploads_invalid_max_uploads) ts.Run(ListMultipartUploads_max_uploads) + ts.Run(ListMultipartUploads_next_markers_not_truncated) ts.Run(ListMultipartUploads_exceeding_max_uploads) ts.Run(ListMultipartUploads_ignore_upload_id_marker) ts.Run(ListMultipartUploads_invalid_uploadId_marker) ts.Run(ListMultipartUploads_keyMarker_not_from_list) + ts.Run(ListMultipartUploads_upload_id_marker_with_key_marker) ts.Run(ListMultipartUploads_delimiter_truncated) ts.Run(ListMultipartUploads_prefix) ts.Run(ListMultipartUploads_both_delimiter_and_prefix) @@ -2191,9 +2193,11 @@ func TestRouter(ts *TestState) { ts.Run(RouterGetUploadsWithKey) ts.Run(RouterCopySourceNotAllowed) ts.Run(RouterListVersionsWithKey) + ts.Run(RouterUploadsWithUploadId) ts.Run(RouterObjectAnnotationNotImplemented) ts.Run(RouterRenameObjectNotImplemented) ts.Run(RouterObjectTorrentMethodNotAllowed) + ts.Run(RouterInvalidURLEscape) } func TestPostObject(ts *TestState) { @@ -3282,10 +3286,12 @@ func GetIntTests() IntTests { "ListMultipartUploads_empty_result": ListMultipartUploads_empty_result, "ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads, "ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads, + "ListMultipartUploads_next_markers_not_truncated": ListMultipartUploads_next_markers_not_truncated, "ListMultipartUploads_exceeding_max_uploads": ListMultipartUploads_exceeding_max_uploads, "ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker, "ListMultipartUploads_invalid_uploadId_marker": ListMultipartUploads_invalid_uploadId_marker, "ListMultipartUploads_keyMarker_not_from_list": ListMultipartUploads_keyMarker_not_from_list, + "ListMultipartUploads_upload_id_marker_with_key_marker": ListMultipartUploads_upload_id_marker_with_key_marker, "ListMultipartUploads_delimiter_truncated": ListMultipartUploads_delimiter_truncated, "ListMultipartUploads_prefix": ListMultipartUploads_prefix, "ListMultipartUploads_both_delimiter_and_prefix": ListMultipartUploads_both_delimiter_and_prefix, @@ -3794,9 +3800,11 @@ func GetIntTests() IntTests { "RouterGetUploadsWithKey": RouterGetUploadsWithKey, "RouterCopySourceNotAllowed": RouterCopySourceNotAllowed, "RouterListVersionsWithKey": RouterListVersionsWithKey, + "RouterUploadsWithUploadId": RouterUploadsWithUploadId, "RouterObjectAnnotationNotImplemented": RouterObjectAnnotationNotImplemented, "RouterRenameObjectNotImplemented": RouterRenameObjectNotImplemented, "RouterObjectTorrentMethodNotAllowed": RouterObjectTorrentMethodNotAllowed, + "RouterInvalidURLEscape": RouterInvalidURLEscape, "UnsignedStreaminPayloadTrailer_malformed_trailer": UnsignedStreaminPayloadTrailer_malformed_trailer, "UnsignedStreamingPayloadTrailer_aborted_connection": UnsignedStreamingPayloadTrailer_aborted_connection, "UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length": UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length, diff --git a/tests/test_rest_list_buckets.sh b/tests/test_rest_list_buckets.sh new file mode 100644 index 0000000..7ca63ad --- /dev/null +++ b/tests/test_rest_list_buckets.sh @@ -0,0 +1,385 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/commands/list_buckets.sh +source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/list_buckets/list_buckets_rest.sh +source ./tests/drivers/user.sh +source ./tests/logger.sh +source ./tests/setup.sh + +export RUN_USERS=true + +# tags: openssl,malformed-message +@test "REST - empty message" { + test_file="test_file" + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1249" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_and_file_v2 "$bucket_name" "$test_file" + assert_success + + echo -en "\r\n" > "$TEST_FILE_FOLDER/empty.txt" + run send_via_openssl_with_timeout "$TEST_FILE_FOLDER/empty.txt" + assert_success +} + +# tags: malformed-message,openssl +@test "REST - deformed message" { + test_file="test_file" + echo -en "abcdefg\r\n\r\n" > "$TEST_FILE_FOLDER/deformed.txt" + run send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/deformed.txt" 400 "BadRequest" "An error occurred when parsing the HTTP request." + assert_success +} + +# tags: curl,ListBuckets,required-headers,Authorization,invalid-header +@test "REST - invalid authorization scheme" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1705" + fi + run list_buckets_check_authorization_scheme_error + assert_success +} + +# tags: curl,ListBuckets,required-headers,Authorization,invalid-header +@test "REST - very invalid credential string" { + run send_rest_go_command_expect_error "400" "AuthorizationHeaderMalformed" "the Credential is mal-formed" "-incorrectCredential" "Credentials" + assert_success +} + +# tags: curl,ListBuckets,required-headers,Authorization,invalid-header +@test "REST - nonexistent key ID" { + run send_rest_go_command_expect_error "403" "InvalidAccessKeyId" "does not exist" "-awsAccessKeyId" "dummy" + assert_success +} + +# tags: curl,ListBuckets,required-headers,Authorization,x-amz-date,invalid-header +@test "REST - invalid year/month/day" { + run send_rest_go_command_expect_error "400" "AuthorizationHeaderMalformed" "incorrect date format" "-invalidYearMonthDay" + assert_success +} + +# tags: curl,ListBuckets,required-headers,Authorization,x-amz-date,invalid-header +@test "REST - incorrect year/month/day" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1514" + fi + run list_buckets_check_request_time_too_skewed_error + assert_success +} + +# tags: curl,ListBuckets,required-headers,Authorization,invalid-header +@test "REST - invalid region" { + run send_rest_go_command_expect_error "400" "AuthorizationHeaderMalformed" "us-eest-1" "-awsRegion" "us-eest-1" + assert_success +} + +# tags: curl,ListBuckets,required-headers,Authorization,invalid-header +@test "REST - invalid service name" { + run send_rest_go_command_expect_error "400" "AuthorizationHeaderMalformed" "incorrect service" "-serviceName" "s2" + assert_success +} + +# tags: curl,ListBuckets,required-headers,Authorization,invalid-header +@test "REST - incorrect signature" { + run send_rest_go_command_expect_error "403" "SignatureDoesNotMatch" "does not match" "-incorrectSignature" + assert_success +} + +# tags: openssl,ListBuckets,required-headers,host,invalid-header +@test "REST - missing host parameter" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1530" + fi + run send_openssl_go_command "400" "-missingHostParam" + assert_success +} + +# tags: curl,ListBuckets,minimal-request +@test "test_rest_list_buckets" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run list_check_buckets_rest "$bucket_name" + assert_success +} + +# tags: curl,ListBuckets,max-buckets,continuation-token +@test "REST - list buckets - continuation token isn't bucket name" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1399" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + # shellcheck disable=SC2153 + run get_bucket_name "$BUCKET_TWO_NAME" + assert_success + bucket_two_name="$output" + + run setup_buckets_v2 "$bucket_name" "$bucket_two_name" + assert_success + + run check_continuation_token + assert_success +} + +# tags: curl,ListBuckets,max-buckets,continuation-token +@test "REST - list buckets - success (multiple pages)" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run get_bucket_name "$BUCKET_TWO_NAME" + assert_success + bucket_two_name="$output" + + run setup_buckets_v2 "$bucket_name" "$bucket_two_name" + assert_success + + run check_for_buckets_with_multiple_pages "$bucket_name" "$bucket_two_name" + assert_success +} + +# tags: curl,ListBuckets,prefix +@test "REST - list buckets w/prefix" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run get_bucket_name "$BUCKET_TWO_NAME" + assert_success + bucket_two_name="$output" + + run setup_buckets_v2 "$bucket_name" "$bucket_two_name" + assert_success + + run list_check_buckets_rest "$bucket_name" "$bucket_two_name" + assert_success + + run list_check_buckets_rest_with_prefix "$bucket_name" + assert_success + + run list_check_buckets_rest_with_prefix "$bucket_two_name" + assert_success +} + +# tags: curl,ListBuckets,minimal-request,user +@test "REST - ListBuckets - correct buckets show up" { + if [ "$SKIP_USERS_TESTS" == "true" ]; then + skip "skip versitygw-specific users tests" + fi + if [ "$DIRECT" == "true" ]; then + skip "https://github.com/versity/versitygw/issues/1704" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_and_user "$bucket_name" "$USERNAME_ONE" "$PASSWORD_ONE" "user" + assert_success + username=${lines[${#lines[@]}-2]} + password=${lines[${#lines[@]}-1]} + + run get_bucket_name "$BUCKET_TWO_NAME" + assert_success + bucket_two_name="$output" + + run setup_bucket "$bucket_two_name" + assert_success + + run change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$bucket_two_name" "$username" + assert_success + + log 5 "username: $username, password: $password" + run list_check_buckets_user "$username" "$password" "$bucket_two_name" + assert_success +} + +# tags: curl,invalid-method +@test "REST - service route - invalid POST route" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1810" + fi + run get_file_name + assert_success + file_name=$output + + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_and_add_file "$bucket_name" "$file_name" + assert_success + + run send_rest_go_command_expect_error_with_specific_arg_names_values "405" "MethodNotAllowed" "is not allowed" 4 "Method" "POST" "ResourceType" "SERVICE" "-method" "POST" + assert_success +} + +# tags: curl,invalid-method +@test "REST - service route - invalid method" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1846" + fi + run send_rest_go_command_expect_error "400" "BadRequest" "An error occurred when parsing the HTTP request" "-method" "GETS" + assert_success +} + +# tags: curl,ListBuckets,required-headers,x-amz-date +@test "REST - ListBuckets - error Content-Type is application/xml" { + run send_rest_go_command_check_header_key_and_value "403" "Content-Type" "application/xml" "-method" "GET" "-omitDate" + assert_success +} + +# tags: curl,ListBuckets,bucket-region,invalid-query +@test "REST - ListBuckets - invalid bucket-region query" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1923" + fi + local invalid_region="abc" + + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + local bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error_with_specific_arg_name_value "400" "InvalidArgument" "Argument value $invalid_region is not a valid AWS Region" \ + "ArgumentName" "bucket-region" "-query" "bucket-region=$invalid_region" + assert_success +} + +# tags: curl,ListBuckets,bucket-region +@test "REST - ListBuckets - incorrect bucket region" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1930" + fi + local test_region="us-east-1" + if [ "$AWS_REGION" == "us-east-1" ]; then + test_region="us-west-1" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + local params=() + if [ "$DIRECT" == "true" ]; then + params+=("-url" "https://s3.$test_region.amazonaws.com" "-awsRegion" "$test_region") + fi + + run list_buckets_bucket_not_in_list "$bucket_name" "$test_region" "${params[@]}" + assert_success +} + +# tags: curl,ListBuckets,bucket-region +@test "REST - ListBuckets - correct bucket region" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_callback "200" "check_bucket_and_region" "-query" "bucket-region=$AWS_REGION" "--" "$bucket_name" "$AWS_REGION" + assert_success +} + +# tags: curl,ListBuckets,required-headers,x-amz-content-sha256 +@test "REST - ListBuckets - missing sha256 hash" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidRequest" "Missing required header for this request: x-amz-content-sha256" "-omitSHA256Hash" + assert_success +} + +# tags: curl,ListBuckets,required-headers,x-amz-content-sha256 +@test "REST - ListBuckets - invalid hash type" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidArgument" "x-amz-content-sha256 must be" "-customSHA256Hash" "ABCDEFG" + assert_success +} + +# tags: curl,ListBuckets,required-headers,x-amz-content-sha256 +@test "REST - ListBuckets - non-matching hash type" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidRequest" "The value of x-amz-content-sha256 header is invalid" "-customSHA256Hash" "STREAMING-UNSIGNED-PAYLOAD-TRAILER" + assert_success +} + +# tags: curl,ListBuckets,required-headers,x-amz-date +@test "REST - ListBuckets - omit date" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1934" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "403" "AccessDenied" "AWS authentication requires a valid Date or x-amz-date header" "-omitDate" + assert_success +} + +# tags: curl,ListBuckets,required-headers,x-amz-date +@test "REST - ListBuckets - invalid date" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1934" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "403" "AccessDenied" "AWS authentication requires a valid Date or x-amz-date header" "-customDate" "ABCDEFG" + assert_success +}