From 09e1b657241f043631f4f92f53db497f74d2fc91 Mon Sep 17 00:00:00 2001 From: upstream Date: Wed, 7 Oct 2026 19:38:54 +0200 Subject: [PATCH] Upstream versity/versitygw state of the touched files before 5128cc1624 --- s3api/controllers/bucket-get_test.go | 1383 ++++++++++++++++++++++++++ s3err/invalid-argument.go | 17 + tests/integration/ListObjects.go | 794 +++++++++++++++ tests/integration/group-tests.go | 26 +- tests/test_rest_list_objects.sh | 369 +++++++ 5 files changed, 2587 insertions(+), 2 deletions(-) create mode 100644 s3api/controllers/bucket-get_test.go create mode 100644 tests/integration/ListObjects.go create mode 100644 tests/test_rest_list_objects.sh diff --git a/s3api/controllers/bucket-get_test.go b/s3api/controllers/bucket-get_test.go new file mode 100644 index 0000000..ac29089 --- /dev/null +++ b/s3api/controllers/bucket-get_test.go @@ -0,0 +1,1383 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_GetBucketTagging(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: map[string]string{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: map[string]string{ + "key": "val", + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + {Key: "key", Value: "val"}, + }, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) { + return tt.input.beRes.(map[string]string), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketTagging, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketOwnershipControls(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: types.ObjectOwnership(""), + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.OwnershipControls{ + Rules: []types.OwnershipControlsRule{{}}, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: types.ObjectOwnershipBucketOwnerEnforced, + }, + output: testOutput{ + response: &Response{ + Data: s3response.OwnershipControls{ + Rules: []types.OwnershipControlsRule{{ObjectOwnership: types.ObjectOwnershipBucketOwnerEnforced}}, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) { + return tt.input.beRes.(types.ObjectOwnership), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketOwnershipControls, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketVersioning(t *testing.T) { + status := types.BucketVersioningStatusEnabled + validRes := s3response.GetBucketVersioningOutput{ + Status: &status, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "non owner user with verified access", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: false, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "user", + Role: auth.RoleUser, + }, + utils.ContextKeyPublicBucket: true, + }, + beRes: validRes, + }, + output: testOutput{ + response: &Response{ + Data: validRes, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.GetBucketVersioningOutput{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.GetBucketVersioningOutput{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + }, + output: testOutput{ + response: &Response{ + Data: validRes, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { + return tt.input.beRes.(s3response.GetBucketVersioningOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketVersioning, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketCors(t *testing.T) { + cors := &auth.CORSConfiguration{ + Rules: []auth.CORSRule{ + { + AllowedOrigins: []auth.CORSOrigin{"origin"}, + AllowedMethods: []auth.CORSHTTPMethod{http.MethodPut}, + AllowedHeaders: []auth.CORSHeader{"X-Amz-Date"}, + }, + }, + } + beRes, err := xml.Marshal(cors) + assert.NoError(t, err) + + var nilResp *auth.CORSConfiguration + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "invalid data from backend", + input: testInput{ + locals: defaultLocals, + beRes: []byte("invalid_data"), + }, + output: testOutput{ + response: &Response{ + Data: nilResp, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: errors.New("failed to parse cors config:"), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: beRes, + }, + output: testOutput{ + response: &Response{ + Data: cors, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketCorsFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketCors, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketPolicy(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrAccessDenied), + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: []byte{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: []byte("mock_policy_resp"), + }, + output: testOutput{ + response: &Response{ + Data: []byte("mock_policy_resp"), + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketPolicy, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketPolicyStatus(t *testing.T) { + mockResp := ` + { + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": ["s3:GetObject"], + "Resource": "arn:aws:s3:::test/*" + } + ] + } + ` + isPublic := true + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrAccessDenied), + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "fails to get bucket policy", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "fails to parse bucket policy", + input: testInput{ + locals: defaultLocals, + beRes: []byte("invalid policy"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.APIError{ + Code: "MalformedPolicy", + Description: "This policy contains invalid Json", + HTTPStatusCode: http.StatusBadRequest, + }, + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: []byte(mockResp), + }, + output: testOutput{ + response: &Response{ + Data: types.PolicyStatus{ + IsPublic: &isPublic, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketPolicyStatus, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_ListObjectVersions(t *testing.T) { + listVersionsResult := s3response.ListVersionsResult{ + Name: utils.GetStringPtr("name"), + Prefix: utils.GetStringPtr("prefix"), + Delimiter: utils.GetStringPtr("delim"), + Versions: []s3response.ObjectVersion{ + {Key: utils.GetStringPtr("my-key")}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid max keys", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-keys": "invalid", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgMaxLimiter(string(utils.LimiterTypeMaxKeys), "invalid"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListVersionsResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListVersionsResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listVersionsResult, + }, + output: testOutput{ + response: &Response{ + Data: listVersionsResult, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + return tt.input.beRes.(s3response.ListVersionsResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListObjectVersions, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetObjectLockConfiguration(t *testing.T) { + cfgBytes, err := json.Marshal( + auth.BucketLockConfig{ + Enabled: true, + }) + assert.NoError(t, err) + + var lockCfg *types.ObjectLockConfiguration + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "invalid data from backend", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + }, + output: testOutput{ + response: &Response{ + Data: lockCfg, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: fmt.Errorf("parse object lock config: "), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: cfgBytes, + }, + output: testOutput{ + response: &Response{ + Data: &types.ObjectLockConfiguration{ + ObjectLockEnabled: types.ObjectLockEnabledEnabled, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectLockConfiguration, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketAcl(t *testing.T) { + aclBytes, err := json.Marshal( + auth.ACL{ + Owner: "root", + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "invalid data from backend", + input: testInput{ + locals: defaultLocals, + beRes: []byte{'d'}, + }, + output: testOutput{ + response: &Response{ + Data: auth.GetBucketAclOutput{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: fmt.Errorf("parse acl: "), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: aclBytes, + }, + output: testOutput{ + response: &Response{ + Data: auth.GetBucketAclOutput{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("root"), + }, + AccessControlList: auth.AccessControlList{ + Grants: []auth.Grant{}, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketAcl, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_ListMultipartUploads(t *testing.T) { + listMpResult := s3response.ListMultipartUploadsResult{ + Prefix: "prefix", + Delimiter: "delim", + Bucket: "bucket", + Uploads: []s3response.Upload{ + {Key: "my-key"}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid max uploads", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-uploads": "invalid", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgMaxLimiter(string(utils.LimiterTypeMaxUploads), "invalid"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListMultipartUploadsResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListMultipartUploadsResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listMpResult, + }, + output: testOutput{ + response: &Response{ + Data: listMpResult, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + return tt.input.beRes.(s3response.ListMultipartUploadsResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListMultipartUploads, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_ListObjectsV2(t *testing.T) { + listV2Result := s3response.ListObjectsV2Result{ + Name: utils.GetStringPtr("name"), + Prefix: utils.GetStringPtr("prefix"), + Delimiter: utils.GetStringPtr("delim"), + Contents: []s3response.Object{ + {Key: utils.GetStringPtr("my-key")}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "negative max keys", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-keys": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgNegativeMaxLimiter(string(utils.LimiterTypeMaxKeys), "-1"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListObjectsV2Result{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listV2Result, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-bucket-region": utils.GetStringPtr(defaultRegion), + }, + Data: listV2Result, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { + return tt.input.beRes.(s3response.ListObjectsV2Result), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListObjectsV2, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_ListObjects(t *testing.T) { + listResult := s3response.ListObjectsResult{ + Name: utils.GetStringPtr("name"), + Prefix: utils.GetStringPtr("prefix"), + Delimiter: utils.GetStringPtr("delim"), + Contents: []s3response.Object{ + {Key: utils.GetStringPtr("my-key")}, + }, + } + // The controller echoes the request marker back, so the element is in + // the response even when the backend leaves it unset. utils.GetStringPtr + // collapses "" to nil, which is the behaviour under test here, so take + // the address of an empty string directly. + emptyMarker := "" + listResultEchoed := listResult + listResultEchoed.Marker = &emptyMarker + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid max keys", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-keys": "bla", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgMaxLimiter(string(utils.LimiterTypeMaxKeys), "bla"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListObjectsResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listResult, + }, + output: testOutput{ + response: &Response{ + Data: listResultEchoed, + Headers: map[string]*string{ + "x-amz-bucket-region": utils.GetStringPtr(defaultRegion), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { + return tt.input.beRes.(s3response.ListObjectsResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListObjects, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetBucketLocation(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response us-east-1", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + Data: s3response.LocationConstraint{ + Value: nil, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + { + name: "successful response", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + utils.ContextKeyRegion: "us-east-2", + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.LocationConstraint{ + Value: utils.GetStringPtr("us-east-2"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + HeadBucketFunc: func(contextMoqParam context.Context, headBucketInput *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { + return &s3.HeadBucketOutput{}, tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketLocation, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} diff --git a/s3err/invalid-argument.go b/s3err/invalid-argument.go index 7bf61cd..27f992f 100644 --- a/s3err/invalid-argument.go +++ b/s3err/invalid-argument.go @@ -74,9 +74,14 @@ const ( InvalidArgSSEInvalidEncryptionMethod InvalidArgSSECRequiresTLS InvalidArgUploadIdWithUploads + InvalidArgBucketRegionEndpoint ) var invalidArgErrResponses = map[InvalidArgErrorCode]InvalidArgumentError{ + InvalidArgBucketRegionEndpoint: { + Description: "Requests with bucket-region specified must be made to the corresponding regional endpoint", + ArgumentName: "bucket-region", + }, InvalidArgMaxBuckets: { Description: "Argument max-buckets must be an integer between 1 and 10000.", ArgumentName: "max-buckets", @@ -384,3 +389,15 @@ func GetInvalidArgObjectOwnership(value string) InvalidArgumentError { Description: fmt.Sprintf("Invalid x-amz-object-ownership header: %s", value), } } + +func GetInvalidArgBucketRegion(value string) InvalidArgumentError { + description := fmt.Sprintf("Argument value %s is not a valid AWS Region", value) + if value == "" { + description = "Argument value is not a valid AWS Region" + } + return InvalidArgumentError{ + ArgumentName: "bucket-region", + // no ArgumentValue is returned for this error + Description: description, + } +} diff --git a/tests/integration/ListObjects.go b/tests/integration/ListObjects.go new file mode 100644 index 0000000..31dfead --- /dev/null +++ b/tests/integration/ListObjects.go @@ -0,0 +1,794 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +func ListObjects_non_existing_bucket(s *S3Conf) error { + testName := "ListObjects_non_existing_bucket" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bckt := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bckt, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchBucket"); err != nil { + return err + } + return nil + }) +} + +func ListObjects_with_prefix(s *S3Conf) error { + testName := "ListObjects_with_prefix" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + prefix := "obj" + objWithPrefix := []string{prefix + "/bar", prefix + "/baz/bla", prefix + "/foo"} + contents, err := putObjects(s3client, append(objWithPrefix, []string{"azy/csf", "hell"}...), bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Prefix: &prefix, + }) + cancel() + if err != nil { + return err + } + + if getString(out.Prefix) != prefix { + return fmt.Errorf("expected prefix %v, instead got %v", + prefix, getString(out.Prefix)) + } + if !compareObjects(contents[2:], out.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", + contents[2:], out.Contents) + } + + return nil + }) +} + +func ListObjects_paginated(s *S3Conf) error { + testName := "ListObjects_paginated" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + _, err := putObjects(s3client, []string{"dir1/subdir/file.txt", "dir1/subdir.ext", "dir1/subdir1.ext", "dir1/subdir2.ext"}, bucket) + if err != nil { + return err + } + + objs, prefixes, err := listObjects(s3client, bucket, "dir1/", "/", 2) + if err != nil { + return err + } + + expected := []string{"dir1/subdir.ext", "dir1/subdir1.ext", "dir1/subdir2.ext"} + if !hasObjNames(objs, expected) { + return fmt.Errorf("expected objects %v, instead got %v", + expected, objStrings(objs)) + } + + expectedPrefix := []string{"dir1/subdir/"} + if !hasPrefixName(prefixes, expectedPrefix) { + return fmt.Errorf("expected prefixes %v, instead got %v", + expectedPrefix, pfxStrings(prefixes)) + } + + return nil + }) +} + +func ListObjects_truncated(s *S3Conf) error { + testName := "ListObjects_truncated" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + maxKeys := int32(2) + contents, err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out1, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + }) + cancel() + if err != nil { + return err + } + + if out1.IsTruncated == nil || !*out1.IsTruncated { + return fmt.Errorf("expected output to be truncated") + } + + if out1.MaxKeys == nil { + return fmt.Errorf("expected non nil max-keys") + } + if *out1.MaxKeys != maxKeys { + return fmt.Errorf("expected max-keys to be %v, instead got %v", + maxKeys, out1.MaxKeys) + } + + if out1.NextMarker == nil { + return fmt.Errorf("expected non nil next marker") + } + if *out1.NextMarker != "baz" { + return fmt.Errorf("expected next-marker to be baz, instead got %v", + *out1.NextMarker) + } + + if !compareObjects(contents[:2], out1.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", + contents[:2], out1.Contents) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out2, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Marker: out1.NextMarker, + }) + cancel() + if err != nil { + return err + } + + if out2.IsTruncated == nil { + return fmt.Errorf("expected non nil is-truncated") + } + if *out2.IsTruncated { + return fmt.Errorf("expected output not to be truncated") + } + + if getString(out2.Marker) != getString(out1.NextMarker) { + return fmt.Errorf("expected marker to be %v, instead got %v", + getString(out1.NextMarker), getString(out2.Marker)) + } + + if !compareObjects(contents[2:], out2.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", + contents[2:], out2.Contents) + } + return nil + }) +} + +func ListObjects_invalid_max_keys(s *S3Conf) error { + testName := "ListObjects_invalid_max_keys" + maxKeys := int32(-5) + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + }) + cancel() + if err := checkApiErr(err, s3err.GetInvalidArgNegativeMaxLimiter("max-keys", fmt.Sprint(maxKeys))); err != nil { + return err + } + + return nil + }) +} + +func ListObjects_max_keys_0(s *S3Conf) error { + testName := "ListObjects_max_keys_0" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objects := []string{"foo", "bar", "baz"} + _, err := putObjects(s3client, objects, bucket) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + maxKeys := int32(0) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + }) + cancel() + if err != nil { + return nil + } + + if len(out.Contents) > 0 { + return fmt.Errorf("unexpected output for list objects with max-keys 0") + } + + return nil + }) +} + +func ListObjects_exceeding_max_keys(s *S3Conf) error { + testName := "ListObjects_exceeding_max_keys" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + maxKeys := int32(233333333) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + }) + cancel() + if err != nil { + return nil + } + + if out.MaxKeys == nil { + return fmt.Errorf("unexpected nil max-keys") + } + if *out.MaxKeys != 1000 { + return fmt.Errorf("expected the max-keys to be %v, instaed got %v", + 1000, *out.MaxKeys) + } + + return nil + }) +} + +func ListObjects_delimiter(s *S3Conf) error { + testName := "ListObjects_delimiter" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + _, err := putObjects(s3client, []string{"foo/bar/baz", "foo/bar/xyzzy", "quux/thud", "asdf"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Delimiter: getPtr("/"), + }) + cancel() + if err != nil { + return err + } + + if getString(out.Delimiter) != "/" { + return fmt.Errorf("expected delimiter to be /, instead got %v", + getString(out.Delimiter)) + } + if len(out.Contents) != 1 || getString(out.Contents[0].Key) != "asdf" { + return fmt.Errorf("expected result [\"asdf\"], instead got %v", + out.Contents) + } + + if !comparePrefixes([]string{"foo/", "quux/"}, out.CommonPrefixes) { + return fmt.Errorf("expected common prefixes to be %v, instead got %v", + []string{"foo/", "quux/"}, out.CommonPrefixes) + } + + return nil + }) +} + +func ListObjects_max_keys_none(s *S3Conf) error { + testName := "ListObjects_max_keys_none" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + _, err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if out.MaxKeys == nil { + return fmt.Errorf("expected non nil max-keys") + } + if *out.MaxKeys != 1000 { + return fmt.Errorf("expected max-keys to be 1000, instead got %v", + out.MaxKeys) + } + + return nil + }) +} + +func ListObjects_marker_not_from_obj_list(s *S3Conf) error { + testName := "ListObjects_marker_not_from_obj_list" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents, err := putObjects(s3client, []string{"foo", "bar", "baz", "qux", "hello", "xyz"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Marker: getPtr("ceil"), + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(contents[2:], out.Contents) { + return fmt.Errorf("expected output to be %v, instead got %v", + contents, out.Contents) + } + + return nil + }) +} + +func ListObjects_with_checksum(s *S3Conf) error { + testName := "ListObjects_with_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents := []types.Object{} + for i, el := range types.ChecksumAlgorithmCrc32.Values() { + key := fmt.Sprintf("obj-%v", i) + size := int64(i * 30) + out, err := putObjectWithData(size, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: el, + }, s3client, withPutObjectChecksumAlgo(el)) + if err != nil { + return err + } + + contents = append(contents, types.Object{ + Key: &key, + ETag: out.res.ETag, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{ + el, + }, + ChecksumType: out.res.ChecksumType, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(contents, res.Contents) { + return fmt.Errorf("expected the objects list to be %v, instead got %v", + contents, res.Contents) + } + + return nil + }) +} + +func ListObjects_list_all_objs(s *S3Conf) error { + testName := "ListObjects_list_all_objs" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents, err := putObjects(s3client, []string{"foo", "bar", "baz", "quxx/ceil", "ceil", "hello/world"}, bucket) + if err != nil { + return err + } + + // Test 1: List all objects without pagination + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + // S3 echoes the request marker back, so the response carries the + // element with an empty value rather than omitting it + if out.Marker == nil { + return fmt.Errorf("expected the Marker to be an empty string, instead got nil") + } + if *out.Marker != "" { + return fmt.Errorf("expected the Marker to be an empty string, instead got %v", + *out.Marker) + } + if out.NextMarker != nil { + return fmt.Errorf("expected the NextMarker to be nil, instead got %v", + *out.NextMarker) + } + if out.Delimiter != nil { + return fmt.Errorf("expected the Delimiter to be nil, instead got %v", + *out.Delimiter) + } + if out.Prefix != nil { + return fmt.Errorf("expected the Prefix to be nil, instead got %v", + *out.Prefix) + } + + if !compareObjects(contents, out.Contents) { + return fmt.Errorf("expected the contents to be %v, instead got %v", + contents, out.Contents) + } + + // Test 2: List all objects with pagination using ListObjectsV2 + var marker *string + var allObjects []types.Object + maxKeys := int32(2) + + for { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + Marker: marker, + }) + cancel() + if err != nil { + return err + } + + allObjects = append(allObjects, out.Contents...) + + if out.NextMarker == nil || !*out.IsTruncated { + break + } + marker = out.NextMarker + } + + if !compareObjects(contents, allObjects) { + return fmt.Errorf("expected the contents to be %v, instead got %v", + contents, allObjects) + } + + return nil + }) +} + +func ListObjects_nested_dir_file_objs(s *S3Conf) error { + testName := "ListObjects_nested_dir_file_objs" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents, err := putObjects(s3client, []string{"foo/bar/", "foo/bar/baz", "foo/bar/quxx"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(contents, res.Contents) { + return fmt.Errorf("expected the objects list to be %+v, instead got %+v", contents, res.Contents) + } + + // Clean up the nested objects to avoid `ErrDirectoryNotEmpty` error on teardown + for _, obj := range []string{"foo/bar/baz", "foo/bar/quxx"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + } + + return nil + }) +} + +func ListObjects_check_owner(s *S3Conf) error { + testName := "ListObjects_check_owner" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs, err := putObjects(s3client, []string{"foo", "bar/baz", "quxx/xyz/eee", "abc/", "bcc"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + for i := range res.Contents { + res.Contents[i].Owner = &types.Owner{ + ID: &s.awsID, + } + } + + if !compareObjects(objs, res.Contents) { + return fmt.Errorf("expected the contents to be %v, instead got %v", + objs, res.Contents) + } + + return nil + + }) +} + +func ListObjects_non_truncated_common_prefixes(s *S3Conf) error { + testName := "ListObjects_non_truncated_common_prefixes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + _, err := putObjects(s3client, []string{"asdf", "boo/bar", "boo/baz/xyzzy", "cquux/thud", "cquux/bla"}, bucket) + if err != nil { + return err + } + + delim, marker, maxKeys := "/", "boo/", int32(1) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Marker: &marker, + Delimiter: &delim, + MaxKeys: &maxKeys, + }) + cancel() + if err != nil { + return err + } + + if res.IsTruncated == nil { + return fmt.Errorf("expected non-nil istruncated") + } + if *res.IsTruncated { + return fmt.Errorf("expected non-truncated result") + } + if res.MaxKeys == nil { + return fmt.Errorf("expected non nil max-keys") + } + if *res.MaxKeys != maxKeys { + return fmt.Errorf("expected max-keys to be %v, instead got %v", + maxKeys, *res.MaxKeys) + } + if getString(res.Delimiter) != delim { + return fmt.Errorf("expected delimiter to be %v, instead got %v", + delim, getString(res.Delimiter)) + } + if getString(res.Marker) != marker { + return fmt.Errorf("expected marker to be %v, instead got %v", + getString(res.Marker), marker) + } + if len(res.Contents) != 0 { + return fmt.Errorf("expected empty contents, instead got %+v", + res.Contents) + } + cPrefs := []string{"cquux/"} + if !comparePrefixes(cPrefs, res.CommonPrefixes) { + return fmt.Errorf("expected common prefixes to be %v, instead got %+v", + cPrefs, sprintPrefixes(res.CommonPrefixes)) + } + + return nil + }) +} + +// ListObjects should not list any pending multipart uploads +// and no pending mp should block the bucket from deletion +func ListObjects_should_not_list_pending_mps(s *S3Conf) error { + testName := "ListObjects_should_not_list_pending_mps" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for i := range 5 { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr(fmt.Sprintf("obj-%d", i)), + }) + cancel() + if err != nil { + return err + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(res.Contents) != 0 { + return fmt.Errorf("expected empty object list result, instead got %v", res.Contents) + } + if len(res.CommonPrefixes) != 0 { + return fmt.Errorf("expected empty object common prefixes result, instead got %v", res.CommonPrefixes) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + return err + }, withSkipTearDown()) +} + +// ListObjects with a marker should not surface pending multipart uploads +// even when real objects are interleaved with the marker boundary. +func ListObjects_mp_masking_with_marker(s *S3Conf) error { + testName := "ListObjects_mp_masking_with_marker" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Create pending multipart uploads with keys that sort after all real objects + for i := range 3 { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)), + }) + cancel() + if err != nil { + return err + } + } + + contents, err := putObjects(s3client, []string{"aaa", "bbb", "ccc"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Marker: getPtr("aaa"), + }) + cancel() + if err != nil { + return err + } + + // Expect only bbb and ccc (after marker "aaa"), no multipart upload objects + if !compareObjects(contents[1:], out.Contents) { + return fmt.Errorf("expected objects %v, instead got %v", + contents[1:], out.Contents) + } + if out.IsTruncated == nil || *out.IsTruncated { + return fmt.Errorf("expected non-truncated result") + } + + return nil + }) +} + +// ListObjects truncation should count only real objects, not pending multipart uploads. +func ListObjects_mp_masking_truncation(s *S3Conf) error { + testName := "ListObjects_mp_masking_truncation" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Create pending multipart uploads with keys that sort after real objects + for i := range 2 { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)), + }) + cancel() + if err != nil { + return err + } + } + + contents, err := putObjects(s3client, []string{"obj-a", "obj-b", "obj-c", "obj-d"}, bucket) + if err != nil { + return err + } + + maxKeys := int32(2) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out1, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: &maxKeys, + }) + cancel() + if err != nil { + return err + } + + if out1.IsTruncated == nil || !*out1.IsTruncated { + return fmt.Errorf("expected first page to be truncated") + } + if !compareObjects(contents[:2], out1.Contents) { + return fmt.Errorf("expected first page objects %v, instead got %v", + contents[:2], out1.Contents) + } + if out1.NextMarker == nil || *out1.NextMarker == "" { + return fmt.Errorf("expected non-empty NextMarker") + } + if *out1.NextMarker != "obj-b" { + return fmt.Errorf("expected NextMarker to be obj-b, instead got %v", *out1.NextMarker) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out2, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Marker: out1.NextMarker, + }) + cancel() + if err != nil { + return err + } + + if out2.IsTruncated == nil || *out2.IsTruncated { + return fmt.Errorf("expected second page to not be truncated") + } + if !compareObjects(contents[2:], out2.Contents) { + return fmt.Errorf("expected second page objects %v, instead got %v", + contents[2:], out2.Contents) + } + + return nil + }) +} + +// ListObjects with a delimiter should not include the .sgwtmp/ multipart prefix +// in common prefixes, even when pending multipart uploads exist. +func ListObjects_mp_masking_delimiter(s *S3Conf) error { + testName := "ListObjects_mp_masking_delimiter" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Create pending multipart uploads + for i := range 2 { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)), + }) + cancel() + if err != nil { + return err + } + } + + _, err := putObjects(s3client, []string{"dir1/file1", "dir2/file2"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Delimiter: getPtr("/"), + }) + cancel() + if err != nil { + return err + } + + if len(out.Contents) != 0 { + return fmt.Errorf("expected empty Contents, instead got %v", out.Contents) + } + if !comparePrefixes([]string{"dir1/", "dir2/"}, out.CommonPrefixes) { + return fmt.Errorf("expected common prefixes [dir1/ dir2/], instead got %v", + sprintPrefixes(out.CommonPrefixes)) + } + + return nil + }) +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 96789c9..328683b 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -110,6 +110,8 @@ func TestListBuckets(ts *TestState) { ts.Sync(ListBuckets_as_admin) ts.Sync(ListBuckets_with_prefix) ts.Sync(ListBuckets_invalid_max_buckets) + ts.Sync(ListBuckets_invalid_bucket_region) + ts.Sync(ListBuckets_bucket_region) ts.Sync(ListBuckets_truncated) ts.Sync(ListBuckets_success) ts.Sync(ListBuckets_empty_success) @@ -407,6 +409,8 @@ func TestCopyObject(ts *TestState) { ts.Run(CopyObject_with_legal_hold) ts.Run(CopyObject_with_retention_lock) ts.Run(CopyObject_conditional_reads) + ts.Run(CopyObject_conditional_writes) + ts.Run(CopyObject_conditional_writes_precedence) ts.Run(CopyObject_object_acl_not_supported) //TODO: remove the condition after implementing checksums in azure if !ts.conf.azureTests { @@ -546,10 +550,12 @@ func TestListMultipartUploads(ts *TestState) { ts.Run(ListMultipartUploads_empty_result) ts.Run(ListMultipartUploads_invalid_max_uploads) ts.Run(ListMultipartUploads_max_uploads) + ts.Run(ListMultipartUploads_next_markers_not_truncated) ts.Run(ListMultipartUploads_exceeding_max_uploads) ts.Run(ListMultipartUploads_ignore_upload_id_marker) ts.Run(ListMultipartUploads_invalid_uploadId_marker) ts.Run(ListMultipartUploads_keyMarker_not_from_list) + ts.Run(ListMultipartUploads_upload_id_marker_with_key_marker) ts.Run(ListMultipartUploads_delimiter_truncated) ts.Run(ListMultipartUploads_prefix) ts.Run(ListMultipartUploads_both_delimiter_and_prefix) @@ -1841,7 +1847,7 @@ func TestS3IAMAccessControl(ts *TestState) { ts.Run(S3IAMAccessControl_condition_if_match_delete_object) ts.Run(S3IAMAccessControl_condition_if_match_versioned_delete) ts.Run(S3IAMAccessControl_condition_conditional_write_keys_ignore_reads) - ts.Run(S3IAMAccessControl_condition_conditional_write_keys_ignore_copies) + ts.Run(S3IAMAccessControl_condition_conditional_write_keys_cover_copies) ts.Run(S3IAMAccessControl_condition_if_match_bucket_level_write) ts.Run(S3IAMAccessControl_condition_if_match_ignores_delete_objects) ts.Run(S3IAMAccessControl_inactive_and_deleted_credentials) @@ -1976,6 +1982,7 @@ func TestAccessControl(ts *TestState) { ts.Run(AccessControl_bucket_policy_condition_binary_operator) ts.Run(AccessControl_bucket_policy_condition_null_operator) ts.Run(AccessControl_bucket_policy_condition_if_none_match_required) + ts.Run(AccessControl_bucket_policy_condition_if_none_match_copy) ts.Run(AccessControl_bucket_policy_condition_if_none_match_value) ts.Run(AccessControl_bucket_policy_condition_if_match_value) ts.Run(AccessControl_bucket_policy_condition_if_match_delete_object) @@ -2036,6 +2043,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_PutObject_new_version_resets_attributes) ts.Run(Versioning_PutObject_over_delete_marker_resets_attributes) ts.Run(Versioning_PutObject_suspended_null_version_resets_attributes) + ts.Run(Versioning_PutObject_conditional_writes_over_delete_marker) // CopyObject action ts.Run(Versioning_CopyObject_invalid_versionId) ts.Run(Versioning_CopyObject_encoded_versionid_separator_invalid_versionId) @@ -2045,6 +2053,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_CopyObject_from_a_delete_marker) ts.Run(Versioning_CopyObject_to_itself) ts.Run(Versioning_CopyObject_to_itself_from_the_current_version) + ts.Run(Versioning_CopyObject_conditional_writes) if !ts.conf.windowsTests { ts.Run(Versioning_CopyObject_special_chars) } @@ -2124,6 +2133,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_Multipart_Upload_overwrite_an_object) ts.Run(Versioning_Multipart_Upload_suspended_overwrite_versioned_object) ts.Run(Versioning_Multipart_Upload_overwrite_keeps_previous_version_metadata) + ts.Run(Versioning_Multipart_Upload_conditional_writes_over_delete_marker) ts.Run(Versioning_UploadPartCopy_invalid_versionId) ts.Run(Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId) ts.Run(Versioning_UploadPartCopy_non_existing_versionId) @@ -2195,6 +2205,7 @@ func TestRouter(ts *TestState) { ts.Run(RouterObjectAnnotationNotImplemented) ts.Run(RouterRenameObjectNotImplemented) ts.Run(RouterObjectTorrentMethodNotAllowed) + ts.Run(RouterInvalidURLEscape) } func TestPostObject(ts *TestState) { @@ -2377,7 +2388,7 @@ func GetIntTests() IntTests { "S3IAMAccessControl_condition_if_match_delete_object": S3IAMAccessControl_condition_if_match_delete_object, "S3IAMAccessControl_condition_if_match_versioned_delete": S3IAMAccessControl_condition_if_match_versioned_delete, "S3IAMAccessControl_condition_conditional_write_keys_ignore_reads": S3IAMAccessControl_condition_conditional_write_keys_ignore_reads, - "S3IAMAccessControl_condition_conditional_write_keys_ignore_copies": S3IAMAccessControl_condition_conditional_write_keys_ignore_copies, + "S3IAMAccessControl_condition_conditional_write_keys_cover_copies": S3IAMAccessControl_condition_conditional_write_keys_cover_copies, "S3IAMAccessControl_condition_if_match_bucket_level_write": S3IAMAccessControl_condition_if_match_bucket_level_write, "S3IAMAccessControl_condition_if_match_ignores_delete_objects": S3IAMAccessControl_condition_if_match_ignores_delete_objects, "S3IAMAccessControl_inactive_and_deleted_credentials": S3IAMAccessControl_inactive_and_deleted_credentials, @@ -2982,6 +2993,8 @@ func GetIntTests() IntTests { "ListBuckets_as_admin": ListBuckets_as_admin, "ListBuckets_with_prefix": ListBuckets_with_prefix, "ListBuckets_invalid_max_buckets": ListBuckets_invalid_max_buckets, + "ListBuckets_invalid_bucket_region": ListBuckets_invalid_bucket_region, + "ListBuckets_bucket_region": ListBuckets_bucket_region, "ListBuckets_truncated": ListBuckets_truncated, "ListBuckets_success": ListBuckets_success, "ListBuckets_empty_success": ListBuckets_empty_success, @@ -3185,6 +3198,8 @@ func GetIntTests() IntTests { "CopyObject_with_legal_hold": CopyObject_with_legal_hold, "CopyObject_with_retention_lock": CopyObject_with_retention_lock, "CopyObject_conditional_reads": CopyObject_conditional_reads, + "CopyObject_conditional_writes": CopyObject_conditional_writes, + "CopyObject_conditional_writes_precedence": CopyObject_conditional_writes_precedence, "CopyObject_object_acl_not_supported": CopyObject_object_acl_not_supported, "CopyObject_with_metadata": CopyObject_with_metadata, "CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm, @@ -3283,10 +3298,12 @@ func GetIntTests() IntTests { "ListMultipartUploads_empty_result": ListMultipartUploads_empty_result, "ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads, "ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads, + "ListMultipartUploads_next_markers_not_truncated": ListMultipartUploads_next_markers_not_truncated, "ListMultipartUploads_exceeding_max_uploads": ListMultipartUploads_exceeding_max_uploads, "ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker, "ListMultipartUploads_invalid_uploadId_marker": ListMultipartUploads_invalid_uploadId_marker, "ListMultipartUploads_keyMarker_not_from_list": ListMultipartUploads_keyMarker_not_from_list, + "ListMultipartUploads_upload_id_marker_with_key_marker": ListMultipartUploads_upload_id_marker_with_key_marker, "ListMultipartUploads_delimiter_truncated": ListMultipartUploads_delimiter_truncated, "ListMultipartUploads_prefix": ListMultipartUploads_prefix, "ListMultipartUploads_both_delimiter_and_prefix": ListMultipartUploads_both_delimiter_and_prefix, @@ -3631,6 +3648,7 @@ func GetIntTests() IntTests { "AccessControl_bucket_policy_condition_binary_operator": AccessControl_bucket_policy_condition_binary_operator, "AccessControl_bucket_policy_condition_null_operator": AccessControl_bucket_policy_condition_null_operator, "AccessControl_bucket_policy_condition_if_none_match_required": AccessControl_bucket_policy_condition_if_none_match_required, + "AccessControl_bucket_policy_condition_if_none_match_copy": AccessControl_bucket_policy_condition_if_none_match_copy, "AccessControl_bucket_policy_condition_if_none_match_value": AccessControl_bucket_policy_condition_if_none_match_value, "AccessControl_bucket_policy_condition_if_match_value": AccessControl_bucket_policy_condition_if_match_value, "AccessControl_bucket_policy_condition_if_match_delete_object": AccessControl_bucket_policy_condition_if_match_delete_object, @@ -3672,6 +3690,7 @@ func GetIntTests() IntTests { "Versioning_PutObject_new_version_resets_attributes": Versioning_PutObject_new_version_resets_attributes, "Versioning_PutObject_over_delete_marker_resets_attributes": Versioning_PutObject_over_delete_marker_resets_attributes, "Versioning_PutObject_suspended_null_version_resets_attributes": Versioning_PutObject_suspended_null_version_resets_attributes, + "Versioning_PutObject_conditional_writes_over_delete_marker": Versioning_PutObject_conditional_writes_over_delete_marker, "Versioning_CopyObject_invalid_versionId": Versioning_CopyObject_invalid_versionId, "Versioning_CopyObject_encoded_versionid_separator_invalid_versionId": Versioning_CopyObject_encoded_versionid_separator_invalid_versionId, "Versioning_CopyObject_success": Versioning_CopyObject_success, @@ -3680,6 +3699,7 @@ func GetIntTests() IntTests { "Versioning_CopyObject_from_a_delete_marker": Versioning_CopyObject_from_a_delete_marker, "Versioning_CopyObject_to_itself": Versioning_CopyObject_to_itself, "Versioning_CopyObject_to_itself_from_the_current_version": Versioning_CopyObject_to_itself_from_the_current_version, + "Versioning_CopyObject_conditional_writes": Versioning_CopyObject_conditional_writes, "Versioning_CopyObject_special_chars": Versioning_CopyObject_special_chars, "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, "Versioning_HeadObject_non_existing_object_version": Versioning_HeadObject_non_existing_object_version, @@ -3746,6 +3766,7 @@ func GetIntTests() IntTests { "Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object, "Versioning_Multipart_Upload_suspended_overwrite_versioned_object": Versioning_Multipart_Upload_suspended_overwrite_versioned_object, "Versioning_Multipart_Upload_overwrite_keeps_previous_version_metadata": Versioning_Multipart_Upload_overwrite_keeps_previous_version_metadata, + "Versioning_Multipart_Upload_conditional_writes_over_delete_marker": Versioning_Multipart_Upload_conditional_writes_over_delete_marker, "Versioning_UploadPartCopy_invalid_versionId": Versioning_UploadPartCopy_invalid_versionId, "Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId": Versioning_UploadPartCopy_encoded_versionid_separator_invalid_versionId, "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, @@ -3799,6 +3820,7 @@ func GetIntTests() IntTests { "RouterObjectAnnotationNotImplemented": RouterObjectAnnotationNotImplemented, "RouterRenameObjectNotImplemented": RouterRenameObjectNotImplemented, "RouterObjectTorrentMethodNotAllowed": RouterObjectTorrentMethodNotAllowed, + "RouterInvalidURLEscape": RouterInvalidURLEscape, "UnsignedStreaminPayloadTrailer_malformed_trailer": UnsignedStreaminPayloadTrailer_malformed_trailer, "UnsignedStreamingPayloadTrailer_aborted_connection": UnsignedStreamingPayloadTrailer_aborted_connection, "UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length": UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length, diff --git a/tests/test_rest_list_objects.sh b/tests/test_rest_list_objects.sh new file mode 100644 index 0000000..f215e70 --- /dev/null +++ b/tests/test_rest_list_objects.sh @@ -0,0 +1,369 @@ +#!/usr/bin/env bats + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/setup.sh +source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/list_objects/list_objects.sh +source ./tests/drivers/list_objects/list_objects_rest.sh +source ./tests/drivers/list_objects/list_objects_s3api.sh +source ./tests/drivers/objects_and_versions.sh + +# tags: curl,ListObjects,minimal-request +@test "test_rest_list_objects" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run get_file_name + assert_success + test_file="$output" + + run setup_bucket_and_file_v2 "$bucket_name" "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" + assert_success + + run list_check_objects_rest "$bucket_name" + assert_success +} + +# tags: curl,ListObjectsV2,continuation-token,invalid-query +@test "REST - list objects v2 - invalid continuation token" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/993" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run get_file_name + assert_success + test_file="$output" + + run get_file_name + assert_success + test_file_two="$output" + + run get_file_name + assert_success + test_file_three="$output" + + run setup_bucket_and_files_v2 "$bucket_name" "$test_file" "$test_file_two" "$test_file_three" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file_two" "$bucket_name" "$test_file_two" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file_three" "$bucket_name" "$test_file_three" + assert_success + + run list_objects_check_params_get_token "$bucket_name" "$test_file" "$test_file_two" "TRUE" + assert_success + continuation_token=$output + + # interestingly, AWS appears to accept continuation tokens that are a few characters off, so have to remove three chars + run list_objects_check_continuation_error "$bucket_name" "${continuation_token:0:${#continuation_token}-3}" + assert_success +} + +# tags: curl,ListObjectsV2,bucket-region,x-amz-bucket-region +@test "REST - ListObjectsV2 - includes bucket header" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1814" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run get_file_name + assert_success + test_file="$output" + + run get_file_name + assert_success + test_file_two="$output" + + run setup_bucket_and_add_files "$bucket_name" "$test_file" "$test_file_two" + assert_success + + run send_rest_go_command_check_header_key_and_value "200" "x-amz-bucket-region" "$AWS_REGION" "-method" "GET" \ + "-bucketName" "$bucket_name" "-query" "list-type=2" + assert_success +} + +# tags: curl,ListObjectsV2,minimal-request +@test "REST - ListObjectsV2 - success" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run get_file_name + assert_success + test_file="$output" + + run get_file_name + assert_success + test_file_two="$output" + + run setup_bucket_and_add_files "$bucket_name" "$test_file" "$test_file_two" + assert_success + + run list_check_objects_rest_v2 "$bucket_name" 2 "$test_file" "$test_file_two" + assert_success +} + +# tags: curl,ListObjects,minimal-request +@test "REST - list objects v1 - no NextMarker without delimiter" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/999" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run get_file_name + assert_success + test_file="$output" + + run get_file_name + assert_success + test_file_two="$output" + + run setup_bucket_and_files_v2 "$bucket_name" "$test_file" "$test_file_two" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file_two" "$bucket_name" "$test_file_two" + assert_success + + run list_objects_v1_check_nextmarker_empty "$bucket_name" + assert_success +} + +# tags: curl,ListObjects,delimiter,prefix +@test "REST - ListObjects - delimiter" { + list_objects_delimiter "1" +} + +# tags: curl,ListObjects,encoding-type,invalid-query +@test "REST - ListObjects - invalid encoding" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1985" + fi + run objects_versions_invalid_encoding "" + assert_success +} + +# tags: curl,ListObjects,encoding-type +@test "REST - ListObjects - encoding success" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1985" + fi + + run objects_versions_encoding_success "" "ListBucketResult" "Contents" + assert_success +} + +# tags: curl,ListObjects,marker,max-keys +@test "REST - ListObjects - marker/max-keys" { + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one_name file_two_name <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one_name" "$file_two_name" | sort) + + run put_objects "$bucket_name" "$file_one_name" "$file_two_name" + assert_success + + run list_objects_check_count_and_keys "$bucket_name" "1" "${sorted_files[0]}" "-query" "max-keys=1" + assert_success + + run list_objects_check_count_and_keys "$bucket_name" "1" "${sorted_files[1]}" "-query" "max-keys=1&marker=${sorted_files[0]}" + assert_success + + run list_objects_check_count_and_keys "$bucket_name" "0" "-query" "max-keys=1&marker=${sorted_files[1]}" + assert_success +} + +# tags: curl,ListObjects,prefix +@test "REST - ListObjects - prefix" { + run setup_bucket_v3 "$BUCKET_ONE_NAME" + assert_success + bucket_name=$output + + local prefix="prefix" + run create_test_files_with_prefix "$prefix" 2 + assert_success + read -r file_one_name file_two_name <<< "$output" + + run put_objects "$bucket_name" "$file_one_name" "$file_two_name" + assert_success + + run list_objects_check_count_and_keys "$bucket_name" "2" "$file_one_name" "$file_two_name" "-query" "prefix=$prefix" + assert_success + + run list_objects_check_count_and_keys "$bucket_name" "0" "-query" "prefix=prefik" + assert_success + + run list_objects_check_count_and_keys "$bucket_name" "1" "$file_one_name" "-query" "prefix=$file_one_name" + assert_success + + run list_objects_check_count_and_keys "$bucket_name" "1" "$file_two_name" "-query" "prefix=$file_two_name" + assert_success +} + +# tags: curl,ListObjectsV2,continuation-token,max-keys +@test "REST - ListObjectsV2 - continuation token" { + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 3 + assert_success + read -r bucket_name file_one file_two file_three <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" "$file_three" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" "$file_three" + assert_success + + run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "" "1" "false" "${sorted_files[0]}" "-query" "list-type=2&max-keys=1" + assert_success + continuation_token=$output + log 5 "continuation token: $continuation_token" + + run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "$continuation_token" "1" "false" "${sorted_files[1]}" "-query" "list-type=2&max-keys=1&continuation-token=$continuation_token" + assert_success + continuation_token=$output + + run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "$continuation_token" "1" "true" "${sorted_files[2]}" "-query" "list-type=2&max-keys=1&continuation-token=$continuation_token" + assert_success +} + +# tags: curl,ListObjects,start-after,invalid-query +@test "ListObjectsV1 - start-after - error" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2004" + fi + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one file_two <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidArgument" "startAfter only supported in REST.GET.BUCKET with list-type=2" \ + "-bucketName" "$bucket_name" "-query" "start-after=$file_one" + assert_success +} + +# tags: curl,ListObjectsV2,start-after,continuation-token +@test "ListObjectsV1 - start-after - doesn't include continuation token" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2007" + fi + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one file_two <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" + assert_success + + run check_start_after_no_continuation_token "$bucket_name" "$file_two" + assert_success +} + +# tags: curl,ListObjectsV2,start-after +@test "ListObjectsV2 - start-after - success" { + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one file_two <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" + assert_success + + run list_objects_check_start_after_response "$bucket_name" "${sorted_files[0]}" "${sorted_files[1]}" + assert_success + + run list_objects_check_start_after_response "$bucket_name" "${sorted_files[1]}" + assert_success +} + +# tags: curl,ListObjectsV2,fetch-owner +@test "ListObjectsV2 - fetch-owner" { + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one file_two <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" + assert_success + + run list_objects_verify_owner_info_missing "$bucket_name" "$file_one" "$file_two" + assert_success + + run list_objects_verify_owner_info_exists "$bucket_name" "$file_one" "$file_two" + assert_success +} + +# shellcheck disable=SC2030 +# tags: curl,ListObjectsV2,delimiter,prefix +@test "ListObjectsV2 - delimiter" { + list_objects_delimiter 2 +} + +list_objects_delimiter() { + run setup_delimiter_test + assert_success + mapfile -t test_info <<< "$output" + bucket_name="${test_info[0]}" + prefix="${test_info[1]}" + + run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "$1" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt" + assert_success +} + +# tags: curl,ListObjects,marker +@test "REST - ListObjects - Marker returned when request has no marker" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run get_file_name + assert_success + test_file="$output" + + run setup_bucket_and_file_v2 "$bucket_name" "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" + assert_success + + run list_objects_v1_check_empty_marker "$bucket_name" + assert_success +}