diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go new file mode 100644 index 0000000..b9a082f --- /dev/null +++ b/s3api/controllers/bucket-post.go @@ -0,0 +1,383 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "net/http" + "net/url" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3api/middlewares" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) DeleteObjects(ctx fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + bypass := auth.BypassModeForRequest(strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true")) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + // The body has to be parsed before authorization, not after: real AWS + // authorizes s3:DeleteObject against each object's own ARN, so the keys + // are part of what is being authorized. The parsed objects go straight + // to VerifyObjectsAccess, which checks policy and object locks for all + // of them in one pass. + var dObj s3response.DeleteObjects + err := xml.Unmarshal(ctx.BodyRaw(), &dObj) + if err != nil { + debuglogger.Logf("error unmarshalling delete objects: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + // S3 caps a single DeleteObjects request at 1000 keys. Reject an + // over-limit request before authorization and before anything reaches + // the backend, so a too-large batch can't be applied partially. + if len(dObj.Objects) > maxDeleteObjects { + debuglogger.Logf("delete objects: %d keys exceeds the limit of %d", + len(dObj.Objects), maxDeleteObjects) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + // The keys come from the body, so they skip the object name check the + // request path gets. A key resolving outside the bucket ("../key") names + // a path outside it on a filesystem backend, so the whole request is + // rejected before any key is authorized or deleted. + for _, obj := range dObj.Objects { + if obj.Key == nil || !utils.IsObjectNameValid(*obj.Key) { + debuglogger.Logf("delete objects: invalid object key: %q", + backend.GetStringFromPtr(obj.Key)) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrBadRequest) + } + } + + // checkErrs holds one entry per requested object — nil where it may + // proceed to the backend, an AWS-shaped denial otherwise. DeleteObjects + // supports partial success, so a denial on one object (policy or object + // lock) must not fail any other: only the objects that clear this check + // are sent to the backend, and the rest are reported as per-object + // errors directly from checkErrs. err here is a whole-request failure + // (readonly mode, or an error resolving policy/lock state), not about + // any one object. + checkErrs, err := c.verifyObjectsAccess(ctx, + auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + IsPublicRequest: IsBucketPublic, + }, dObj.Objects, bypass) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + toDelete := make([]types.ObjectIdentifier, 0, len(dObj.Objects)) + for i, obj := range dObj.Objects { + if checkErrs[i] == nil { + toDelete = append(toDelete, obj) + } + } + + var backendResult s3response.DeleteResult + if len(toDelete) > 0 { + backendResult, err = c.be.DeleteObjects(ctx.RequestCtx(), + &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: toDelete, + }, + }) + } + + result := utils.MergeDeleteObjectsResult(dObj.Objects, checkErrs, backendResult) + // In quiet mode the response lists only the objects that could not be + // deleted. + if dObj.Quiet { + result.Deleted = nil + } + + return &Response{ + Data: result, + MetaOpts: &MetaOptions{ + ObjectCount: int64(len(dObj.Objects)), + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectRemovedDeleteObjects, + }, + }, err +} + +func (c S3ApiController) POSTObject(ctx fiber.Ctx) (*Response, error) { + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + parsed := utils.ContextKeyObjectPostResult.Get(ctx).(middlewares.PostObjectResult) + bucket := ctx.Params("bucket") + contentType := parsed.Fields["content-type"] + if contentType == "" { + contentType = defaultContentType + } + contentEncoding := parsed.Fields["content-encoding"] + contentDisposition := parsed.Fields["content-disposition"] + contentLanguage := parsed.Fields["content-language"] + cacheControl := parsed.Fields["cache-control"] + expires := parsed.Fields["expires"] + websiteRedirectLocation := parsed.Fields["x-amz-website-redirect-location"] + storageClass := parsed.Fields["x-amz-storage-class"] + + key := parsed.Fields["key"] + + // parse POST policy — absent for anonymous uploads to public buckets + if !IsBucketPublic { + policyBase64 := parsed.Fields["policy"] + policy, err := auth.ParsePOSTPolicyBase64(policyBase64) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // Evaluate post policy + err = policy.Evaluate(auth.PostPolicyEvalInput{ + Bucket: bucket, + Key: key, + ContentLength: parsed.ContentLength, + Fields: parsed.Fields, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + } + + // convert object tagging from raw XML to Query string + // to pass PutObject, which expects the tagging to be a query string + var tagging string + if taggingXML, ok := parsed.Fields["tagging"]; ok { + var err error + tagging, err = utils.ConvertTaggingXMLToQueryString([]byte(taggingXML)) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + } + + // parse checksum headers + checksums, err := utils.ParseCalculatedChecksumFields(parsed.Fields) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // extract metadata + metadata, err := utils.ExtractMetadataFromFields(parsed.Fields) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = utils.ValidateWebsiteRedirectLocation(websiteRedirectLocation) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // A POST upload is an s3:PutObject on the object named by the form's + // key field, so it is authorized against that object's ARN — the same + // resource PutObject is — not the bucket's. Tagging the object also + // takes s3:PutObjectTagging, but only for a non-empty tag set + actions := auth.ObjectUploadActions(tagging, "", "", "") + + err = c.verifyAccess(ctx, + auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: actions, + IsPublicRequest: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.CheckObjectAccess(ctx, bucket, acct, []types.ObjectIdentifier{{Key: &key}}, auth.BypassOverwrite, IsBucketPublic, c.be, c.iam, true) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.PutObject(ctx.RequestCtx(), s3response.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ContentType: &contentType, + ContentEncoding: &contentEncoding, + ContentDisposition: &contentDisposition, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, + WebsiteRedirectLocation: &websiteRedirectLocation, + Body: parsed.FileRdr, + ContentLength: &parsed.ContentLength, + Tagging: &tagging, + Metadata: metadata, + StorageClass: types.StorageClass(storageClass), + ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]), + ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]), + ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]), + ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]), + ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]), + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + if successActionRedirect, ok := parsed.Fields["success_action_redirect"]; ok { + u, err := url.Parse(successActionRedirect) + if err == nil { + q := u.Query() + q.Set("bucket", bucket) + q.Set("key", key) + q.Set("etag", res.ETag) + u.RawQuery = q.Encode() + redirectURI := u.String() + + return &Response{ + Headers: map[string]*string{ + "Location": &redirectURI, + }, + MetaOpts: &MetaOptions{ + ContentLength: parsed.FileRdr.Length(), + BucketOwner: parsedAcl.Owner, + ObjectETag: &res.ETag, + ObjectSize: parsed.FileRdr.Length(), + EventName: s3event.EventObjectCreatedPost, + Status: http.StatusSeeOther, + }, + }, nil + } + } + + respStatus := http.StatusNoContent + var respBody any + location := utils.GenerateObjectLocation(ctx, c.virtualDomain, bucket, key) + + if successStatus, ok := parsed.Fields["success_action_status"]; ok { + switch successStatus { + case "200": + respStatus = http.StatusOK + case "201": + respStatus = http.StatusCreated + respBody = &s3response.PostResponse{ + Bucket: bucket, + Key: key, + ETag: res.ETag, + Location: location, + } + } + } + + return &Response{ + Headers: map[string]*string{ + "Etag": &res.ETag, + "Location": &location, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-sha512": res.ChecksumSHA512, + "x-amz-checksum-md5": res.ChecksumMD5, + "x-amz-checksum-xxhash64": res.ChecksumXXHASH64, + "x-amz-checksum-xxhash3": res.ChecksumXXHASH3, + "x-amz-checksum-xxhash128": res.ChecksumXXHASH128, + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-version-id": utils.GetStringPtr(res.VersionID), + }, + Data: respBody, + MetaOpts: &MetaOptions{ + ContentLength: parsed.FileRdr.Length(), + BucketOwner: parsedAcl.Owner, + ObjectETag: &res.ETag, + ObjectSize: parsed.FileRdr.Length(), + EventName: s3event.EventObjectCreatedPost, + Status: respStatus, + }, + }, nil +} diff --git a/s3api/controllers/bucket-post_test.go b/s3api/controllers/bucket-post_test.go new file mode 100644 index 0000000..e7c1dc0 --- /dev/null +++ b/s3api/controllers/bucket-post_test.go @@ -0,0 +1,972 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/base64" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/middlewares" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_DeleteObjects(t *testing.T) { + validBody, err := xml.Marshal(s3response.DeleteObjects{ + Objects: []types.ObjectIdentifier{ + {Key: utils.GetStringPtr("obj")}, + }, + }) + assert.NoError(t, err) + + validRes := s3response.DeleteResult{ + Deleted: []types.DeletedObject{ + {Key: utils.GetStringPtr("obj")}, + }, + } + + partialSuccessBody, err := xml.Marshal(s3response.DeleteObjects{ + Objects: []types.ObjectIdentifier{ + {Key: utils.GetStringPtr("locked")}, + {Key: utils.GetStringPtr("ok")}, + }, + }) + assert.NoError(t, err) + + // 1000 keys is the S3 limit for a single DeleteObjects request; 1001 + // must be rejected before any of them reaches the backend. The counts + // are spelled out rather than derived from the limit constant so the + // test pins the documented boundary. + const keyLimit = 1000 + keyLimitObjs := make([]types.ObjectIdentifier, keyLimit+1) + for i := range keyLimitObjs { + keyLimitObjs[i] = types.ObjectIdentifier{Key: utils.GetStringPtr(fmt.Sprintf("key-%d", i))} + } + atLimitBody, err := xml.Marshal(s3response.DeleteObjects{Objects: keyLimitObjs[:keyLimit]}) + assert.NoError(t, err) + + overLimitBody, err := xml.Marshal(s3response.DeleteObjects{Objects: keyLimitObjs}) + assert.NoError(t, err) + + emptyBody, err := xml.Marshal(s3response.DeleteObjects{Objects: []types.ObjectIdentifier{}}) + assert.NoError(t, err) + + invalidKeyBody, err := xml.Marshal(s3response.DeleteObjects{ + Objects: []types.ObjectIdentifier{ + {Key: utils.GetStringPtr("obj")}, + {Key: utils.GetStringPtr("../../outside/file")}, + }, + }) + assert.NoError(t, err) + + missingKeyBody, err := xml.Marshal(s3response.DeleteObjects{ + Objects: []types.ObjectIdentifier{ + {Key: utils.GetStringPtr("obj")}, + {VersionId: utils.GetStringPtr("null")}, + }, + }) + assert.NoError(t, err) + + quietBody, err := xml.Marshal(s3response.DeleteObjects{ + Objects: []types.ObjectIdentifier{ + {Key: utils.GetStringPtr("ok")}, + {Key: utils.GetStringPtr("failed")}, + }, + Quiet: true, + }) + assert.NoError(t, err) + + failedCode, failedMessage := "InternalError", "We encountered an internal error. Please try again." + quietBackendRes := s3response.DeleteResult{ + Deleted: []types.DeletedObject{{Key: utils.GetStringPtr("ok")}}, + Error: []types.Error{ + {Key: utils.GetStringPtr("failed"), Code: &failedCode, Message: &failedMessage}, + }, + } + + lockConfig, err := json.Marshal(auth.BucketLockConfig{Enabled: true}) + assert.NoError(t, err) + + legalHoldOn, legalHoldOff := true, false + lockedObjectCode := "AccessDenied" + lockedObjectMessage := "Access Denied because object protected by object lock." + + tests := []struct { + name string + input testInput + output testOutput + configureMock func(be *BackendMock) + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + body: validBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "check object access returns error", + input: testInput{ + locals: defaultLocals, + body: validBody, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLocked), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrObjectLocked), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.DeleteResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + body: validBody, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + }, + output: testOutput{ + response: &Response{ + Data: s3response.DeleteResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 1, + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validBody, + beRes: validRes, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + }, + output: testOutput{ + response: &Response{ + Data: validRes, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 1, + }, + }, + }, + }, + { + name: "quiet mode reports only the objects not deleted", + input: testInput{ + locals: defaultLocals, + body: quietBody, + beRes: quietBackendRes, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + }, + output: testOutput{ + response: &Response{ + Data: s3response.DeleteResult{ + Error: []types.Error{ + {Key: utils.GetStringPtr("failed"), Code: &failedCode, Message: &failedMessage}, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 2, + }, + }, + }, + }, + { + name: "empty delete list", + input: testInput{ + locals: defaultLocals, + body: emptyBody, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + }, + output: testOutput{ + response: &Response{ + Data: s3response.DeleteResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 0, + }, + }, + }, + }, + { + name: "exactly at the 1000 key limit", + input: testInput{ + locals: defaultLocals, + body: atLimitBody, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + }, + output: testOutput{ + response: &Response{ + Data: s3response.DeleteResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 1000, + }, + }, + }, + configureMock: func(be *BackendMock) { + be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + assert.Len(t, deleteObjectsInput.Delete.Objects, 1000) + return s3response.DeleteResult{}, nil + } + }, + }, + { + name: "over the 1000 key limit", + input: testInput{ + locals: defaultLocals, + body: overLimitBody, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + configureMock: func(be *BackendMock) { + be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + t.Error("backend DeleteObjects called for an over-limit request") + return s3response.DeleteResult{}, nil + } + }, + }, + { + name: "object key escaping the bucket", + input: testInput{ + locals: defaultLocals, + body: invalidKeyBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrBadRequest), + }, + configureMock: func(be *BackendMock) { + be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + t.Error("backend DeleteObjects called for a request with an invalid key") + return s3response.DeleteResult{}, nil + } + }, + }, + { + name: "missing object key", + input: testInput{ + locals: defaultLocals, + body: missingKeyBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrBadRequest), + }, + configureMock: func(be *BackendMock) { + be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + t.Error("backend DeleteObjects called for a request with a missing key") + return s3response.DeleteResult{}, nil + } + }, + }, + { + name: "partial success: one object locked, one succeeds", + input: testInput{ + locals: defaultLocals, + body: partialSuccessBody, + }, + output: testOutput{ + response: &Response{ + Data: s3response.DeleteResult{ + Deleted: []types.DeletedObject{ + {Key: utils.GetStringPtr("ok")}, + }, + Error: []types.Error{ + {Key: utils.GetStringPtr("locked"), Code: &lockedObjectCode, Message: &lockedObjectMessage}, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 2, + }, + }, + }, + configureMock: func(be *BackendMock) { + be.GetObjectLockConfigurationFunc = func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return lockConfig, nil + } + be.GetBucketVersioningFunc = func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { + return s3response.GetBucketVersioningOutput{}, nil + } + be.GetObjectRetentionFunc = func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) { + return []byte("{}"), nil + } + be.GetObjectLegalHoldFunc = func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) { + if object == "locked" { + return &legalHoldOn, nil + } + return &legalHoldOff, nil + } + be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + // Only the object that cleared the lock check should + // ever reach the backend. + assert.Len(t, deleteObjectsInput.Delete.Objects, 1) + assert.Equal(t, "ok", *deleteObjectsInput.Delete.Objects[0].Key) + return s3response.DeleteResult{ + Deleted: []types.DeletedObject{{Key: utils.GetStringPtr("ok")}}, + }, nil + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + return tt.input.beRes.(s3response.DeleteResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, tt.input.extraMockErr + }, + } + if tt.configureMock != nil { + tt.configureMock(be) + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteObjects, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +// mockMpFileReader wraps an io.Reader and satisfies utils.MpFileReader. +// It tracks the number of bytes delivered to callers so that Length returns +// the same value that finalFileReader.Length would return after a real upload. +type mockMpFileReader struct { + r io.Reader + bytesRead int64 +} + +func (m *mockMpFileReader) Read(p []byte) (int, error) { + n, err := m.r.Read(p) + m.bytesRead += int64(n) + return n, err +} + +func (m *mockMpFileReader) Length() int64 { return m.bytesRead } + +func newMockFileReader(content string) *mockMpFileReader { + return &mockMpFileReader{r: strings.NewReader(content)} +} + +func TestS3ApiController_POSTObject(t *testing.T) { + encodePOSTPolicyForControllerTest := func(t *testing.T, expiration time.Time, conditions []any) string { + t.Helper() + + policy := map[string]any{ + "expiration": expiration.UTC().Format(time.RFC3339), + "conditions": conditions, + } + + b, err := json.Marshal(policy) + assert.NoError(t, err) + + return base64.StdEncoding.EncodeToString(b) + } + postObjectLocalsForTest := func(parsed middlewares.PostObjectResult) map[utils.ContextKey]any { + return map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + utils.ContextKeyRegion: "us-east-1", + utils.ContextKeyObjectPostResult: parsed, + } + } + marshalObjectTaggingForControllerTest := func(t *testing.T, tags []s3response.Tag) string { + t.Helper() + + data, err := xml.Marshal(s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: tags, + }, + }) + assert.NoError(t, err) + + return string(data) + } + + validTaggingXML := marshalObjectTaggingForControllerTest(t, []s3response.Tag{ + {Key: "project", Value: "alpha team"}, + }) + baseFields := map[string]string{ + "key": "uploads/photo.jpg", + "file": "ignored", + "x-amz-signature": "ignored", + } + basePolicy := encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + }) + baseFields["policy"] = basePolicy + + location := "http://example.com/bucket/uploads%2Fphoto.jpg" + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: false, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "user", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "user", + Role: auth.RoleUser, + }, + utils.ContextKeyRegion: "us-east-1", + utils.ContextKeyObjectPostResult: middlewares.PostObjectResult{ + Fields: baseFields, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }, + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "user", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid policy", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": "%%%not-base64%%%", + "file": "ignored", + "x-amz-signature": "ignored", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.InvalidPolicyDocument.InvalidBase64Encoding(), + }, + }, + { + name: "policy evaluation fails on extra field", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": basePolicy, + "file": "ignored", + "x-amz-signature": "ignored", + "unexpected": "value", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.InvalidPolicyDocument.ExtraInputField("unexpected"), + }, + }, + { + name: "invalid tagging xml", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$tagging", "invalid-xml"}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "tagging": "invalid-xml", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "invalid checksum fields", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$x-amz-checksum-crc32", "invalid_base64_string"}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "x-amz-checksum-crc32": "invalid_base64_string", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"), + }, + }, + { + name: "metadata too large", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"starts-with", "$x-amz-meta-big", ""}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "x-amz-meta-big": strings.Repeat("a", 2050), + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetMetadataTooLargeErr(2053, 2048), + }, + }, + { + name: "overwriting a locked object", + input: testInput{ + // object lock enabled on the bucket; the mock reports a + // legal hold on every object + extraMockResp: []byte(`{"Enabled":true}`), + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: baseFields, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrObjectLocked), + }, + }, + { + name: "backend returns error", + input: testInput{ + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: baseFields, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful redirect response", + input: testInput{ + beRes: s3response.PutObjectOutput{ + ETag: "etag-123", + VersionID: "vid-123", + }, + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$success_action_redirect", "https://client.example/upload-complete"}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "success_action_redirect": "https://client.example/upload-complete", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "Location": utils.GetStringPtr("https://client.example/upload-complete?bucket=bucket&etag=etag-123&key=uploads%2Fphoto.jpg"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ContentLength: int64(len("payload")), + ObjectETag: utils.GetStringPtr("etag-123"), + ObjectSize: int64(len("payload")), + EventName: s3event.EventObjectCreatedPost, + Status: 303, + }, + }, + }, + }, + { + name: "successful created response", + input: testInput{ + beRes: s3response.PutObjectOutput{ + ETag: "etag-123", + VersionID: "vid-123", + ChecksumCRC32: utils.GetStringPtr("crc32-out"), + ChecksumType: types.ChecksumTypeComposite, + }, + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/photo.jpg", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + []any{"starts-with", "$key", "uploads/"}, + []any{"eq", "$success_action_status", "201"}, + []any{"eq", "$tagging", validTaggingXML}, + []any{"eq", "$x-amz-meta-owner", "alice"}, + []any{"eq", "$x-amz-checksum-crc32", "ww2FVQ=="}, + []any{"eq", "$cache-control", "max-age=60"}, + []any{"eq", "$content-type", "image/jpeg"}, + []any{"eq", "$content-disposition", "inline"}, + []any{"eq", "$content-encoding", "gzip"}, + []any{"eq", "$content-language", "en-US"}, + []any{"eq", "$expires", "Fri, 21 Mar 2026 00:00:00 GMT"}, + []any{"eq", "$x-amz-storage-class", string(types.StorageClassGlacier)}, + }), + "file": "ignored", + "x-amz-signature": "ignored", + "success_action_status": "201", + "tagging": validTaggingXML, + "x-amz-meta-owner": "alice", + "x-amz-checksum-crc32": "ww2FVQ==", + "cache-control": "max-age=60", + "content-type": "image/jpeg", + "content-disposition": "inline", + "content-encoding": "gzip", + "content-language": "en-US", + "expires": "Fri, 21 Mar 2026 00:00:00 GMT", + "x-amz-storage-class": string(types.StorageClassGlacier), + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "Etag": utils.GetStringPtr("etag-123"), + "Location": &location, + "x-amz-checksum-crc32": utils.GetStringPtr("crc32-out"), + "x-amz-checksum-crc32c": utils.GetStringPtr(""), + "x-amz-checksum-crc64nvme": utils.GetStringPtr(""), + "x-amz-checksum-sha1": utils.GetStringPtr(""), + "x-amz-checksum-sha256": utils.GetStringPtr(""), + "x-amz-checksum-sha512": utils.GetStringPtr(""), + "x-amz-checksum-md5": utils.GetStringPtr(""), + "x-amz-checksum-xxhash64": utils.GetStringPtr(""), + "x-amz-checksum-xxhash3": utils.GetStringPtr(""), + "x-amz-checksum-xxhash128": utils.GetStringPtr(""), + "x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)), + "x-amz-version-id": utils.GetStringPtr("vid-123"), + }, + Data: &s3response.PostResponse{ + Bucket: "bucket", + Key: "uploads/photo.jpg", + ETag: "etag-123", + Location: location, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ContentLength: int64(len("payload")), + ObjectETag: utils.GetStringPtr("etag-123"), + ObjectSize: int64(len("payload")), + EventName: s3event.EventObjectCreatedPost, + Status: http.StatusCreated, + }, + }, + }, + }, + { + name: "default content type when field is omitted", + input: testInput{ + beRes: s3response.PutObjectOutput{ + ETag: "etag-123", + }, + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: baseFields, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "Etag": utils.GetStringPtr("etag-123"), + "Location": &location, + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, + "x-amz-checksum-type": nil, + "x-amz-version-id": nil, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ContentLength: int64(len("payload")), + ObjectETag: utils.GetStringPtr("etag-123"), + ObjectSize: int64(len("payload")), + EventName: s3event.EventObjectCreatedPost, + Status: http.StatusNoContent, + }, + }, + }, + }, + { + name: "anonymous upload with policy is evaluated", + input: testInput{ + locals: postObjectLocalsForTest(middlewares.PostObjectResult{ + Fields: map[string]string{ + "key": "uploads/anon.bin", + "policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{ + map[string]string{"bucket": "bucket"}, + // key condition intentionally omitted -> ExtraInputField for "key" + }), + "file": "ignored", + }, + FileRdr: newMockFileReader("payload"), + ContentLength: int64(len("payload")), + }), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.InvalidPolicyDocument.ExtraInputField("key"), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) { + if tt.input.beErr != nil { + return s3response.PutObjectOutput{}, tt.input.beErr + } + + // Drain the body as a real backend would, so that FileRdr.Length() + // reflects the actual bytes written after PutObject returns. + body, err := io.ReadAll(putObjectInput.Body) + assert.NoError(t, err) + + if tt.name == "anonymous upload succeeds without policy" { + assert.Equal(t, "uploads/anon.bin", *putObjectInput.Key) + assert.Equal(t, "anon-payload", string(body)) + } + + if tt.name == "default content type when field is omitted" { + assert.Equal(t, defaultContentType, *putObjectInput.ContentType) + } + + if tt.name == "successful created response" { + assert.Equal(t, "bucket", *putObjectInput.Bucket) + assert.Equal(t, "uploads/photo.jpg", *putObjectInput.Key) + assert.Equal(t, "image/jpeg", *putObjectInput.ContentType) + assert.Equal(t, "gzip", *putObjectInput.ContentEncoding) + assert.Equal(t, "inline", *putObjectInput.ContentDisposition) + assert.Equal(t, "en-US", *putObjectInput.ContentLanguage) + assert.Equal(t, "max-age=60", *putObjectInput.CacheControl) + assert.Equal(t, "Fri, 21 Mar 2026 00:00:00 GMT", *putObjectInput.Expires) + assert.Equal(t, types.StorageClassGlacier, putObjectInput.StorageClass) + assert.Equal(t, int64(len("payload")), *putObjectInput.ContentLength) + assert.Equal(t, "project=alpha+team", *putObjectInput.Tagging) + assert.Equal(t, map[string]string{"owner": "alice"}, putObjectInput.Metadata) + assert.Equal(t, utils.GetStringPtr("ww2FVQ=="), putObjectInput.ChecksumCRC32) + assert.Equal(t, "payload", string(body)) + } + + return tt.input.beRes.(s3response.PutObjectOutput), nil + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { + return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented) + }, + GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + if tt.input.extraMockResp != nil { + return tt.input.extraMockResp.([]byte), nil + } + return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) + }, + GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration) + }, + GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) { + legalHold := true + return &legalHold, nil + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.POSTObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }, + ) + }) + } +} diff --git a/tests/integration/DeleteObjects.go b/tests/integration/DeleteObjects.go new file mode 100644 index 0000000..bf97149 --- /dev/null +++ b/tests/integration/DeleteObjects.go @@ -0,0 +1,497 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +func DeleteObjects_non_existing_bucket(s *S3Conf) error { + testName := "DeleteObjects_non_existing_bucket" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bckt := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bckt, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{{Key: getPtr("obj1")}, {Key: getPtr("obj2")}}, + }, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)) + }) +} + +func DeleteObjects_empty_input(s *S3Conf) error { + testName := "DeleteObjects_empty_input" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents, err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{}, + }, + }) + cancel() + if err != nil { + return err + } + + if len(out.Deleted) != 0 { + return fmt.Errorf("expected deleted object count 0, instead got %v", + len(out.Deleted)) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected 0 errors, instead got %v", len(out.Errors)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(contents, res.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", + contents, res.Contents) + } + + return nil + }) +} + +func DeleteObjects_non_existing_objects(s *S3Conf) error { + testName := "DeleteObjects_empty_input" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + delObjects := []types.ObjectIdentifier{{Key: getPtr("obj1")}, {Key: getPtr("obj2")}} + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: delObjects, + }, + }) + cancel() + if err != nil { + return err + } + + if len(out.Deleted) != 2 { + return fmt.Errorf("expected deleted object count 2, instead got %v", + len(out.Deleted)) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected 0 errors, instead got %v, %v", + len(out.Errors), out.Errors) + } + + return nil + }) +} + +func DeleteObjects_success(s *S3Conf) error { + testName := "DeleteObjects_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objects, objToDel := []string{"obj1", "obj2", "obj3"}, []string{"foo", "bar", "baz"} + contents, err := putObjects(s3client, append(objToDel, objects...), bucket) + if err != nil { + return err + } + + delObjects := []types.ObjectIdentifier{} + delResult := []types.DeletedObject{} + for _, key := range objToDel { + k := key + delObjects = append(delObjects, types.ObjectIdentifier{Key: &k}) + delResult = append(delResult, types.DeletedObject{Key: &k}) + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: delObjects, + }, + }) + cancel() + if err != nil { + return err + } + + if len(out.Deleted) != 3 { + return fmt.Errorf("expected deleted object count 3, instead got %v", + len(out.Deleted)) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected 2 errors, instead got %v", + len(out.Errors)) + } + + if !compareDelObjects(delResult, out.Deleted) { + return fmt.Errorf("unexpected deleted output") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(contents[3:], res.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", + contents[3:], res.Contents) + } + + return nil + }) +} + +// DeleteObjects_quiet_mode checks that a quiet-mode request deletes the +// objects but lists none of them, since quiet mode reports only the objects +// that could not be deleted. +func DeleteObjects_quiet_mode(s *S3Conf) error { + testName := "DeleteObjects_quiet_mode" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents, err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{{Key: getPtr("bar")}, {Key: getPtr("baz")}}, + Quiet: getBoolPtr(true), + }, + }) + cancel() + if err != nil { + return err + } + + if len(out.Deleted) != 0 { + return fmt.Errorf("expected no deleted objects in quiet mode, instead got %v", + len(out.Deleted)) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected no errors, instead got %v", len(out.Errors)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(contents[2:], res.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", + contents[2:], res.Contents) + } + + return nil + }) +} + +// DeleteObjects_iam_mixed_denials_and_success covers a single batch mixing +// every DeleteObjects outcome at once: a key the identity policy denies, a +// governance-locked key with no bypass, and keys the caller may freely +// delete. All three outcomes land in one response — no top-level error — +// with Deleted and Errors each preserving the order the keys were +// requested in. +func DeleteObjects_iam_mixed_denials_and_success(s *S3Conf) error { + testName := "DeleteObjects_iam_mixed_denials_and_success" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"allowed/one", "allowed/two", "denied/one"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &key}) + cancel() + if err != nil { + return err + } + } + if err := putGovernanceLockedObject(s, bucket, "locked/one"); err != nil { + return err + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3DeleteObject, + Resource: []string{objectArn(bucket, "allowed/*"), objectArn(bucket, "locked/*")}, + }), + }) + if err != nil { + return err + } + defer cleanup() + + delObjects := []types.ObjectIdentifier{ + {Key: getPtr("allowed/one")}, + {Key: getPtr("denied/one")}, + {Key: getPtr("locked/one")}, + {Key: getPtr("allowed/two")}, + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err) + } + + if err := checkDeletedKeysInOrder(out.Deleted, []string{"allowed/one", "allowed/two"}); err != nil { + return err + } + return checkDeleteObjectsErrsInOrder(out.Errors, []keyDenial{ + {"denied/one", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "denied/one"))}, + {"locked/one", s3err.GetAPIError(s3err.ErrObjectLocked)}, + }) + }, withLock()) +} + +// DeleteObjects_iam_all_access_denied covers a batch where the identity +// policy grants nothing at all: the call itself still succeeds — no +// top-level error — with every object reported denied in Errors, in +// request order, and nothing in Deleted. +func DeleteObjects_iam_all_access_denied(s *S3Conf) error { + testName := "DeleteObjects_iam_all_access_denied" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"one", "two", "three"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &key}) + cancel() + if err != nil { + return err + } + } + + user, cleanup, err := newS3IAMUser(root, s, nil) + if err != nil { + return err + } + defer cleanup() + + delObjects := []types.ObjectIdentifier{ + {Key: getPtr("one")}, {Key: getPtr("two")}, {Key: getPtr("three")}, + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err) + } + + if len(out.Deleted) != 0 { + return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted) + } + return checkDeleteObjectsErrsInOrder(out.Errors, []keyDenial{ + {"one", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "one"))}, + {"two", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "two"))}, + {"three", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "three"))}, + }) + }) +} + +// DeleteObjects_iam_all_locked covers a batch where every object is +// governance-locked and none is deleted: the call still succeeds — no +// top-level error — with every object reported denied in Errors, in +// request order, and nothing in Deleted. Omitting the bypass header +// entirely reports the generic object-lock message; sending the header +// without s3:BypassGovernanceRetention reports the specific AccessDenied +// naming that action instead — the same distinction the single-object +// DELETE path makes, now confirmed for the batch path too. +func DeleteObjects_iam_all_locked(s *S3Conf) error { + testName := "DeleteObjects_iam_all_locked" + return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error { + for _, key := range []string{"locked/one", "locked/two"} { + if err := putGovernanceLockedObject(s, bucket, key); err != nil { + return err + } + } + + user, cleanup, err := newS3IAMUser(root, s, map[string]string{ + "p": policyDoc(accessStatement{ + Effect: "Allow", Action: actS3DeleteObject, Resource: objectsArn(bucket), + }), + }) + if err != nil { + return err + } + defer cleanup() + + delObjects := []types.ObjectIdentifier{{Key: getPtr("locked/one")}, {Key: getPtr("locked/two")}} + + // No bypass header at all: the generic object-lock message. + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err) + } + if len(out.Deleted) != 0 { + return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted) + } + if err := checkDeleteObjectsErrsInOrder(out.Errors, []keyDenial{ + {"locked/one", s3err.GetAPIError(s3err.ErrObjectLocked)}, + {"locked/two", s3err.GetAPIError(s3err.ErrObjectLocked)}, + }); err != nil { + return fmt.Errorf("without bypass header: %w", err) + } + + // Bypass header sent, but the identity policy doesn't grant + // s3:BypassGovernanceRetention: a specific AccessDenied naming that + // action, not the generic object-lock message. + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + BypassGovernanceRetention: getPtr(true), + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err) + } + if len(out.Deleted) != 0 { + return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted) + } + if err := checkDeleteObjectsErrsInOrder(out.Errors, []keyDenial{ + {"locked/one", wantImplicitDeny(user.arn, actS3BypassGovernance, objectArn(bucket, "locked/one"))}, + {"locked/two", wantImplicitDeny(user.arn, actS3BypassGovernance, objectArn(bucket, "locked/two"))}, + }); err != nil { + return fmt.Errorf("with bypass header, no permission: %w", err) + } + return nil + }, withLock()) +} + +// DeleteObjects_key_limit pins the S3 limit of 1000 keys per request: a +// batch of exactly 1000 succeeds, and 1001 is rejected with MalformedXML +// before any of the keys is deleted. +func DeleteObjects_key_limit(s *S3Conf) error { + testName := "DeleteObjects_key_limit" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + delObjects := make([]types.ObjectIdentifier, 1001) + for i := range delObjects { + delObjects[i] = types.ObjectIdentifier{Key: getPtr(fmt.Sprintf("key-%d", i))} + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects[:1000]}, + }) + cancel() + if err != nil { + return fmt.Errorf("expected a 1000 key delete to succeed: %w", err) + } + if len(out.Deleted) != 1000 { + return fmt.Errorf("expected 1000 deleted objects, instead got %v", len(out.Deleted)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{Objects: delObjects}, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)); err != nil { + return fmt.Errorf("expected 1001 keys to be rejected: %w", err) + } + + return nil + }) +} + +func DeleteObjects_invalid_object_keys(s *S3Conf) error { + testName := "DeleteObjects_invalid_object_keys" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, otherObj := "my-obj", "other-obj" + _, err := putObjects(s3client, []string{obj, otherObj}, bucket) + if err != nil { + return err + } + + // Every key resolves outside the bucket or to the bucket itself; + // the ones climbing back into it name obj. A request holding any of + // them is rejected as a whole, so its valid key isn't deleted + // either. + for _, key := range []string{ + ".", + "..", + "/", + "../", + "../../../etc/passwd", + "../" + bucket + "/" + obj, + "dir/../../" + bucket + "/" + obj, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{ + {Key: &otherObj}, + {Key: &key}, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrBadRequest)); err != nil { + return fmt.Errorf("key %q: %w", key, err) + } + } + + for _, key := range []string{obj, otherObj} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &key, + }) + cancel() + if err != nil { + return fmt.Errorf("head %v: %w", key, err) + } + } + + return nil + }) +} diff --git a/tests/test_rest_delete_objects.sh b/tests/test_rest_delete_objects.sh new file mode 100644 index 0000000..29c4890 --- /dev/null +++ b/tests/test_rest_delete_objects.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bats + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/delete_objects/delete_objects_rest.sh +source ./tests/setup.sh + +# tags: curl, DeleteObjects, malformed-message +@test "REST - DeleteObjects - missing payload" { + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + run send_rest_go_command_expect_error "400" "MissingRequestBodyError" "is empty" "-bucketName" "$bucket_name" "-method" "POST" "-query" "delete" "-contentMD5" + assert_success +} + +# tags: curl, DeleteObjects, malformed-message +@test "REST - DeleteObjects - no objects added to payload" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2109" + fi + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + run send_rest_go_command_expect_error "400" "MalformedXML" "was not well-formed" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" "-contentMD5" + assert_success + + run get_object_rest "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/$file_name-copy" + assert_success +} + +# tags: curl, DeleteObjects +@test "REST - DeleteObjects - nonexistent key" { + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + local incorrect_file_name="${file_name}a" + run send_rest_go_command_callback "200" "check_xml_element_contains" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \ + "-contentMD5" "-objectsToDelete" "key=$incorrect_file_name" "--" "$incorrect_file_name" "DeleteResult" "Deleted" "Key" + assert_success + + run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" + assert_success +} + +# tags: curl, DeleteObjects +@test "REST - DeleteObjects - ETag mismatch" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2112" + fi + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + run send_rest_go_command_callback "200" "check_delete_objects_precondition_error" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \ + "-contentMD5" "-objectsToDelete" "key=$file_name;eTag=abc" "--" "$file_name" + assert_success + + run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" + assert_success +} + +# tags: curl, DeleteObjects +@test "REST - DeleteObject - ETag match" { + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + run get_etag_rest "$bucket_name" "$file_name" + assert_success + etag="$output" + + run send_rest_go_command_callback "200" "check_xml_element_contains" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \ + "-contentMD5" "-objectsToDelete" "key=$file_name;eTag=$etag" "--" "$file_name" "DeleteResult" "Deleted" "Key" + assert_success + + run verify_object_not_found "$bucket_name" "$file_name" + assert_success +} + +# tags: curl, DeleteObjects, versioning +@test "REST - DeleteObjects - version ID mismatch" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2119" + fi + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + incorrect_version="abc" + run send_rest_go_command_callback "200" "check_delete_objects_version_error" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \ + "-contentMD5" "-objectsToDelete" "key=$file_name;versionId=$incorrect_version" "--" "$file_name" "$incorrect_version" + assert_success + + run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" + assert_success +} + +# tags: curl, DeleteObjects, versioning +@test "REST - DeleteObjects - version ID" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "cannot change versioning status for static buckets" + fi + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_bucket_versioning_rest "$bucket_name" "Enabled" + assert_success + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + run send_rest_go_command_callback "200" "parse_version_id" "-bucketName" "$bucket_name" "-query" "versions" "--" "true" + assert_success + version_id="$output" + + run send_rest_go_command_callback "200" "check_xml_element_contains" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \ + "-contentMD5" "-objectsToDelete" "key=$file_name;versionId=$version_id" "--" "$file_name" "DeleteResult" "Deleted" "Key" + assert_success + + run verify_object_not_found "$bucket_name" "$file_name" + assert_success +} + +# tags: curl, DeleteObjects +@test "REST - DeleteObjects - quiet mode" { + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + run send_rest_go_command_callback "200" "check_for_empty_or_nonexistent_element" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \ + "-contentMD5" "-objectsToDelete" "key=${file_name}" "-deleteObjectsQuietMode" "--" "DeleteResult" + assert_success + + run verify_object_not_found "$bucket_name" "$file_name" + assert_success +}