Upstream versity/versitygw state of the touched files before 63922ed89d
This commit is contained in:
upstream committed
1 parent
5f0e2e55ba
commit
edbb6c2d74
4 files changed
+2030
No files matched your search
@@ -0,0 +1,383 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/debuglogger"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
func (c S3ApiController) DeleteObjects(ctx fiber.Ctx) (*Response, error) {
|
||||
bucket := ctx.Params("bucket")
|
||||
bypass := auth.BypassModeForRequest(strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true"))
|
||||
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
|
||||
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
|
||||
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
|
||||
IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx)
|
||||
|
||||
// The body has to be parsed before authorization, not after: real AWS
|
||||
// authorizes s3:DeleteObject against each object's own ARN, so the keys
|
||||
// are part of what is being authorized. The parsed objects go straight
|
||||
// to VerifyObjectsAccess, which checks policy and object locks for all
|
||||
// of them in one pass.
|
||||
var dObj s3response.DeleteObjects
|
||||
err := xml.Unmarshal(ctx.BodyRaw(), &dObj)
|
||||
if err != nil {
|
||||
debuglogger.Logf("error unmarshalling delete objects: %v", err)
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
// S3 caps a single DeleteObjects request at 1000 keys. Reject an
|
||||
// over-limit request before authorization and before anything reaches
|
||||
// the backend, so a too-large batch can't be applied partially.
|
||||
if len(dObj.Objects) > maxDeleteObjects {
|
||||
debuglogger.Logf("delete objects: %d keys exceeds the limit of %d",
|
||||
len(dObj.Objects), maxDeleteObjects)
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, s3err.GetAPIError(s3err.ErrMalformedXML)
|
||||
}
|
||||
|
||||
// The keys come from the body, so they skip the object name check the
|
||||
// request path gets. A key resolving outside the bucket ("../key") names
|
||||
// a path outside it on a filesystem backend, so the whole request is
|
||||
// rejected before any key is authorized or deleted.
|
||||
for _, obj := range dObj.Objects {
|
||||
if obj.Key == nil || !utils.IsObjectNameValid(*obj.Key) {
|
||||
debuglogger.Logf("delete objects: invalid object key: %q",
|
||||
backend.GetStringFromPtr(obj.Key))
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, s3err.GetAPIError(s3err.ErrBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
// checkErrs holds one entry per requested object — nil where it may
|
||||
// proceed to the backend, an AWS-shaped denial otherwise. DeleteObjects
|
||||
// supports partial success, so a denial on one object (policy or object
|
||||
// lock) must not fail any other: only the objects that clear this check
|
||||
// are sent to the backend, and the rest are reported as per-object
|
||||
// errors directly from checkErrs. err here is a whole-request failure
|
||||
// (readonly mode, or an error resolving policy/lock state), not about
|
||||
// any one object.
|
||||
checkErrs, err := c.verifyObjectsAccess(ctx,
|
||||
auth.AccessOptions{
|
||||
Acl: parsedAcl,
|
||||
AclPermission: auth.PermissionWrite,
|
||||
IsRoot: isRoot,
|
||||
Acc: acct,
|
||||
Bucket: bucket,
|
||||
IsPublicRequest: IsBucketPublic,
|
||||
}, dObj.Objects, bypass)
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
toDelete := make([]types.ObjectIdentifier, 0, len(dObj.Objects))
|
||||
for i, obj := range dObj.Objects {
|
||||
if checkErrs[i] == nil {
|
||||
toDelete = append(toDelete, obj)
|
||||
}
|
||||
}
|
||||
|
||||
var backendResult s3response.DeleteResult
|
||||
if len(toDelete) > 0 {
|
||||
backendResult, err = c.be.DeleteObjects(ctx.RequestCtx(),
|
||||
&s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{
|
||||
Objects: toDelete,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
result := utils.MergeDeleteObjectsResult(dObj.Objects, checkErrs, backendResult)
|
||||
// In quiet mode the response lists only the objects that could not be
|
||||
// deleted.
|
||||
if dObj.Quiet {
|
||||
result.Deleted = nil
|
||||
}
|
||||
|
||||
return &Response{
|
||||
Data: result,
|
||||
MetaOpts: &MetaOptions{
|
||||
ObjectCount: int64(len(dObj.Objects)),
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
EventName: s3event.EventObjectRemovedDeleteObjects,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
func (c S3ApiController) POSTObject(ctx fiber.Ctx) (*Response, error) {
|
||||
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
|
||||
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
|
||||
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
|
||||
IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx)
|
||||
|
||||
parsed := utils.ContextKeyObjectPostResult.Get(ctx).(middlewares.PostObjectResult)
|
||||
bucket := ctx.Params("bucket")
|
||||
contentType := parsed.Fields["content-type"]
|
||||
if contentType == "" {
|
||||
contentType = defaultContentType
|
||||
}
|
||||
contentEncoding := parsed.Fields["content-encoding"]
|
||||
contentDisposition := parsed.Fields["content-disposition"]
|
||||
contentLanguage := parsed.Fields["content-language"]
|
||||
cacheControl := parsed.Fields["cache-control"]
|
||||
expires := parsed.Fields["expires"]
|
||||
websiteRedirectLocation := parsed.Fields["x-amz-website-redirect-location"]
|
||||
storageClass := parsed.Fields["x-amz-storage-class"]
|
||||
|
||||
key := parsed.Fields["key"]
|
||||
|
||||
// parse POST policy — absent for anonymous uploads to public buckets
|
||||
if !IsBucketPublic {
|
||||
policyBase64 := parsed.Fields["policy"]
|
||||
policy, err := auth.ParsePOSTPolicyBase64(policyBase64)
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
// Evaluate post policy
|
||||
err = policy.Evaluate(auth.PostPolicyEvalInput{
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
ContentLength: parsed.ContentLength,
|
||||
Fields: parsed.Fields,
|
||||
})
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
}
|
||||
|
||||
// convert object tagging from raw XML to Query string
|
||||
// to pass PutObject, which expects the tagging to be a query string
|
||||
var tagging string
|
||||
if taggingXML, ok := parsed.Fields["tagging"]; ok {
|
||||
var err error
|
||||
tagging, err = utils.ConvertTaggingXMLToQueryString([]byte(taggingXML))
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
}
|
||||
|
||||
// parse checksum headers
|
||||
checksums, err := utils.ParseCalculatedChecksumFields(parsed.Fields)
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
// extract metadata
|
||||
metadata, err := utils.ExtractMetadataFromFields(parsed.Fields)
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
err = utils.ValidateWebsiteRedirectLocation(websiteRedirectLocation)
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
// A POST upload is an s3:PutObject on the object named by the form's
|
||||
// key field, so it is authorized against that object's ARN — the same
|
||||
// resource PutObject is — not the bucket's. Tagging the object also
|
||||
// takes s3:PutObjectTagging, but only for a non-empty tag set
|
||||
actions := auth.ObjectUploadActions(tagging, "", "", "")
|
||||
|
||||
err = c.verifyAccess(ctx,
|
||||
auth.AccessOptions{
|
||||
Acl: parsedAcl,
|
||||
AclPermission: auth.PermissionWrite,
|
||||
IsRoot: isRoot,
|
||||
Acc: acct,
|
||||
Bucket: bucket,
|
||||
Object: key,
|
||||
Actions: actions,
|
||||
IsPublicRequest: IsBucketPublic,
|
||||
})
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
err = auth.CheckObjectAccess(ctx, bucket, acct, []types.ObjectIdentifier{{Key: &key}}, auth.BypassOverwrite, IsBucketPublic, c.be, c.iam, true)
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
res, err := c.be.PutObject(ctx.RequestCtx(), s3response.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
ContentType: &contentType,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ContentDisposition: &contentDisposition,
|
||||
ContentLanguage: &contentLanguage,
|
||||
CacheControl: &cacheControl,
|
||||
Expires: &expires,
|
||||
WebsiteRedirectLocation: &websiteRedirectLocation,
|
||||
Body: parsed.FileRdr,
|
||||
ContentLength: &parsed.ContentLength,
|
||||
Tagging: &tagging,
|
||||
Metadata: metadata,
|
||||
StorageClass: types.StorageClass(storageClass),
|
||||
ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]),
|
||||
ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]),
|
||||
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
|
||||
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
|
||||
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
|
||||
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
|
||||
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
|
||||
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
|
||||
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
|
||||
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
|
||||
})
|
||||
if err != nil {
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
if successActionRedirect, ok := parsed.Fields["success_action_redirect"]; ok {
|
||||
u, err := url.Parse(successActionRedirect)
|
||||
if err == nil {
|
||||
q := u.Query()
|
||||
q.Set("bucket", bucket)
|
||||
q.Set("key", key)
|
||||
q.Set("etag", res.ETag)
|
||||
u.RawQuery = q.Encode()
|
||||
redirectURI := u.String()
|
||||
|
||||
return &Response{
|
||||
Headers: map[string]*string{
|
||||
"Location": &redirectURI,
|
||||
},
|
||||
MetaOpts: &MetaOptions{
|
||||
ContentLength: parsed.FileRdr.Length(),
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
ObjectETag: &res.ETag,
|
||||
ObjectSize: parsed.FileRdr.Length(),
|
||||
EventName: s3event.EventObjectCreatedPost,
|
||||
Status: http.StatusSeeOther,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
respStatus := http.StatusNoContent
|
||||
var respBody any
|
||||
location := utils.GenerateObjectLocation(ctx, c.virtualDomain, bucket, key)
|
||||
|
||||
if successStatus, ok := parsed.Fields["success_action_status"]; ok {
|
||||
switch successStatus {
|
||||
case "200":
|
||||
respStatus = http.StatusOK
|
||||
case "201":
|
||||
respStatus = http.StatusCreated
|
||||
respBody = &s3response.PostResponse{
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
ETag: res.ETag,
|
||||
Location: location,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &Response{
|
||||
Headers: map[string]*string{
|
||||
"Etag": &res.ETag,
|
||||
"Location": &location,
|
||||
"x-amz-checksum-crc32": res.ChecksumCRC32,
|
||||
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
|
||||
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
|
||||
"x-amz-checksum-sha1": res.ChecksumSHA1,
|
||||
"x-amz-checksum-sha256": res.ChecksumSHA256,
|
||||
"x-amz-checksum-sha512": res.ChecksumSHA512,
|
||||
"x-amz-checksum-md5": res.ChecksumMD5,
|
||||
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
|
||||
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
|
||||
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
|
||||
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
|
||||
"x-amz-version-id": utils.GetStringPtr(res.VersionID),
|
||||
},
|
||||
Data: respBody,
|
||||
MetaOpts: &MetaOptions{
|
||||
ContentLength: parsed.FileRdr.Length(),
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
ObjectETag: &res.ETag,
|
||||
ObjectSize: parsed.FileRdr.Length(),
|
||||
EventName: s3event.EventObjectCreatedPost,
|
||||
Status: respStatus,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,972 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
func TestS3ApiController_DeleteObjects(t *testing.T) {
|
||||
validBody, err := xml.Marshal(s3response.DeleteObjects{
|
||||
Objects: []types.ObjectIdentifier{
|
||||
{Key: utils.GetStringPtr("obj")},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
validRes := s3response.DeleteResult{
|
||||
Deleted: []types.DeletedObject{
|
||||
{Key: utils.GetStringPtr("obj")},
|
||||
},
|
||||
}
|
||||
|
||||
partialSuccessBody, err := xml.Marshal(s3response.DeleteObjects{
|
||||
Objects: []types.ObjectIdentifier{
|
||||
{Key: utils.GetStringPtr("locked")},
|
||||
{Key: utils.GetStringPtr("ok")},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// 1000 keys is the S3 limit for a single DeleteObjects request; 1001
|
||||
// must be rejected before any of them reaches the backend. The counts
|
||||
// are spelled out rather than derived from the limit constant so the
|
||||
// test pins the documented boundary.
|
||||
const keyLimit = 1000
|
||||
keyLimitObjs := make([]types.ObjectIdentifier, keyLimit+1)
|
||||
for i := range keyLimitObjs {
|
||||
keyLimitObjs[i] = types.ObjectIdentifier{Key: utils.GetStringPtr(fmt.Sprintf("key-%d", i))}
|
||||
}
|
||||
atLimitBody, err := xml.Marshal(s3response.DeleteObjects{Objects: keyLimitObjs[:keyLimit]})
|
||||
assert.NoError(t, err)
|
||||
|
||||
overLimitBody, err := xml.Marshal(s3response.DeleteObjects{Objects: keyLimitObjs})
|
||||
assert.NoError(t, err)
|
||||
|
||||
emptyBody, err := xml.Marshal(s3response.DeleteObjects{Objects: []types.ObjectIdentifier{}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
invalidKeyBody, err := xml.Marshal(s3response.DeleteObjects{
|
||||
Objects: []types.ObjectIdentifier{
|
||||
{Key: utils.GetStringPtr("obj")},
|
||||
{Key: utils.GetStringPtr("../../outside/file")},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
missingKeyBody, err := xml.Marshal(s3response.DeleteObjects{
|
||||
Objects: []types.ObjectIdentifier{
|
||||
{Key: utils.GetStringPtr("obj")},
|
||||
{VersionId: utils.GetStringPtr("null")},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
quietBody, err := xml.Marshal(s3response.DeleteObjects{
|
||||
Objects: []types.ObjectIdentifier{
|
||||
{Key: utils.GetStringPtr("ok")},
|
||||
{Key: utils.GetStringPtr("failed")},
|
||||
},
|
||||
Quiet: true,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
failedCode, failedMessage := "InternalError", "We encountered an internal error. Please try again."
|
||||
quietBackendRes := s3response.DeleteResult{
|
||||
Deleted: []types.DeletedObject{{Key: utils.GetStringPtr("ok")}},
|
||||
Error: []types.Error{
|
||||
{Key: utils.GetStringPtr("failed"), Code: &failedCode, Message: &failedMessage},
|
||||
},
|
||||
}
|
||||
|
||||
lockConfig, err := json.Marshal(auth.BucketLockConfig{Enabled: true})
|
||||
assert.NoError(t, err)
|
||||
|
||||
legalHoldOn, legalHoldOff := true, false
|
||||
lockedObjectCode := "AccessDenied"
|
||||
lockedObjectMessage := "Access Denied because object protected by object lock."
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input testInput
|
||||
output testOutput
|
||||
configureMock func(be *BackendMock)
|
||||
}{
|
||||
{
|
||||
name: "verify access fails",
|
||||
input: testInput{
|
||||
locals: accessDeniedLocals,
|
||||
body: validBody,
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrAccessDenied),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid request body",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: []byte("invalid_body"),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrInvalidRequest),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "check object access returns error",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: validBody,
|
||||
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLocked),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrObjectLocked),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "backend returns error",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
beRes: s3response.DeleteResult{},
|
||||
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
||||
body: validBody,
|
||||
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Data: s3response.DeleteResult{},
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
EventName: s3event.EventObjectRemovedDeleteObjects,
|
||||
ObjectCount: 1,
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "successful response",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: validBody,
|
||||
beRes: validRes,
|
||||
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Data: validRes,
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
EventName: s3event.EventObjectRemovedDeleteObjects,
|
||||
ObjectCount: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "quiet mode reports only the objects not deleted",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: quietBody,
|
||||
beRes: quietBackendRes,
|
||||
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Data: s3response.DeleteResult{
|
||||
Error: []types.Error{
|
||||
{Key: utils.GetStringPtr("failed"), Code: &failedCode, Message: &failedMessage},
|
||||
},
|
||||
},
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
EventName: s3event.EventObjectRemovedDeleteObjects,
|
||||
ObjectCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty delete list",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: emptyBody,
|
||||
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Data: s3response.DeleteResult{},
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
EventName: s3event.EventObjectRemovedDeleteObjects,
|
||||
ObjectCount: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "exactly at the 1000 key limit",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: atLimitBody,
|
||||
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Data: s3response.DeleteResult{},
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
EventName: s3event.EventObjectRemovedDeleteObjects,
|
||||
ObjectCount: 1000,
|
||||
},
|
||||
},
|
||||
},
|
||||
configureMock: func(be *BackendMock) {
|
||||
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
assert.Len(t, deleteObjectsInput.Delete.Objects, 1000)
|
||||
return s3response.DeleteResult{}, nil
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "over the 1000 key limit",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: overLimitBody,
|
||||
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrMalformedXML),
|
||||
},
|
||||
configureMock: func(be *BackendMock) {
|
||||
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
t.Error("backend DeleteObjects called for an over-limit request")
|
||||
return s3response.DeleteResult{}, nil
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "object key escaping the bucket",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: invalidKeyBody,
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrBadRequest),
|
||||
},
|
||||
configureMock: func(be *BackendMock) {
|
||||
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
t.Error("backend DeleteObjects called for a request with an invalid key")
|
||||
return s3response.DeleteResult{}, nil
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "missing object key",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: missingKeyBody,
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrBadRequest),
|
||||
},
|
||||
configureMock: func(be *BackendMock) {
|
||||
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
t.Error("backend DeleteObjects called for a request with a missing key")
|
||||
return s3response.DeleteResult{}, nil
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "partial success: one object locked, one succeeds",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
body: partialSuccessBody,
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Data: s3response.DeleteResult{
|
||||
Deleted: []types.DeletedObject{
|
||||
{Key: utils.GetStringPtr("ok")},
|
||||
},
|
||||
Error: []types.Error{
|
||||
{Key: utils.GetStringPtr("locked"), Code: &lockedObjectCode, Message: &lockedObjectMessage},
|
||||
},
|
||||
},
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
EventName: s3event.EventObjectRemovedDeleteObjects,
|
||||
ObjectCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
configureMock: func(be *BackendMock) {
|
||||
be.GetObjectLockConfigurationFunc = func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return lockConfig, nil
|
||||
}
|
||||
be.GetBucketVersioningFunc = func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
||||
return s3response.GetBucketVersioningOutput{}, nil
|
||||
}
|
||||
be.GetObjectRetentionFunc = func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
be.GetObjectLegalHoldFunc = func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
if object == "locked" {
|
||||
return &legalHoldOn, nil
|
||||
}
|
||||
return &legalHoldOff, nil
|
||||
}
|
||||
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
// Only the object that cleared the lock check should
|
||||
// ever reach the backend.
|
||||
assert.Len(t, deleteObjectsInput.Delete.Objects, 1)
|
||||
assert.Equal(t, "ok", *deleteObjectsInput.Delete.Objects[0].Key)
|
||||
return s3response.DeleteResult{
|
||||
Deleted: []types.DeletedObject{{Key: utils.GetStringPtr("ok")}},
|
||||
}, nil
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
be := &BackendMock{
|
||||
DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
return tt.input.beRes.(s3response.DeleteResult), tt.input.beErr
|
||||
},
|
||||
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, tt.input.extraMockErr
|
||||
},
|
||||
}
|
||||
if tt.configureMock != nil {
|
||||
tt.configureMock(be)
|
||||
}
|
||||
|
||||
ctrl := S3ApiController{
|
||||
be: be,
|
||||
}
|
||||
|
||||
testController(
|
||||
t,
|
||||
ctrl.DeleteObjects,
|
||||
tt.output.response,
|
||||
tt.output.err,
|
||||
ctxInputs{
|
||||
locals: tt.input.locals,
|
||||
body: tt.input.body,
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// mockMpFileReader wraps an io.Reader and satisfies utils.MpFileReader.
|
||||
// It tracks the number of bytes delivered to callers so that Length returns
|
||||
// the same value that finalFileReader.Length would return after a real upload.
|
||||
type mockMpFileReader struct {
|
||||
r io.Reader
|
||||
bytesRead int64
|
||||
}
|
||||
|
||||
func (m *mockMpFileReader) Read(p []byte) (int, error) {
|
||||
n, err := m.r.Read(p)
|
||||
m.bytesRead += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (m *mockMpFileReader) Length() int64 { return m.bytesRead }
|
||||
|
||||
func newMockFileReader(content string) *mockMpFileReader {
|
||||
return &mockMpFileReader{r: strings.NewReader(content)}
|
||||
}
|
||||
|
||||
func TestS3ApiController_POSTObject(t *testing.T) {
|
||||
encodePOSTPolicyForControllerTest := func(t *testing.T, expiration time.Time, conditions []any) string {
|
||||
t.Helper()
|
||||
|
||||
policy := map[string]any{
|
||||
"expiration": expiration.UTC().Format(time.RFC3339),
|
||||
"conditions": conditions,
|
||||
}
|
||||
|
||||
b, err := json.Marshal(policy)
|
||||
assert.NoError(t, err)
|
||||
|
||||
return base64.StdEncoding.EncodeToString(b)
|
||||
}
|
||||
postObjectLocalsForTest := func(parsed middlewares.PostObjectResult) map[utils.ContextKey]any {
|
||||
return map[utils.ContextKey]any{
|
||||
utils.ContextKeyIsRoot: true,
|
||||
utils.ContextKeyParsedAcl: auth.ACL{
|
||||
Owner: "root",
|
||||
},
|
||||
utils.ContextKeyAccount: auth.Account{
|
||||
Access: "root",
|
||||
Role: auth.RoleAdmin,
|
||||
},
|
||||
utils.ContextKeyRegion: "us-east-1",
|
||||
utils.ContextKeyObjectPostResult: parsed,
|
||||
}
|
||||
}
|
||||
marshalObjectTaggingForControllerTest := func(t *testing.T, tags []s3response.Tag) string {
|
||||
t.Helper()
|
||||
|
||||
data, err := xml.Marshal(s3response.Tagging{
|
||||
TagSet: s3response.TagSet{
|
||||
Tags: tags,
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
return string(data)
|
||||
}
|
||||
|
||||
validTaggingXML := marshalObjectTaggingForControllerTest(t, []s3response.Tag{
|
||||
{Key: "project", Value: "alpha team"},
|
||||
})
|
||||
baseFields := map[string]string{
|
||||
"key": "uploads/photo.jpg",
|
||||
"file": "ignored",
|
||||
"x-amz-signature": "ignored",
|
||||
}
|
||||
basePolicy := encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
|
||||
map[string]string{"bucket": "bucket"},
|
||||
[]any{"starts-with", "$key", "uploads/"},
|
||||
})
|
||||
baseFields["policy"] = basePolicy
|
||||
|
||||
location := "http://example.com/bucket/uploads%2Fphoto.jpg"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input testInput
|
||||
output testOutput
|
||||
}{
|
||||
{
|
||||
name: "verify access fails",
|
||||
input: testInput{
|
||||
locals: map[utils.ContextKey]any{
|
||||
utils.ContextKeyIsRoot: false,
|
||||
utils.ContextKeyParsedAcl: auth.ACL{
|
||||
Owner: "user",
|
||||
},
|
||||
utils.ContextKeyAccount: auth.Account{
|
||||
Access: "user",
|
||||
Role: auth.RoleUser,
|
||||
},
|
||||
utils.ContextKeyRegion: "us-east-1",
|
||||
utils.ContextKeyObjectPostResult: middlewares.PostObjectResult{
|
||||
Fields: baseFields,
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
},
|
||||
},
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "user",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrAccessDenied),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid policy",
|
||||
input: testInput{
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: map[string]string{
|
||||
"key": "uploads/photo.jpg",
|
||||
"policy": "%%%not-base64%%%",
|
||||
"file": "ignored",
|
||||
"x-amz-signature": "ignored",
|
||||
},
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.InvalidPolicyDocument.InvalidBase64Encoding(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "policy evaluation fails on extra field",
|
||||
input: testInput{
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: map[string]string{
|
||||
"key": "uploads/photo.jpg",
|
||||
"policy": basePolicy,
|
||||
"file": "ignored",
|
||||
"x-amz-signature": "ignored",
|
||||
"unexpected": "value",
|
||||
},
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.InvalidPolicyDocument.ExtraInputField("unexpected"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid tagging xml",
|
||||
input: testInput{
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: map[string]string{
|
||||
"key": "uploads/photo.jpg",
|
||||
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
|
||||
map[string]string{"bucket": "bucket"},
|
||||
[]any{"starts-with", "$key", "uploads/"},
|
||||
[]any{"eq", "$tagging", "invalid-xml"},
|
||||
}),
|
||||
"file": "ignored",
|
||||
"x-amz-signature": "ignored",
|
||||
"tagging": "invalid-xml",
|
||||
},
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrMalformedXML),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid checksum fields",
|
||||
input: testInput{
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: map[string]string{
|
||||
"key": "uploads/photo.jpg",
|
||||
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
|
||||
map[string]string{"bucket": "bucket"},
|
||||
[]any{"starts-with", "$key", "uploads/"},
|
||||
[]any{"eq", "$x-amz-checksum-crc32", "invalid_base64_string"},
|
||||
}),
|
||||
"file": "ignored",
|
||||
"x-amz-signature": "ignored",
|
||||
"x-amz-checksum-crc32": "invalid_base64_string",
|
||||
},
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "metadata too large",
|
||||
input: testInput{
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: map[string]string{
|
||||
"key": "uploads/photo.jpg",
|
||||
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
|
||||
map[string]string{"bucket": "bucket"},
|
||||
[]any{"starts-with", "$key", "uploads/"},
|
||||
[]any{"starts-with", "$x-amz-meta-big", ""},
|
||||
}),
|
||||
"file": "ignored",
|
||||
"x-amz-signature": "ignored",
|
||||
"x-amz-meta-big": strings.Repeat("a", 2050),
|
||||
},
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetMetadataTooLargeErr(2053, 2048),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "overwriting a locked object",
|
||||
input: testInput{
|
||||
// object lock enabled on the bucket; the mock reports a
|
||||
// legal hold on every object
|
||||
extraMockResp: []byte(`{"Enabled":true}`),
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: baseFields,
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrObjectLocked),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "backend returns error",
|
||||
input: testInput{
|
||||
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: baseFields,
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "successful redirect response",
|
||||
input: testInput{
|
||||
beRes: s3response.PutObjectOutput{
|
||||
ETag: "etag-123",
|
||||
VersionID: "vid-123",
|
||||
},
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: map[string]string{
|
||||
"key": "uploads/photo.jpg",
|
||||
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
|
||||
map[string]string{"bucket": "bucket"},
|
||||
[]any{"starts-with", "$key", "uploads/"},
|
||||
[]any{"eq", "$success_action_redirect", "https://client.example/upload-complete"},
|
||||
}),
|
||||
"file": "ignored",
|
||||
"x-amz-signature": "ignored",
|
||||
"success_action_redirect": "https://client.example/upload-complete",
|
||||
},
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Headers: map[string]*string{
|
||||
"Location": utils.GetStringPtr("https://client.example/upload-complete?bucket=bucket&etag=etag-123&key=uploads%2Fphoto.jpg"),
|
||||
},
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
ContentLength: int64(len("payload")),
|
||||
ObjectETag: utils.GetStringPtr("etag-123"),
|
||||
ObjectSize: int64(len("payload")),
|
||||
EventName: s3event.EventObjectCreatedPost,
|
||||
Status: 303,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "successful created response",
|
||||
input: testInput{
|
||||
beRes: s3response.PutObjectOutput{
|
||||
ETag: "etag-123",
|
||||
VersionID: "vid-123",
|
||||
ChecksumCRC32: utils.GetStringPtr("crc32-out"),
|
||||
ChecksumType: types.ChecksumTypeComposite,
|
||||
},
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: map[string]string{
|
||||
"key": "uploads/photo.jpg",
|
||||
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
|
||||
map[string]string{"bucket": "bucket"},
|
||||
[]any{"starts-with", "$key", "uploads/"},
|
||||
[]any{"eq", "$success_action_status", "201"},
|
||||
[]any{"eq", "$tagging", validTaggingXML},
|
||||
[]any{"eq", "$x-amz-meta-owner", "alice"},
|
||||
[]any{"eq", "$x-amz-checksum-crc32", "ww2FVQ=="},
|
||||
[]any{"eq", "$cache-control", "max-age=60"},
|
||||
[]any{"eq", "$content-type", "image/jpeg"},
|
||||
[]any{"eq", "$content-disposition", "inline"},
|
||||
[]any{"eq", "$content-encoding", "gzip"},
|
||||
[]any{"eq", "$content-language", "en-US"},
|
||||
[]any{"eq", "$expires", "Fri, 21 Mar 2026 00:00:00 GMT"},
|
||||
[]any{"eq", "$x-amz-storage-class", string(types.StorageClassGlacier)},
|
||||
}),
|
||||
"file": "ignored",
|
||||
"x-amz-signature": "ignored",
|
||||
"success_action_status": "201",
|
||||
"tagging": validTaggingXML,
|
||||
"x-amz-meta-owner": "alice",
|
||||
"x-amz-checksum-crc32": "ww2FVQ==",
|
||||
"cache-control": "max-age=60",
|
||||
"content-type": "image/jpeg",
|
||||
"content-disposition": "inline",
|
||||
"content-encoding": "gzip",
|
||||
"content-language": "en-US",
|
||||
"expires": "Fri, 21 Mar 2026 00:00:00 GMT",
|
||||
"x-amz-storage-class": string(types.StorageClassGlacier),
|
||||
},
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Headers: map[string]*string{
|
||||
"Etag": utils.GetStringPtr("etag-123"),
|
||||
"Location": &location,
|
||||
"x-amz-checksum-crc32": utils.GetStringPtr("crc32-out"),
|
||||
"x-amz-checksum-crc32c": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-crc64nvme": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-sha1": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-sha256": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-sha512": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-md5": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-xxhash64": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-xxhash3": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-xxhash128": utils.GetStringPtr(""),
|
||||
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
|
||||
"x-amz-version-id": utils.GetStringPtr("vid-123"),
|
||||
},
|
||||
Data: &s3response.PostResponse{
|
||||
Bucket: "bucket",
|
||||
Key: "uploads/photo.jpg",
|
||||
ETag: "etag-123",
|
||||
Location: location,
|
||||
},
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
ContentLength: int64(len("payload")),
|
||||
ObjectETag: utils.GetStringPtr("etag-123"),
|
||||
ObjectSize: int64(len("payload")),
|
||||
EventName: s3event.EventObjectCreatedPost,
|
||||
Status: http.StatusCreated,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "default content type when field is omitted",
|
||||
input: testInput{
|
||||
beRes: s3response.PutObjectOutput{
|
||||
ETag: "etag-123",
|
||||
},
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: baseFields,
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
Headers: map[string]*string{
|
||||
"Etag": utils.GetStringPtr("etag-123"),
|
||||
"Location": &location,
|
||||
"x-amz-checksum-crc32": nil,
|
||||
"x-amz-checksum-crc32c": nil,
|
||||
"x-amz-checksum-crc64nvme": nil,
|
||||
"x-amz-checksum-sha1": nil,
|
||||
"x-amz-checksum-sha256": nil,
|
||||
"x-amz-checksum-sha512": nil,
|
||||
"x-amz-checksum-md5": nil,
|
||||
"x-amz-checksum-xxhash64": nil,
|
||||
"x-amz-checksum-xxhash3": nil,
|
||||
"x-amz-checksum-xxhash128": nil,
|
||||
"x-amz-checksum-type": nil,
|
||||
"x-amz-version-id": nil,
|
||||
},
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
ContentLength: int64(len("payload")),
|
||||
ObjectETag: utils.GetStringPtr("etag-123"),
|
||||
ObjectSize: int64(len("payload")),
|
||||
EventName: s3event.EventObjectCreatedPost,
|
||||
Status: http.StatusNoContent,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "anonymous upload with policy is evaluated",
|
||||
input: testInput{
|
||||
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
|
||||
Fields: map[string]string{
|
||||
"key": "uploads/anon.bin",
|
||||
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
|
||||
map[string]string{"bucket": "bucket"},
|
||||
// key condition intentionally omitted -> ExtraInputField for "key"
|
||||
}),
|
||||
"file": "ignored",
|
||||
},
|
||||
FileRdr: newMockFileReader("payload"),
|
||||
ContentLength: int64(len("payload")),
|
||||
}),
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.InvalidPolicyDocument.ExtraInputField("key"),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
be := &BackendMock{
|
||||
PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
|
||||
if tt.input.beErr != nil {
|
||||
return s3response.PutObjectOutput{}, tt.input.beErr
|
||||
}
|
||||
|
||||
// Drain the body as a real backend would, so that FileRdr.Length()
|
||||
// reflects the actual bytes written after PutObject returns.
|
||||
body, err := io.ReadAll(putObjectInput.Body)
|
||||
assert.NoError(t, err)
|
||||
|
||||
if tt.name == "anonymous upload succeeds without policy" {
|
||||
assert.Equal(t, "uploads/anon.bin", *putObjectInput.Key)
|
||||
assert.Equal(t, "anon-payload", string(body))
|
||||
}
|
||||
|
||||
if tt.name == "default content type when field is omitted" {
|
||||
assert.Equal(t, defaultContentType, *putObjectInput.ContentType)
|
||||
}
|
||||
|
||||
if tt.name == "successful created response" {
|
||||
assert.Equal(t, "bucket", *putObjectInput.Bucket)
|
||||
assert.Equal(t, "uploads/photo.jpg", *putObjectInput.Key)
|
||||
assert.Equal(t, "image/jpeg", *putObjectInput.ContentType)
|
||||
assert.Equal(t, "gzip", *putObjectInput.ContentEncoding)
|
||||
assert.Equal(t, "inline", *putObjectInput.ContentDisposition)
|
||||
assert.Equal(t, "en-US", *putObjectInput.ContentLanguage)
|
||||
assert.Equal(t, "max-age=60", *putObjectInput.CacheControl)
|
||||
assert.Equal(t, "Fri, 21 Mar 2026 00:00:00 GMT", *putObjectInput.Expires)
|
||||
assert.Equal(t, types.StorageClassGlacier, putObjectInput.StorageClass)
|
||||
assert.Equal(t, int64(len("payload")), *putObjectInput.ContentLength)
|
||||
assert.Equal(t, "project=alpha+team", *putObjectInput.Tagging)
|
||||
assert.Equal(t, map[string]string{"owner": "alice"}, putObjectInput.Metadata)
|
||||
assert.Equal(t, utils.GetStringPtr("ww2FVQ=="), putObjectInput.ChecksumCRC32)
|
||||
assert.Equal(t, "payload", string(body))
|
||||
}
|
||||
|
||||
return tt.input.beRes.(s3response.PutObjectOutput), nil
|
||||
},
|
||||
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
},
|
||||
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
|
||||
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
|
||||
if tt.input.extraMockResp != nil {
|
||||
return tt.input.extraMockResp.([]byte), nil
|
||||
}
|
||||
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
|
||||
},
|
||||
GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
|
||||
},
|
||||
GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
legalHold := true
|
||||
return &legalHold, nil
|
||||
},
|
||||
}
|
||||
|
||||
ctrl := S3ApiController{
|
||||
be: be,
|
||||
}
|
||||
|
||||
testController(
|
||||
t,
|
||||
ctrl.POSTObject,
|
||||
tt.output.response,
|
||||
tt.output.err,
|
||||
ctxInputs{
|
||||
locals: tt.input.locals,
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,497 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/iam"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
func DeleteObjects_non_existing_bucket(s *S3Conf) error {
|
||||
testName := "DeleteObjects_non_existing_bucket"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
bckt := getBucketName()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bckt,
|
||||
Delete: &types.Delete{
|
||||
Objects: []types.ObjectIdentifier{{Key: getPtr("obj1")}, {Key: getPtr("obj2")}},
|
||||
},
|
||||
})
|
||||
cancel()
|
||||
return checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket))
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteObjects_empty_input(s *S3Conf) error {
|
||||
testName := "DeleteObjects_empty_input"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
contents, err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{
|
||||
Objects: []types.ObjectIdentifier{},
|
||||
},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(out.Deleted) != 0 {
|
||||
return fmt.Errorf("expected deleted object count 0, instead got %v",
|
||||
len(out.Deleted))
|
||||
}
|
||||
if len(out.Errors) != 0 {
|
||||
return fmt.Errorf("expected 0 errors, instead got %v", len(out.Errors))
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !compareObjects(contents, res.Contents) {
|
||||
return fmt.Errorf("expected the output to be %v, instead got %v",
|
||||
contents, res.Contents)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteObjects_non_existing_objects(s *S3Conf) error {
|
||||
testName := "DeleteObjects_empty_input"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
delObjects := []types.ObjectIdentifier{{Key: getPtr("obj1")}, {Key: getPtr("obj2")}}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{
|
||||
Objects: delObjects,
|
||||
},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(out.Deleted) != 2 {
|
||||
return fmt.Errorf("expected deleted object count 2, instead got %v",
|
||||
len(out.Deleted))
|
||||
}
|
||||
if len(out.Errors) != 0 {
|
||||
return fmt.Errorf("expected 0 errors, instead got %v, %v",
|
||||
len(out.Errors), out.Errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteObjects_success(s *S3Conf) error {
|
||||
testName := "DeleteObjects_success"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
objects, objToDel := []string{"obj1", "obj2", "obj3"}, []string{"foo", "bar", "baz"}
|
||||
contents, err := putObjects(s3client, append(objToDel, objects...), bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
delObjects := []types.ObjectIdentifier{}
|
||||
delResult := []types.DeletedObject{}
|
||||
for _, key := range objToDel {
|
||||
k := key
|
||||
delObjects = append(delObjects, types.ObjectIdentifier{Key: &k})
|
||||
delResult = append(delResult, types.DeletedObject{Key: &k})
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{
|
||||
Objects: delObjects,
|
||||
},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(out.Deleted) != 3 {
|
||||
return fmt.Errorf("expected deleted object count 3, instead got %v",
|
||||
len(out.Deleted))
|
||||
}
|
||||
if len(out.Errors) != 0 {
|
||||
return fmt.Errorf("expected 2 errors, instead got %v",
|
||||
len(out.Errors))
|
||||
}
|
||||
|
||||
if !compareDelObjects(delResult, out.Deleted) {
|
||||
return fmt.Errorf("unexpected deleted output")
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !compareObjects(contents[3:], res.Contents) {
|
||||
return fmt.Errorf("expected the output to be %v, instead got %v",
|
||||
contents[3:], res.Contents)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteObjects_quiet_mode checks that a quiet-mode request deletes the
|
||||
// objects but lists none of them, since quiet mode reports only the objects
|
||||
// that could not be deleted.
|
||||
func DeleteObjects_quiet_mode(s *S3Conf) error {
|
||||
testName := "DeleteObjects_quiet_mode"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
contents, err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{
|
||||
Objects: []types.ObjectIdentifier{{Key: getPtr("bar")}, {Key: getPtr("baz")}},
|
||||
Quiet: getBoolPtr(true),
|
||||
},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(out.Deleted) != 0 {
|
||||
return fmt.Errorf("expected no deleted objects in quiet mode, instead got %v",
|
||||
len(out.Deleted))
|
||||
}
|
||||
if len(out.Errors) != 0 {
|
||||
return fmt.Errorf("expected no errors, instead got %v", len(out.Errors))
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !compareObjects(contents[2:], res.Contents) {
|
||||
return fmt.Errorf("expected the output to be %v, instead got %v",
|
||||
contents[2:], res.Contents)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteObjects_iam_mixed_denials_and_success covers a single batch mixing
|
||||
// every DeleteObjects outcome at once: a key the identity policy denies, a
|
||||
// governance-locked key with no bypass, and keys the caller may freely
|
||||
// delete. All three outcomes land in one response — no top-level error —
|
||||
// with Deleted and Errors each preserving the order the keys were
|
||||
// requested in.
|
||||
func DeleteObjects_iam_mixed_denials_and_success(s *S3Conf) error {
|
||||
testName := "DeleteObjects_iam_mixed_denials_and_success"
|
||||
return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error {
|
||||
for _, key := range []string{"allowed/one", "allowed/two", "denied/one"} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &key})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := putGovernanceLockedObject(s, bucket, "locked/one"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
user, cleanup, err := newS3IAMUser(root, s, map[string]string{
|
||||
"p": policyDoc(accessStatement{
|
||||
Effect: "Allow", Action: actS3DeleteObject,
|
||||
Resource: []string{objectArn(bucket, "allowed/*"), objectArn(bucket, "locked/*")},
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
delObjects := []types.ObjectIdentifier{
|
||||
{Key: getPtr("allowed/one")},
|
||||
{Key: getPtr("denied/one")},
|
||||
{Key: getPtr("locked/one")},
|
||||
{Key: getPtr("allowed/two")},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{Objects: delObjects},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err)
|
||||
}
|
||||
|
||||
if err := checkDeletedKeysInOrder(out.Deleted, []string{"allowed/one", "allowed/two"}); err != nil {
|
||||
return err
|
||||
}
|
||||
return checkDeleteObjectsErrsInOrder(out.Errors, []keyDenial{
|
||||
{"denied/one", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "denied/one"))},
|
||||
{"locked/one", s3err.GetAPIError(s3err.ErrObjectLocked)},
|
||||
})
|
||||
}, withLock())
|
||||
}
|
||||
|
||||
// DeleteObjects_iam_all_access_denied covers a batch where the identity
|
||||
// policy grants nothing at all: the call itself still succeeds — no
|
||||
// top-level error — with every object reported denied in Errors, in
|
||||
// request order, and nothing in Deleted.
|
||||
func DeleteObjects_iam_all_access_denied(s *S3Conf) error {
|
||||
testName := "DeleteObjects_iam_all_access_denied"
|
||||
return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error {
|
||||
for _, key := range []string{"one", "two", "three"} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s.GetClient().PutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &key})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
user, cleanup, err := newS3IAMUser(root, s, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
delObjects := []types.ObjectIdentifier{
|
||||
{Key: getPtr("one")}, {Key: getPtr("two")}, {Key: getPtr("three")},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{Objects: delObjects},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err)
|
||||
}
|
||||
|
||||
if len(out.Deleted) != 0 {
|
||||
return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted)
|
||||
}
|
||||
return checkDeleteObjectsErrsInOrder(out.Errors, []keyDenial{
|
||||
{"one", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "one"))},
|
||||
{"two", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "two"))},
|
||||
{"three", wantImplicitDeny(user.arn, actS3DeleteObject, objectArn(bucket, "three"))},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteObjects_iam_all_locked covers a batch where every object is
|
||||
// governance-locked and none is deleted: the call still succeeds — no
|
||||
// top-level error — with every object reported denied in Errors, in
|
||||
// request order, and nothing in Deleted. Omitting the bypass header
|
||||
// entirely reports the generic object-lock message; sending the header
|
||||
// without s3:BypassGovernanceRetention reports the specific AccessDenied
|
||||
// naming that action instead — the same distinction the single-object
|
||||
// DELETE path makes, now confirmed for the batch path too.
|
||||
func DeleteObjects_iam_all_locked(s *S3Conf) error {
|
||||
testName := "DeleteObjects_iam_all_locked"
|
||||
return s3IAMActionHandler(s, testName, func(root *iam.Client, bucket string) error {
|
||||
for _, key := range []string{"locked/one", "locked/two"} {
|
||||
if err := putGovernanceLockedObject(s, bucket, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
user, cleanup, err := newS3IAMUser(root, s, map[string]string{
|
||||
"p": policyDoc(accessStatement{
|
||||
Effect: "Allow", Action: actS3DeleteObject, Resource: objectsArn(bucket),
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
delObjects := []types.ObjectIdentifier{{Key: getPtr("locked/one")}, {Key: getPtr("locked/two")}}
|
||||
|
||||
// No bypass header at all: the generic object-lock message.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{Objects: delObjects},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err)
|
||||
}
|
||||
if len(out.Deleted) != 0 {
|
||||
return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted)
|
||||
}
|
||||
if err := checkDeleteObjectsErrsInOrder(out.Errors, []keyDenial{
|
||||
{"locked/one", s3err.GetAPIError(s3err.ErrObjectLocked)},
|
||||
{"locked/two", s3err.GetAPIError(s3err.ErrObjectLocked)},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("without bypass header: %w", err)
|
||||
}
|
||||
|
||||
// Bypass header sent, but the identity policy doesn't grant
|
||||
// s3:BypassGovernanceRetention: a specific AccessDenied naming that
|
||||
// action, not the generic object-lock message.
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err = user.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{Objects: delObjects},
|
||||
BypassGovernanceRetention: getPtr(true),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("expected DeleteObjects to succeed with per-object denials, not fail outright: %w", err)
|
||||
}
|
||||
if len(out.Deleted) != 0 {
|
||||
return fmt.Errorf("expected nothing deleted, got %+v", out.Deleted)
|
||||
}
|
||||
if err := checkDeleteObjectsErrsInOrder(out.Errors, []keyDenial{
|
||||
{"locked/one", wantImplicitDeny(user.arn, actS3BypassGovernance, objectArn(bucket, "locked/one"))},
|
||||
{"locked/two", wantImplicitDeny(user.arn, actS3BypassGovernance, objectArn(bucket, "locked/two"))},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("with bypass header, no permission: %w", err)
|
||||
}
|
||||
return nil
|
||||
}, withLock())
|
||||
}
|
||||
|
||||
// DeleteObjects_key_limit pins the S3 limit of 1000 keys per request: a
|
||||
// batch of exactly 1000 succeeds, and 1001 is rejected with MalformedXML
|
||||
// before any of the keys is deleted.
|
||||
func DeleteObjects_key_limit(s *S3Conf) error {
|
||||
testName := "DeleteObjects_key_limit"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
delObjects := make([]types.ObjectIdentifier, 1001)
|
||||
for i := range delObjects {
|
||||
delObjects[i] = types.ObjectIdentifier{Key: getPtr(fmt.Sprintf("key-%d", i))}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{Objects: delObjects[:1000]},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("expected a 1000 key delete to succeed: %w", err)
|
||||
}
|
||||
if len(out.Deleted) != 1000 {
|
||||
return fmt.Errorf("expected 1000 deleted objects, instead got %v", len(out.Deleted))
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{Objects: delObjects},
|
||||
})
|
||||
cancel()
|
||||
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMalformedXML)); err != nil {
|
||||
return fmt.Errorf("expected 1001 keys to be rejected: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteObjects_invalid_object_keys(s *S3Conf) error {
|
||||
testName := "DeleteObjects_invalid_object_keys"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
obj, otherObj := "my-obj", "other-obj"
|
||||
_, err := putObjects(s3client, []string{obj, otherObj}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Every key resolves outside the bucket or to the bucket itself;
|
||||
// the ones climbing back into it name obj. A request holding any of
|
||||
// them is rejected as a whole, so its valid key isn't deleted
|
||||
// either.
|
||||
for _, key := range []string{
|
||||
".",
|
||||
"..",
|
||||
"/",
|
||||
"../",
|
||||
"../../../etc/passwd",
|
||||
"../" + bucket + "/" + obj,
|
||||
"dir/../../" + bucket + "/" + obj,
|
||||
} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delete: &types.Delete{
|
||||
Objects: []types.ObjectIdentifier{
|
||||
{Key: &otherObj},
|
||||
{Key: &key},
|
||||
},
|
||||
},
|
||||
})
|
||||
cancel()
|
||||
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrBadRequest)); err != nil {
|
||||
return fmt.Errorf("key %q: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range []string{obj, otherObj} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("head %v: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
# Copyright 2026 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
load ./bats-support/load
|
||||
load ./bats-assert/load
|
||||
|
||||
source ./tests/drivers/create_bucket/create_bucket_rest.sh
|
||||
source ./tests/drivers/delete_objects/delete_objects_rest.sh
|
||||
source ./tests/setup.sh
|
||||
|
||||
# tags: curl, DeleteObjects, malformed-message
|
||||
@test "REST - DeleteObjects - missing payload" {
|
||||
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
read -r bucket_name file_name <<< "$output"
|
||||
|
||||
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
|
||||
run send_rest_go_command_expect_error "400" "MissingRequestBodyError" "is empty" "-bucketName" "$bucket_name" "-method" "POST" "-query" "delete" "-contentMD5"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# tags: curl, DeleteObjects, malformed-message
|
||||
@test "REST - DeleteObjects - no objects added to payload" {
|
||||
if [ "$DIRECT" != "true" ]; then
|
||||
skip "https://github.com/versity/versitygw/issues/2109"
|
||||
fi
|
||||
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
read -r bucket_name file_name <<< "$output"
|
||||
|
||||
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
|
||||
run send_rest_go_command_expect_error "400" "MalformedXML" "was not well-formed" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" "-contentMD5"
|
||||
assert_success
|
||||
|
||||
run get_object_rest "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/$file_name-copy"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# tags: curl, DeleteObjects
|
||||
@test "REST - DeleteObjects - nonexistent key" {
|
||||
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
read -r bucket_name file_name <<< "$output"
|
||||
|
||||
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
|
||||
local incorrect_file_name="${file_name}a"
|
||||
run send_rest_go_command_callback "200" "check_xml_element_contains" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \
|
||||
"-contentMD5" "-objectsToDelete" "key=$incorrect_file_name" "--" "$incorrect_file_name" "DeleteResult" "Deleted" "Key"
|
||||
assert_success
|
||||
|
||||
run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# tags: curl, DeleteObjects
|
||||
@test "REST - DeleteObjects - ETag mismatch" {
|
||||
if [ "$DIRECT" != "true" ]; then
|
||||
skip "https://github.com/versity/versitygw/issues/2112"
|
||||
fi
|
||||
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
read -r bucket_name file_name <<< "$output"
|
||||
|
||||
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
|
||||
run send_rest_go_command_callback "200" "check_delete_objects_precondition_error" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \
|
||||
"-contentMD5" "-objectsToDelete" "key=$file_name;eTag=abc" "--" "$file_name"
|
||||
assert_success
|
||||
|
||||
run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# tags: curl, DeleteObjects
|
||||
@test "REST - DeleteObject - ETag match" {
|
||||
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
read -r bucket_name file_name <<< "$output"
|
||||
|
||||
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
|
||||
run get_etag_rest "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
etag="$output"
|
||||
|
||||
run send_rest_go_command_callback "200" "check_xml_element_contains" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \
|
||||
"-contentMD5" "-objectsToDelete" "key=$file_name;eTag=$etag" "--" "$file_name" "DeleteResult" "Deleted" "Key"
|
||||
assert_success
|
||||
|
||||
run verify_object_not_found "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# tags: curl, DeleteObjects, versioning
|
||||
@test "REST - DeleteObjects - version ID mismatch" {
|
||||
if [ "$DIRECT" != "true" ]; then
|
||||
skip "https://github.com/versity/versitygw/issues/2119"
|
||||
fi
|
||||
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
read -r bucket_name file_name <<< "$output"
|
||||
|
||||
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
|
||||
incorrect_version="abc"
|
||||
run send_rest_go_command_callback "200" "check_delete_objects_version_error" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \
|
||||
"-contentMD5" "-objectsToDelete" "key=$file_name;versionId=$incorrect_version" "--" "$file_name" "$incorrect_version"
|
||||
assert_success
|
||||
|
||||
run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# tags: curl, DeleteObjects, versioning
|
||||
@test "REST - DeleteObjects - version ID" {
|
||||
if [ "$RECREATE_BUCKETS" == "false" ]; then
|
||||
skip "cannot change versioning status for static buckets"
|
||||
fi
|
||||
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
read -r bucket_name file_name <<< "$output"
|
||||
|
||||
run put_bucket_versioning_rest "$bucket_name" "Enabled"
|
||||
assert_success
|
||||
|
||||
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
|
||||
run send_rest_go_command_callback "200" "parse_version_id" "-bucketName" "$bucket_name" "-query" "versions" "--" "true"
|
||||
assert_success
|
||||
version_id="$output"
|
||||
|
||||
run send_rest_go_command_callback "200" "check_xml_element_contains" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \
|
||||
"-contentMD5" "-objectsToDelete" "key=$file_name;versionId=$version_id" "--" "$file_name" "DeleteResult" "Deleted" "Key"
|
||||
assert_success
|
||||
|
||||
run verify_object_not_found "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# tags: curl, DeleteObjects
|
||||
@test "REST - DeleteObjects - quiet mode" {
|
||||
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
read -r bucket_name file_name <<< "$output"
|
||||
|
||||
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
|
||||
run send_rest_go_command_callback "200" "check_for_empty_or_nonexistent_element" "-bucketName" "$bucket_name" "-commandType" "deleteObjects" \
|
||||
"-contentMD5" "-objectsToDelete" "key=${file_name}" "-deleteObjectsQuietMode" "--" "DeleteResult"
|
||||
assert_success
|
||||
|
||||
run verify_object_not_found "$bucket_name" "$file_name"
|
||||
assert_success
|
||||
}
|
||||
Reference in new issue
Block a user