Upstream versity/versitygw state of the touched files before 63922ed89d

This commit is contained in:
upstream committed 2026-10-02 16:02:57 +02:00
1 parent 5f0e2e55ba
commit edbb6c2d74
4 files changed
+2030

No files matched your search

+383
View File
@@ -0,0 +1,383 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package controllers
import (
"encoding/xml"
"net/http"
"net/url"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v3"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3response"
)
func (c S3ApiController) DeleteObjects(ctx fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
bypass := auth.BypassModeForRequest(strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true"))
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx)
// The body has to be parsed before authorization, not after: real AWS
// authorizes s3:DeleteObject against each object's own ARN, so the keys
// are part of what is being authorized. The parsed objects go straight
// to VerifyObjectsAccess, which checks policy and object locks for all
// of them in one pass.
var dObj s3response.DeleteObjects
err := xml.Unmarshal(ctx.BodyRaw(), &dObj)
if err != nil {
debuglogger.Logf("error unmarshalling delete objects: %v", err)
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
// S3 caps a single DeleteObjects request at 1000 keys. Reject an
// over-limit request before authorization and before anything reaches
// the backend, so a too-large batch can't be applied partially.
if len(dObj.Objects) > maxDeleteObjects {
debuglogger.Logf("delete objects: %d keys exceeds the limit of %d",
len(dObj.Objects), maxDeleteObjects)
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrMalformedXML)
}
// The keys come from the body, so they skip the object name check the
// request path gets. A key resolving outside the bucket ("../key") names
// a path outside it on a filesystem backend, so the whole request is
// rejected before any key is authorized or deleted.
for _, obj := range dObj.Objects {
if obj.Key == nil || !utils.IsObjectNameValid(*obj.Key) {
debuglogger.Logf("delete objects: invalid object key: %q",
backend.GetStringFromPtr(obj.Key))
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrBadRequest)
}
}
// checkErrs holds one entry per requested object — nil where it may
// proceed to the backend, an AWS-shaped denial otherwise. DeleteObjects
// supports partial success, so a denial on one object (policy or object
// lock) must not fail any other: only the objects that clear this check
// are sent to the backend, and the rest are reported as per-object
// errors directly from checkErrs. err here is a whole-request failure
// (readonly mode, or an error resolving policy/lock state), not about
// any one object.
checkErrs, err := c.verifyObjectsAccess(ctx,
auth.AccessOptions{
Acl: parsedAcl,
AclPermission: auth.PermissionWrite,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
IsPublicRequest: IsBucketPublic,
}, dObj.Objects, bypass)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
toDelete := make([]types.ObjectIdentifier, 0, len(dObj.Objects))
for i, obj := range dObj.Objects {
if checkErrs[i] == nil {
toDelete = append(toDelete, obj)
}
}
var backendResult s3response.DeleteResult
if len(toDelete) > 0 {
backendResult, err = c.be.DeleteObjects(ctx.RequestCtx(),
&s3.DeleteObjectsInput{
Bucket: &bucket,
Delete: &types.Delete{
Objects: toDelete,
},
})
}
result := utils.MergeDeleteObjectsResult(dObj.Objects, checkErrs, backendResult)
// In quiet mode the response lists only the objects that could not be
// deleted.
if dObj.Quiet {
result.Deleted = nil
}
return &Response{
Data: result,
MetaOpts: &MetaOptions{
ObjectCount: int64(len(dObj.Objects)),
BucketOwner: parsedAcl.Owner,
EventName: s3event.EventObjectRemovedDeleteObjects,
},
}, err
}
func (c S3ApiController) POSTObject(ctx fiber.Ctx) (*Response, error) {
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx)
parsed := utils.ContextKeyObjectPostResult.Get(ctx).(middlewares.PostObjectResult)
bucket := ctx.Params("bucket")
contentType := parsed.Fields["content-type"]
if contentType == "" {
contentType = defaultContentType
}
contentEncoding := parsed.Fields["content-encoding"]
contentDisposition := parsed.Fields["content-disposition"]
contentLanguage := parsed.Fields["content-language"]
cacheControl := parsed.Fields["cache-control"]
expires := parsed.Fields["expires"]
websiteRedirectLocation := parsed.Fields["x-amz-website-redirect-location"]
storageClass := parsed.Fields["x-amz-storage-class"]
key := parsed.Fields["key"]
// parse POST policy — absent for anonymous uploads to public buckets
if !IsBucketPublic {
policyBase64 := parsed.Fields["policy"]
policy, err := auth.ParsePOSTPolicyBase64(policyBase64)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
// Evaluate post policy
err = policy.Evaluate(auth.PostPolicyEvalInput{
Bucket: bucket,
Key: key,
ContentLength: parsed.ContentLength,
Fields: parsed.Fields,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
}
// convert object tagging from raw XML to Query string
// to pass PutObject, which expects the tagging to be a query string
var tagging string
if taggingXML, ok := parsed.Fields["tagging"]; ok {
var err error
tagging, err = utils.ConvertTaggingXMLToQueryString([]byte(taggingXML))
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
}
// parse checksum headers
checksums, err := utils.ParseCalculatedChecksumFields(parsed.Fields)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
// extract metadata
metadata, err := utils.ExtractMetadataFromFields(parsed.Fields)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
err = utils.ValidateWebsiteRedirectLocation(websiteRedirectLocation)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
// A POST upload is an s3:PutObject on the object named by the form's
// key field, so it is authorized against that object's ARN — the same
// resource PutObject is — not the bucket's. Tagging the object also
// takes s3:PutObjectTagging, but only for a non-empty tag set
actions := auth.ObjectUploadActions(tagging, "", "", "")
err = c.verifyAccess(ctx,
auth.AccessOptions{
Acl: parsedAcl,
AclPermission: auth.PermissionWrite,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Actions: actions,
IsPublicRequest: IsBucketPublic,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
err = auth.CheckObjectAccess(ctx, bucket, acct, []types.ObjectIdentifier{{Key: &key}}, auth.BypassOverwrite, IsBucketPublic, c.be, c.iam, true)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
res, err := c.be.PutObject(ctx.RequestCtx(), s3response.PutObjectInput{
Bucket: &bucket,
Key: &key,
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ContentDisposition: &contentDisposition,
ContentLanguage: &contentLanguage,
CacheControl: &cacheControl,
Expires: &expires,
WebsiteRedirectLocation: &websiteRedirectLocation,
Body: parsed.FileRdr,
ContentLength: &parsed.ContentLength,
Tagging: &tagging,
Metadata: metadata,
StorageClass: types.StorageClass(storageClass),
ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, err
}
if successActionRedirect, ok := parsed.Fields["success_action_redirect"]; ok {
u, err := url.Parse(successActionRedirect)
if err == nil {
q := u.Query()
q.Set("bucket", bucket)
q.Set("key", key)
q.Set("etag", res.ETag)
u.RawQuery = q.Encode()
redirectURI := u.String()
return &Response{
Headers: map[string]*string{
"Location": &redirectURI,
},
MetaOpts: &MetaOptions{
ContentLength: parsed.FileRdr.Length(),
BucketOwner: parsedAcl.Owner,
ObjectETag: &res.ETag,
ObjectSize: parsed.FileRdr.Length(),
EventName: s3event.EventObjectCreatedPost,
Status: http.StatusSeeOther,
},
}, nil
}
}
respStatus := http.StatusNoContent
var respBody any
location := utils.GenerateObjectLocation(ctx, c.virtualDomain, bucket, key)
if successStatus, ok := parsed.Fields["success_action_status"]; ok {
switch successStatus {
case "200":
respStatus = http.StatusOK
case "201":
respStatus = http.StatusCreated
respBody = &s3response.PostResponse{
Bucket: bucket,
Key: key,
ETag: res.ETag,
Location: location,
}
}
}
return &Response{
Headers: map[string]*string{
"Etag": &res.ETag,
"Location": &location,
"x-amz-checksum-crc32": res.ChecksumCRC32,
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-version-id": utils.GetStringPtr(res.VersionID),
},
Data: respBody,
MetaOpts: &MetaOptions{
ContentLength: parsed.FileRdr.Length(),
BucketOwner: parsedAcl.Owner,
ObjectETag: &res.ETag,
ObjectSize: parsed.FileRdr.Length(),
EventName: s3event.EventObjectCreatedPost,
Status: respStatus,
},
}, nil
}
+972
View File
@@ -0,0 +1,972 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package controllers
import (
"context"
"encoding/base64"
"encoding/json"
"encoding/xml"
"fmt"
"io"
"net/http"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3response"
)
func TestS3ApiController_DeleteObjects(t *testing.T) {
validBody, err := xml.Marshal(s3response.DeleteObjects{
Objects: []types.ObjectIdentifier{
{Key: utils.GetStringPtr("obj")},
},
})
assert.NoError(t, err)
validRes := s3response.DeleteResult{
Deleted: []types.DeletedObject{
{Key: utils.GetStringPtr("obj")},
},
}
partialSuccessBody, err := xml.Marshal(s3response.DeleteObjects{
Objects: []types.ObjectIdentifier{
{Key: utils.GetStringPtr("locked")},
{Key: utils.GetStringPtr("ok")},
},
})
assert.NoError(t, err)
// 1000 keys is the S3 limit for a single DeleteObjects request; 1001
// must be rejected before any of them reaches the backend. The counts
// are spelled out rather than derived from the limit constant so the
// test pins the documented boundary.
const keyLimit = 1000
keyLimitObjs := make([]types.ObjectIdentifier, keyLimit+1)
for i := range keyLimitObjs {
keyLimitObjs[i] = types.ObjectIdentifier{Key: utils.GetStringPtr(fmt.Sprintf("key-%d", i))}
}
atLimitBody, err := xml.Marshal(s3response.DeleteObjects{Objects: keyLimitObjs[:keyLimit]})
assert.NoError(t, err)
overLimitBody, err := xml.Marshal(s3response.DeleteObjects{Objects: keyLimitObjs})
assert.NoError(t, err)
emptyBody, err := xml.Marshal(s3response.DeleteObjects{Objects: []types.ObjectIdentifier{}})
assert.NoError(t, err)
invalidKeyBody, err := xml.Marshal(s3response.DeleteObjects{
Objects: []types.ObjectIdentifier{
{Key: utils.GetStringPtr("obj")},
{Key: utils.GetStringPtr("../../outside/file")},
},
})
assert.NoError(t, err)
missingKeyBody, err := xml.Marshal(s3response.DeleteObjects{
Objects: []types.ObjectIdentifier{
{Key: utils.GetStringPtr("obj")},
{VersionId: utils.GetStringPtr("null")},
},
})
assert.NoError(t, err)
quietBody, err := xml.Marshal(s3response.DeleteObjects{
Objects: []types.ObjectIdentifier{
{Key: utils.GetStringPtr("ok")},
{Key: utils.GetStringPtr("failed")},
},
Quiet: true,
})
assert.NoError(t, err)
failedCode, failedMessage := "InternalError", "We encountered an internal error. Please try again."
quietBackendRes := s3response.DeleteResult{
Deleted: []types.DeletedObject{{Key: utils.GetStringPtr("ok")}},
Error: []types.Error{
{Key: utils.GetStringPtr("failed"), Code: &failedCode, Message: &failedMessage},
},
}
lockConfig, err := json.Marshal(auth.BucketLockConfig{Enabled: true})
assert.NoError(t, err)
legalHoldOn, legalHoldOff := true, false
lockedObjectCode := "AccessDenied"
lockedObjectMessage := "Access Denied because object protected by object lock."
tests := []struct {
name string
input testInput
output testOutput
configureMock func(be *BackendMock)
}{
{
name: "verify access fails",
input: testInput{
locals: accessDeniedLocals,
body: validBody,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrAccessDenied),
},
},
{
name: "invalid request body",
input: testInput{
locals: defaultLocals,
body: []byte("invalid_body"),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidRequest),
},
},
{
name: "check object access returns error",
input: testInput{
locals: defaultLocals,
body: validBody,
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLocked),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrObjectLocked),
},
},
{
name: "backend returns error",
input: testInput{
locals: defaultLocals,
beRes: s3response.DeleteResult{},
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
body: validBody,
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: s3response.DeleteResult{},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventObjectRemovedDeleteObjects,
ObjectCount: 1,
},
},
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
},
},
{
name: "successful response",
input: testInput{
locals: defaultLocals,
body: validBody,
beRes: validRes,
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: validRes,
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventObjectRemovedDeleteObjects,
ObjectCount: 1,
},
},
},
},
{
name: "quiet mode reports only the objects not deleted",
input: testInput{
locals: defaultLocals,
body: quietBody,
beRes: quietBackendRes,
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: s3response.DeleteResult{
Error: []types.Error{
{Key: utils.GetStringPtr("failed"), Code: &failedCode, Message: &failedMessage},
},
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventObjectRemovedDeleteObjects,
ObjectCount: 2,
},
},
},
},
{
name: "empty delete list",
input: testInput{
locals: defaultLocals,
body: emptyBody,
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: s3response.DeleteResult{},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventObjectRemovedDeleteObjects,
ObjectCount: 0,
},
},
},
},
{
name: "exactly at the 1000 key limit",
input: testInput{
locals: defaultLocals,
body: atLimitBody,
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
Data: s3response.DeleteResult{},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventObjectRemovedDeleteObjects,
ObjectCount: 1000,
},
},
},
configureMock: func(be *BackendMock) {
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
assert.Len(t, deleteObjectsInput.Delete.Objects, 1000)
return s3response.DeleteResult{}, nil
}
},
},
{
name: "over the 1000 key limit",
input: testInput{
locals: defaultLocals,
body: overLimitBody,
extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMalformedXML),
},
configureMock: func(be *BackendMock) {
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
t.Error("backend DeleteObjects called for an over-limit request")
return s3response.DeleteResult{}, nil
}
},
},
{
name: "object key escaping the bucket",
input: testInput{
locals: defaultLocals,
body: invalidKeyBody,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrBadRequest),
},
configureMock: func(be *BackendMock) {
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
t.Error("backend DeleteObjects called for a request with an invalid key")
return s3response.DeleteResult{}, nil
}
},
},
{
name: "missing object key",
input: testInput{
locals: defaultLocals,
body: missingKeyBody,
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrBadRequest),
},
configureMock: func(be *BackendMock) {
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
t.Error("backend DeleteObjects called for a request with a missing key")
return s3response.DeleteResult{}, nil
}
},
},
{
name: "partial success: one object locked, one succeeds",
input: testInput{
locals: defaultLocals,
body: partialSuccessBody,
},
output: testOutput{
response: &Response{
Data: s3response.DeleteResult{
Deleted: []types.DeletedObject{
{Key: utils.GetStringPtr("ok")},
},
Error: []types.Error{
{Key: utils.GetStringPtr("locked"), Code: &lockedObjectCode, Message: &lockedObjectMessage},
},
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
EventName: s3event.EventObjectRemovedDeleteObjects,
ObjectCount: 2,
},
},
},
configureMock: func(be *BackendMock) {
be.GetObjectLockConfigurationFunc = func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return lockConfig, nil
}
be.GetBucketVersioningFunc = func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
return s3response.GetBucketVersioningOutput{}, nil
}
be.GetObjectRetentionFunc = func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
return []byte("{}"), nil
}
be.GetObjectLegalHoldFunc = func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
if object == "locked" {
return &legalHoldOn, nil
}
return &legalHoldOff, nil
}
be.DeleteObjectsFunc = func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
// Only the object that cleared the lock check should
// ever reach the backend.
assert.Len(t, deleteObjectsInput.Delete.Objects, 1)
assert.Equal(t, "ok", *deleteObjectsInput.Delete.Objects[0].Key)
return s3response.DeleteResult{
Deleted: []types.DeletedObject{{Key: utils.GetStringPtr("ok")}},
}, nil
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
be := &BackendMock{
DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
return tt.input.beRes.(s3response.DeleteResult), tt.input.beErr
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, tt.input.extraMockErr
},
}
if tt.configureMock != nil {
tt.configureMock(be)
}
ctrl := S3ApiController{
be: be,
}
testController(
t,
ctrl.DeleteObjects,
tt.output.response,
tt.output.err,
ctxInputs{
locals: tt.input.locals,
body: tt.input.body,
})
})
}
}
// mockMpFileReader wraps an io.Reader and satisfies utils.MpFileReader.
// It tracks the number of bytes delivered to callers so that Length returns
// the same value that finalFileReader.Length would return after a real upload.
type mockMpFileReader struct {
r io.Reader
bytesRead int64
}
func (m *mockMpFileReader) Read(p []byte) (int, error) {
n, err := m.r.Read(p)
m.bytesRead += int64(n)
return n, err
}
func (m *mockMpFileReader) Length() int64 { return m.bytesRead }
func newMockFileReader(content string) *mockMpFileReader {
return &mockMpFileReader{r: strings.NewReader(content)}
}
func TestS3ApiController_POSTObject(t *testing.T) {
encodePOSTPolicyForControllerTest := func(t *testing.T, expiration time.Time, conditions []any) string {
t.Helper()
policy := map[string]any{
"expiration": expiration.UTC().Format(time.RFC3339),
"conditions": conditions,
}
b, err := json.Marshal(policy)
assert.NoError(t, err)
return base64.StdEncoding.EncodeToString(b)
}
postObjectLocalsForTest := func(parsed middlewares.PostObjectResult) map[utils.ContextKey]any {
return map[utils.ContextKey]any{
utils.ContextKeyIsRoot: true,
utils.ContextKeyParsedAcl: auth.ACL{
Owner: "root",
},
utils.ContextKeyAccount: auth.Account{
Access: "root",
Role: auth.RoleAdmin,
},
utils.ContextKeyRegion: "us-east-1",
utils.ContextKeyObjectPostResult: parsed,
}
}
marshalObjectTaggingForControllerTest := func(t *testing.T, tags []s3response.Tag) string {
t.Helper()
data, err := xml.Marshal(s3response.Tagging{
TagSet: s3response.TagSet{
Tags: tags,
},
})
assert.NoError(t, err)
return string(data)
}
validTaggingXML := marshalObjectTaggingForControllerTest(t, []s3response.Tag{
{Key: "project", Value: "alpha team"},
})
baseFields := map[string]string{
"key": "uploads/photo.jpg",
"file": "ignored",
"x-amz-signature": "ignored",
}
basePolicy := encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
})
baseFields["policy"] = basePolicy
location := "http://example.com/bucket/uploads%2Fphoto.jpg"
tests := []struct {
name string
input testInput
output testOutput
}{
{
name: "verify access fails",
input: testInput{
locals: map[utils.ContextKey]any{
utils.ContextKeyIsRoot: false,
utils.ContextKeyParsedAcl: auth.ACL{
Owner: "user",
},
utils.ContextKeyAccount: auth.Account{
Access: "user",
Role: auth.RoleUser,
},
utils.ContextKeyRegion: "us-east-1",
utils.ContextKeyObjectPostResult: middlewares.PostObjectResult{
Fields: baseFields,
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
},
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "user",
},
},
err: s3err.GetAPIError(s3err.ErrAccessDenied),
},
},
{
name: "invalid policy",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": "%%%not-base64%%%",
"file": "ignored",
"x-amz-signature": "ignored",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.InvalidPolicyDocument.InvalidBase64Encoding(),
},
},
{
name: "policy evaluation fails on extra field",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": basePolicy,
"file": "ignored",
"x-amz-signature": "ignored",
"unexpected": "value",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.InvalidPolicyDocument.ExtraInputField("unexpected"),
},
},
{
name: "invalid tagging xml",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$tagging", "invalid-xml"},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"tagging": "invalid-xml",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrMalformedXML),
},
},
{
name: "invalid checksum fields",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$x-amz-checksum-crc32", "invalid_base64_string"},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"x-amz-checksum-crc32": "invalid_base64_string",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32"),
},
},
{
name: "metadata too large",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"starts-with", "$x-amz-meta-big", ""},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"x-amz-meta-big": strings.Repeat("a", 2050),
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetMetadataTooLargeErr(2053, 2048),
},
},
{
name: "overwriting a locked object",
input: testInput{
// object lock enabled on the bucket; the mock reports a
// legal hold on every object
extraMockResp: []byte(`{"Enabled":true}`),
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: baseFields,
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrObjectLocked),
},
},
{
name: "backend returns error",
input: testInput{
beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket),
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: baseFields,
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
},
},
{
name: "successful redirect response",
input: testInput{
beRes: s3response.PutObjectOutput{
ETag: "etag-123",
VersionID: "vid-123",
},
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$success_action_redirect", "https://client.example/upload-complete"},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"success_action_redirect": "https://client.example/upload-complete",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
Headers: map[string]*string{
"Location": utils.GetStringPtr("https://client.example/upload-complete?bucket=bucket&etag=etag-123&key=uploads%2Fphoto.jpg"),
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
ContentLength: int64(len("payload")),
ObjectETag: utils.GetStringPtr("etag-123"),
ObjectSize: int64(len("payload")),
EventName: s3event.EventObjectCreatedPost,
Status: 303,
},
},
},
},
{
name: "successful created response",
input: testInput{
beRes: s3response.PutObjectOutput{
ETag: "etag-123",
VersionID: "vid-123",
ChecksumCRC32: utils.GetStringPtr("crc32-out"),
ChecksumType: types.ChecksumTypeComposite,
},
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/photo.jpg",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
[]any{"starts-with", "$key", "uploads/"},
[]any{"eq", "$success_action_status", "201"},
[]any{"eq", "$tagging", validTaggingXML},
[]any{"eq", "$x-amz-meta-owner", "alice"},
[]any{"eq", "$x-amz-checksum-crc32", "ww2FVQ=="},
[]any{"eq", "$cache-control", "max-age=60"},
[]any{"eq", "$content-type", "image/jpeg"},
[]any{"eq", "$content-disposition", "inline"},
[]any{"eq", "$content-encoding", "gzip"},
[]any{"eq", "$content-language", "en-US"},
[]any{"eq", "$expires", "Fri, 21 Mar 2026 00:00:00 GMT"},
[]any{"eq", "$x-amz-storage-class", string(types.StorageClassGlacier)},
}),
"file": "ignored",
"x-amz-signature": "ignored",
"success_action_status": "201",
"tagging": validTaggingXML,
"x-amz-meta-owner": "alice",
"x-amz-checksum-crc32": "ww2FVQ==",
"cache-control": "max-age=60",
"content-type": "image/jpeg",
"content-disposition": "inline",
"content-encoding": "gzip",
"content-language": "en-US",
"expires": "Fri, 21 Mar 2026 00:00:00 GMT",
"x-amz-storage-class": string(types.StorageClassGlacier),
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
Headers: map[string]*string{
"Etag": utils.GetStringPtr("etag-123"),
"Location": &location,
"x-amz-checksum-crc32": utils.GetStringPtr("crc32-out"),
"x-amz-checksum-crc32c": utils.GetStringPtr(""),
"x-amz-checksum-crc64nvme": utils.GetStringPtr(""),
"x-amz-checksum-sha1": utils.GetStringPtr(""),
"x-amz-checksum-sha256": utils.GetStringPtr(""),
"x-amz-checksum-sha512": utils.GetStringPtr(""),
"x-amz-checksum-md5": utils.GetStringPtr(""),
"x-amz-checksum-xxhash64": utils.GetStringPtr(""),
"x-amz-checksum-xxhash3": utils.GetStringPtr(""),
"x-amz-checksum-xxhash128": utils.GetStringPtr(""),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("vid-123"),
},
Data: &s3response.PostResponse{
Bucket: "bucket",
Key: "uploads/photo.jpg",
ETag: "etag-123",
Location: location,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
ContentLength: int64(len("payload")),
ObjectETag: utils.GetStringPtr("etag-123"),
ObjectSize: int64(len("payload")),
EventName: s3event.EventObjectCreatedPost,
Status: http.StatusCreated,
},
},
},
},
{
name: "default content type when field is omitted",
input: testInput{
beRes: s3response.PutObjectOutput{
ETag: "etag-123",
},
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: baseFields,
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
Headers: map[string]*string{
"Etag": utils.GetStringPtr("etag-123"),
"Location": &location,
"x-amz-checksum-crc32": nil,
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-checksum-type": nil,
"x-amz-version-id": nil,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
ContentLength: int64(len("payload")),
ObjectETag: utils.GetStringPtr("etag-123"),
ObjectSize: int64(len("payload")),
EventName: s3event.EventObjectCreatedPost,
Status: http.StatusNoContent,
},
},
},
},
{
name: "anonymous upload with policy is evaluated",
input: testInput{
locals: postObjectLocalsForTest(middlewares.PostObjectResult{
Fields: map[string]string{
"key": "uploads/anon.bin",
"policy": encodePOSTPolicyForControllerTest(t, time.Now().Add(15*time.Minute), []any{
map[string]string{"bucket": "bucket"},
// key condition intentionally omitted -> ExtraInputField for "key"
}),
"file": "ignored",
},
FileRdr: newMockFileReader("payload"),
ContentLength: int64(len("payload")),
}),
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.InvalidPolicyDocument.ExtraInputField("key"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
be := &BackendMock{
PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
if tt.input.beErr != nil {
return s3response.PutObjectOutput{}, tt.input.beErr
}
// Drain the body as a real backend would, so that FileRdr.Length()
// reflects the actual bytes written after PutObject returns.
body, err := io.ReadAll(putObjectInput.Body)
assert.NoError(t, err)
if tt.name == "anonymous upload succeeds without policy" {
assert.Equal(t, "uploads/anon.bin", *putObjectInput.Key)
assert.Equal(t, "anon-payload", string(body))
}
if tt.name == "default content type when field is omitted" {
assert.Equal(t, defaultContentType, *putObjectInput.ContentType)
}
if tt.name == "successful created response" {
assert.Equal(t, "bucket", *putObjectInput.Bucket)
assert.Equal(t, "uploads/photo.jpg", *putObjectInput.Key)
assert.Equal(t, "image/jpeg", *putObjectInput.ContentType)
assert.Equal(t, "gzip", *putObjectInput.ContentEncoding)
assert.Equal(t, "inline", *putObjectInput.ContentDisposition)
assert.Equal(t, "en-US", *putObjectInput.ContentLanguage)
assert.Equal(t, "max-age=60", *putObjectInput.CacheControl)
assert.Equal(t, "Fri, 21 Mar 2026 00:00:00 GMT", *putObjectInput.Expires)
assert.Equal(t, types.StorageClassGlacier, putObjectInput.StorageClass)
assert.Equal(t, int64(len("payload")), *putObjectInput.ContentLength)
assert.Equal(t, "project=alpha+team", *putObjectInput.Tagging)
assert.Equal(t, map[string]string{"owner": "alice"}, putObjectInput.Metadata)
assert.Equal(t, utils.GetStringPtr("ww2FVQ=="), putObjectInput.ChecksumCRC32)
assert.Equal(t, "payload", string(body))
}
return tt.input.beRes.(s3response.PutObjectOutput), nil
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
},
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
if tt.input.extraMockResp != nil {
return tt.input.extraMockResp.([]byte), nil
}
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
},
GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
},
GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
legalHold := true
return &legalHold, nil
},
}
ctrl := S3ApiController{
be: be,
}
testController(
t,
ctrl.POSTObject,
tt.output.response,
tt.output.err,
ctxInputs{
locals: tt.input.locals,
},
)
})
}
}