diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go new file mode 100644 index 0000000..5e49076 --- /dev/null +++ b/s3api/controllers/object-get.go @@ -0,0 +1,639 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "fmt" + "math" + "net/http" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) GetObjectTagging(ctx fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + if err := utils.ValidateVersionId(ctx); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + action := auth.GetObjectTaggingAction + if versionId != "" { + action = auth.GetObjectVersionTaggingAction + } + + err := c.verifyAccess(ctx, auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: []auth.Action{action}, + IsPublicRequest: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetObjectTagging(ctx.RequestCtx(), bucket, key, versionId) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + tags := s3response.Tagging{ + TagSet: s3response.TagSet{Tags: []s3response.Tag{}}, + } + + for key, val := range data { + tags.TagSet.Tags = append(tags.TagSet.Tags, + s3response.Tag{Key: key, Value: val}) + } + + return &Response{ + Data: tags, + Headers: map[string]*string{ + "x-amz-version-id": &versionId, + }, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, nil +} + +func (c S3ApiController) GetObjectRetention(ctx fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + if err := utils.ValidateVersionId(ctx); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err := c.verifyAccess(ctx, auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: []auth.Action{auth.GetObjectRetentionAction}, + IsPublicRequest: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetObjectRetention(ctx.RequestCtx(), bucket, key, versionId) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + retention, err := auth.ParseObjectLockRetentionOutput(data) + return &Response{ + Data: retention, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObjectLegalHold(ctx fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + if err := utils.ValidateVersionId(ctx); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err := c.verifyAccess(ctx, auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: []auth.Action{auth.GetObjectLegalHoldAction}, + IsPublicRequest: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetObjectLegalHold(ctx.RequestCtx(), bucket, key, versionId) + return &Response{ + Data: auth.ParseObjectLegalHoldOutput(data), + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObjectAcl(ctx fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := c.verifyAccess(ctx, auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionReadAcp, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: []auth.Action{auth.GetObjectAclAction}, + IsPublicRequest: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + res, err := c.be.GetObjectAcl(ctx.RequestCtx(), &s3.GetObjectAclInput{ + Bucket: &bucket, + Key: &key, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) ListParts(ctx fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + ssec := utils.ExtractSSECHeaders(ctx) + uploadId := ctx.Query("uploadId") + partNumberMarker := ctx.Query("part-number-marker") + maxPartsStr := ctx.Query("max-parts") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := c.verifyAccess(ctx, auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: []auth.Action{auth.ListMultipartUploadPartsAction}, + IsPublicRequest: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // parse/validate the part number marker + _, err = utils.ParseMaxLimiter(partNumberMarker, utils.LimiterTypePartNumberMarker) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // parse the max parts + maxParts, err := utils.ParseMaxLimiter(maxPartsStr, utils.LimiterTypeMaxParts) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.ListParts(ctx.RequestCtx(), &s3.ListPartsInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + PartNumberMarker: &partNumberMarker, + MaxParts: &maxParts, + SSECustomerAlgorithm: ssec.Algorithm, + SSECustomerKey: ssec.Key, + SSECustomerKeyMD5: ssec.KeyMD5, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObjectAttributes(ctx fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + maxPartsStr := ctx.Get("X-Amz-Max-Parts") + partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + if err := utils.ValidateVersionId(ctx); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + action := auth.GetObjectAttributesAction + if versionId != "" { + action = auth.GetObjectVersionAttributesAction + } + + err := c.verifyAccess(ctx, auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: []auth.Action{action}, + IsPublicRequest: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + ssec, err := c.parseSSECHeaders(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var maxParts *int32 + // parse max parts + parsed, err := utils.ParseMaxLimiter(maxPartsStr, utils.LimiterTypeMaxParts) + if err == nil { + maxParts = &parsed + } + + // parse the object attributes + attrs, err := utils.ParseObjectAttributes(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.GetObjectAttributes(ctx.RequestCtx(), + &s3.GetObjectAttributesInput{ + SSECustomerAlgorithm: ssec.Algorithm, + SSECustomerKey: ssec.Key, + SSECustomerKeyMD5: ssec.KeyMD5, + Bucket: &bucket, + Key: &key, + PartNumberMarker: &partNumberMarker, + MaxParts: maxParts, + VersionId: &versionId, + }) + if err != nil { + headers := map[string]*string{ + "x-amz-version-id": res.VersionId, + } + if res.DeleteMarker != nil && *res.DeleteMarker { + headers["x-amz-delete-marker"] = utils.GetStringPtr("true") + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + headers := map[string]*string{ + "x-amz-version-id": res.VersionId, + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, iso8601TimeFormatExtended), + } + if res.DeleteMarker != nil && *res.DeleteMarker { + headers["x-amz-delete-marker"] = utils.GetStringPtr("true") + } + + return &Response{ + Headers: headers, + Data: utils.FilterObjectAttributes(attrs, res), + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObject(ctx fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + acceptRange := ctx.Get("Range") + checksumMode := types.ChecksumMode(strings.ToUpper(ctx.Get("x-amz-checksum-mode"))) + partNumberQuery := int32(fiber.Query[int](ctx, "partNumber", -1)) + + // Extract response override query parameters + responseOverrides := map[string]*string{ + "Cache-Control": utils.GetQueryParam(ctx, "response-cache-control"), + "Content-Disposition": utils.GetQueryParam(ctx, "response-content-disposition"), + "Content-Encoding": utils.GetQueryParam(ctx, "response-content-encoding"), + "Content-Language": utils.GetQueryParam(ctx, "response-content-language"), + "Content-Type": utils.GetQueryParam(ctx, "response-content-type"), + "Expires": utils.GetQueryParam(ctx, "response-expires"), + } + + // Check if any response override parameters are present + hasResponseOverrides := false + for _, override := range responseOverrides { + if override != nil { + hasResponseOverrides = true + break + } + } + + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucketRequest := utils.ContextKeyPublicBucket.IsSet(ctx) + utils.ContextKeySkipResBodyLog.Set(ctx, true) + + // Validate that response override parameters are not used with anonymous requests + if hasResponseOverrides && isPublicBucketRequest { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders) + } + + if err := utils.ValidateVersionId(ctx); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + action := auth.GetObjectAction + if ctx.Request().URI().QueryArgs().Has("versionId") { + action = auth.GetObjectVersionAction + } + + err := c.verifyAccess(ctx, auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: []auth.Action{action}, + IsPublicRequest: isPublicBucketRequest, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + ssec, err := c.parseSSECHeaders(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var partNumber *int32 + if ctx.Request().URI().QueryArgs().Has("partNumber") { + if partNumberQuery < minPartNumber || partNumberQuery > int32(c.effectiveMpMaxParts()) { + debuglogger.Logf("invalid part number: %d", partNumberQuery) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, ctx.Query("partNumber")) + } + + if acceptRange != "" { + debuglogger.Logf("Range and partNumber cannot both be specified") + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrRangeAndPartNumber) + } + + partNumber = &partNumberQuery + } + + // validate the checksum mode + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode") + } + + conditionalHeaders := utils.ParsePreconditionHeaders(ctx) + + res, err := c.be.GetObject(ctx.RequestCtx(), &s3.GetObjectInput{ + SSECustomerAlgorithm: ssec.Algorithm, + SSECustomerKey: ssec.Key, + SSECustomerKeyMD5: ssec.KeyMD5, + Bucket: &bucket, + Key: &key, + Range: &acceptRange, + IfMatch: conditionalHeaders.IfMatch, + IfNoneMatch: conditionalHeaders.IfNoneMatch, + IfModifiedSince: conditionalHeaders.IfModSince, + IfUnmodifiedSince: conditionalHeaders.IfUnmodeSince, + VersionId: &versionId, + ChecksumMode: checksumMode, + PartNumber: partNumber, + }) + if err != nil { + var headers map[string]*string + if res != nil { + headers = map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), + } + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // Set x-amz-meta-... headers + utils.SetMetaHeaders(ctx, res.Metadata) + + status := http.StatusOK + if res.ContentRange != nil && *res.ContentRange != "" { + status = http.StatusPartialContent + } + + if res.Body != nil { + // -1 will stream response body until EOF if content length not set + contentLen := -1 + if res.ContentLength != nil { + if *res.ContentLength > int64(math.MaxInt) { + debuglogger.Logf("content length %v int overflow", + *res.ContentLength) + return &Response{ + MetaOpts: &MetaOptions{ + ContentLength: utils.GetInt64(res.ContentLength), + BucketOwner: parsedAcl.Owner, + Status: status, + }, + }, s3err.GetInvalidRangeErr("", *res.ContentLength) + } + contentLen = int(*res.ContentLength) + } + utils.StreamResponseBody(ctx, res.Body, contentLen) + } + + return &Response{ + Headers: map[string]*string{ + "ETag": res.ETag, + "x-amz-restore": res.Restore, + "accept-ranges": res.AcceptRanges, + "Content-Range": res.ContentRange, + "Content-Disposition": utils.ApplyOverride(res.ContentDisposition, responseOverrides["Content-Disposition"]), + "Content-Encoding": utils.ApplyOverride(res.ContentEncoding, responseOverrides["Content-Encoding"]), + "Content-Language": utils.ApplyOverride(res.ContentLanguage, responseOverrides["Content-Language"]), + "Cache-Control": utils.ApplyOverride(res.CacheControl, responseOverrides["Cache-Control"]), + "Expires": utils.ApplyOverride(res.ExpiresString, responseOverrides["Expires"]), + "x-amz-website-redirect-location": res.WebsiteRedirectLocation, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-sha512": res.ChecksumSHA512, + "x-amz-checksum-md5": res.ChecksumMD5, + "x-amz-checksum-xxhash64": res.ChecksumXXHASH64, + "x-amz-checksum-xxhash3": res.ChecksumXXHASH3, + "x-amz-checksum-xxhash128": res.ChecksumXXHASH128, + "Content-Type": utils.ApplyOverride(res.ContentType, responseOverrides["Content-Type"]), + "x-amz-version-id": res.VersionId, + "Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength), + "x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount), + "x-amz-tagging-count": utils.ConvertPtrToStringPtr(res.TagCount), + "x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode), + "x-amz-object-lock-legal-hold": utils.ConvertToStringPtr(res.ObjectLockLegalHoldStatus), + "x-amz-storage-class": utils.ConvertToStringPtr(res.StorageClass), + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-object-lock-retain-until-date": utils.FormatDatePtrToString(res.ObjectLockRetainUntilDate, time.RFC3339), + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), + "x-amz-server-side-encryption-customer-algorithm": res.SSECustomerAlgorithm, + "x-amz-server-side-encryption-customer-key-MD5": res.SSECustomerKeyMD5, + }, + MetaOpts: &MetaOptions{ + ContentLength: utils.GetInt64(res.ContentLength), + BucketOwner: parsedAcl.Owner, + Status: status, + }, + }, nil +} diff --git a/s3api/controllers/object-get_test.go b/s3api/controllers/object-get_test.go new file mode 100644 index 0000000..a9b7d70 --- /dev/null +++ b/s3api/controllers/object-get_test.go @@ -0,0 +1,1006 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/oklog/ulid/v2" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_GetObjectTagging(t *testing.T) { + versionId := ulid.Make().String() + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "empty versionId query param", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "versionId": "", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgEmptyVersionId, ""), + }, + }, + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: map[string]string{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + queries: map[string]string{ + "versionId": versionId, + }, + locals: defaultLocals, + beRes: map[string]string{ + "key": "val", + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-version-id": utils.GetStringPtr(versionId), + }, + Data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + {Key: "key", Value: "val"}, + }, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectTaggingFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (map[string]string, error) { + return tt.input.beRes.(map[string]string), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectTagging, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetObjectRetention(t *testing.T) { + retBytes, err := json.Marshal(types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + }) + assert.NoError(t, err) + + var retention *types.ObjectLockRetention + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "empty versionId query param", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "versionId": "", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgEmptyVersionId, ""), + }, + }, + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "invalid data from backend", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + }, + output: testOutput{ + response: &Response{ + Data: retention, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: fmt.Errorf("parse object lock retention: "), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: retBytes, + }, + output: testOutput{ + response: &Response{ + Data: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectRetention, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetObjectLegalHold(t *testing.T) { + var legalHold *bool + var emptyLegalHold *s3response.GetObjectLegalHoldResult + status := true + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "empty versionId query param", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "versionId": "", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgEmptyVersionId, ""), + }, + }, + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: legalHold, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: emptyLegalHold, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: &status, + }, + output: testOutput{ + response: &Response{ + Data: &s3response.GetObjectLegalHoldResult{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) { + return tt.input.beRes.(*bool), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectLegalHold, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetObjectAcl(t *testing.T) { + var emptyRes *s3.GetObjectAclOutput + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: emptyRes, + beErr: s3err.GetAPIError(s3err.ErrNotImplemented), + }, + output: testOutput{ + response: &Response{ + Data: emptyRes, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNotImplemented), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: &s3.GetObjectAclOutput{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("something"), + }, + }, + }, + output: testOutput{ + response: &Response{ + Data: &s3.GetObjectAclOutput{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("something"), + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) { + return tt.input.beRes.(*s3.GetObjectAclOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectAcl, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_ListParts(t *testing.T) { + listPartsResult := s3response.ListPartsResult{ + Bucket: "my-bucket", + Key: "obj", + IsTruncated: false, + Parts: []s3response.Part{ + {ETag: "ETag"}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid part number marker", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "part-number-marker": "foo", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgMaxLimiter(string(utils.LimiterTypePartNumberMarker), "foo"), + }, + }, + { + name: "negative max parts", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-parts": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgNegativeMaxLimiter(string(utils.LimiterTypeMaxParts), "-1"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListPartsResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListPartsResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listPartsResult, + headers: map[string]string{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": "invalid", + "X-Amz-Server-Side-Encryption-Customer-Key": "malformed-key", + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": "malformed-md5", + }, + }, + output: testOutput{ + response: &Response{ + Data: listPartsResult, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListPartsFunc: func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error) { + if tt.input.headers != nil { + assert.Equal(t, "invalid", *listPartsInput.SSECustomerAlgorithm) + assert.Equal(t, "malformed-key", *listPartsInput.SSECustomerKey) + assert.Equal(t, "malformed-md5", *listPartsInput.SSECustomerKeyMD5) + } + return tt.input.beRes.(s3response.ListPartsResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListParts, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + headers: tt.input.headers, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetObjectAttributes(t *testing.T) { + delMarker, lastModTime, etag := true, time.Now(), "ETag" + timeFormatted := lastModTime.UTC().Format(iso8601TimeFormatExtended) + + validRes := s3response.GetObjectAttributesResponse{ + DeleteMarker: &delMarker, + LastModified: &lastModTime, + VersionId: utils.GetStringPtr("versionId"), + ETag: &etag, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "empty versionId query param", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "versionId": "", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgEmptyVersionId, ""), + }, + }, + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid object attributes", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Object-Attributes": "invalid_attribute", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgObjectAttributes, "invalid_attribute"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + headers: map[string]string{ + "X-Amz-Object-Attributes": "ETag", + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-version-id": utils.GetStringPtr("versionId"), + "x-amz-delete-marker": utils.GetStringPtr("true"), + }, + Data: nil, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + headers: map[string]string{ + "X-Amz-Object-Attributes": "ETag", + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-version-id": utils.GetStringPtr("versionId"), + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": &timeFormatted, + }, + Data: s3response.GetObjectAttributesResponse{ + ETag: &etag, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) { + return tt.input.beRes.(s3response.GetObjectAttributesResponse), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectAttributes, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + headers: tt.input.headers, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetObject(t *testing.T) { + tm := time.Now() + cLength := int64(11) + rdr := io.NopCloser(strings.NewReader("hello world")) + delMarker := true + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "empty versionId query param", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "versionId": "", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgEmptyVersionId, ""), + }, + }, + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid checksum mode", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "x-amz-checksum-mode": "invalid_checksum_mode", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), + }, + }, + { + name: "invalid part number", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "partNumber": "-2", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, "-2"), + }, + }, + { + name: "both partNumber and Range", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "partNumber": "2", + }, + headers: map[string]string{ + "Range": "bytes=10-20", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrRangeAndPartNumber), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + beRes: &s3.GetObjectOutput{ + DeleteMarker: &delMarker, + LastModified: &tm, + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.GetStringPtr(tm.UTC().Format(timefmt)), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + }, + }, + { + name: "successful response", + input: testInput{ + headers: map[string]string{ + "Range": "100-200", + }, + queries: map[string]string{ + "versionId": "01BX5ZZKBKACTAV9WEVGEMMVRZ", + }, + locals: defaultLocals, + beRes: &s3.GetObjectOutput{ + ETag: utils.GetStringPtr("ETag"), + ContentType: utils.GetStringPtr("application/xml"), + ContentLength: &cLength, + Body: rdr, + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": utils.GetStringPtr("ETag"), + "x-amz-restore": nil, + "accept-ranges": nil, + "Content-Range": nil, + "Content-Disposition": nil, + "Content-Encoding": nil, + "Content-Language": nil, + "Cache-Control": nil, + "Expires": nil, + "x-amz-website-redirect-location": nil, + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, + "x-amz-version-id": nil, + "x-amz-mp-parts-count": nil, + "x-amz-object-lock-mode": nil, + "x-amz-object-lock-legal-hold": nil, + "x-amz-storage-class": nil, + "x-amz-checksum-type": nil, + "x-amz-object-lock-retain-until-date": nil, + "Last-Modified": nil, + "x-amz-tagging-count": nil, + "x-amz-server-side-encryption-customer-algorithm": nil, + "x-amz-server-side-encryption-customer-key-MD5": nil, + "Content-Type": utils.GetStringPtr("application/xml"), + "Content-Length": utils.GetStringPtr("11"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusOK, + ContentLength: cLength, + }, + }, + }, + }, + { + name: "successful partial response", + input: testInput{ + locals: defaultLocals, + beRes: &s3.GetObjectOutput{ + ContentRange: utils.GetStringPtr("bytes 2-4/15"), + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": nil, + "x-amz-restore": nil, + "accept-ranges": nil, + "Content-Range": utils.GetStringPtr("bytes 2-4/15"), + "Content-Disposition": nil, + "Content-Encoding": nil, + "Content-Language": nil, + "Cache-Control": nil, + "Expires": nil, + "x-amz-website-redirect-location": nil, + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, + "x-amz-version-id": nil, + "x-amz-mp-parts-count": nil, + "x-amz-object-lock-mode": nil, + "x-amz-object-lock-legal-hold": nil, + "x-amz-storage-class": nil, + "x-amz-checksum-type": nil, + "x-amz-object-lock-retain-until-date": nil, + "Last-Modified": nil, + "x-amz-tagging-count": nil, + "x-amz-server-side-encryption-customer-algorithm": nil, + "x-amz-server-side-encryption-customer-key-MD5": nil, + "Content-Type": nil, + "Content-Length": nil, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusPartialContent, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + return tt.input.beRes.(*s3.GetObjectOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + headers: tt.input.headers, + queries: tt.input.queries, + }) + }) + } +} diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go new file mode 100644 index 0000000..2f023d3 --- /dev/null +++ b/s3api/controllers/object-head.go @@ -0,0 +1,231 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v3" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +func (c S3ApiController) HeadObject(ctx fiber.Ctx) (*Response, error) { + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + // url values + bucket := ctx.Params("bucket") + partNumberQuery := int32(fiber.Query(ctx, "partNumber", -1)) + versionId := ctx.Query("versionId") + objRange := ctx.Get("Range") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + + // Extract response override query parameters + responseOverrides := map[string]*string{ + "Cache-Control": utils.GetQueryParam(ctx, "response-cache-control"), + "Content-Disposition": utils.GetQueryParam(ctx, "response-content-disposition"), + "Content-Encoding": utils.GetQueryParam(ctx, "response-content-encoding"), + "Content-Language": utils.GetQueryParam(ctx, "response-content-language"), + "Content-Type": utils.GetQueryParam(ctx, "response-content-type"), + "Expires": utils.GetQueryParam(ctx, "response-expires"), + } + + // Check if any response override parameters are present + hasResponseOverrides := false + for _, override := range responseOverrides { + if override != nil { + hasResponseOverrides = true + break + } + } + + // Validate that response override parameters are not used with anonymous requests + if hasResponseOverrides && isPublicBucket { + debuglogger.Logf("anonymous access is denied with response override params") + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders) + } + + if err := utils.ValidateVersionId(ctx); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + action := auth.GetObjectAction + if ctx.Request().URI().QueryArgs().Has("versionId") { + action = auth.GetObjectVersionAction + } + + err := c.verifyAccess(ctx, + auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Actions: []auth.Action{action}, + IsPublicRequest: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + ssec, err := c.parseSSECHeaders(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var partNumber *int32 + if ctx.Request().URI().QueryArgs().Has("partNumber") { + if partNumberQuery < minPartNumber || partNumberQuery > int32(c.effectiveMpMaxParts()) { + debuglogger.Logf("invalid part number: %d", partNumberQuery) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, ctx.Query("partNumber")) + } + + if objRange != "" { + debuglogger.Logf("Range and partNumber cannot both be specified") + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrRangeAndPartNumber) + } + + partNumber = &partNumberQuery + } + + checksumMode := types.ChecksumMode(strings.ToUpper(ctx.Get("x-amz-checksum-mode"))) + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode") + } + + conditionalHeaders := utils.ParsePreconditionHeaders(ctx) + + res, err := c.be.HeadObject(ctx.RequestCtx(), + &s3.HeadObjectInput{ + SSECustomerAlgorithm: ssec.Algorithm, + SSECustomerKey: ssec.Key, + SSECustomerKeyMD5: ssec.KeyMD5, + Bucket: &bucket, + Key: &key, + PartNumber: partNumber, + VersionId: &versionId, + ChecksumMode: checksumMode, + Range: &objRange, + IfMatch: conditionalHeaders.IfMatch, + IfNoneMatch: conditionalHeaders.IfNoneMatch, + IfModifiedSince: conditionalHeaders.IfModSince, + IfUnmodifiedSince: conditionalHeaders.IfUnmodeSince, + }) + if err != nil { + var headers map[string]*string + if res != nil { + headers = map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.GetStringPtr(res.LastModified.UTC().Format(timefmt)), + } + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // Set the metadata headers + utils.SetMetaHeaders(ctx, res.Metadata) + + status := http.StatusOK + if res.ContentRange != nil && *res.ContentRange != "" { + status = http.StatusPartialContent + } + + return &Response{ + Headers: map[string]*string{ + "Content-Range": res.ContentRange, + "Content-Disposition": utils.ApplyOverride(res.ContentDisposition, responseOverrides["Content-Disposition"]), + "Content-Encoding": utils.ApplyOverride(res.ContentEncoding, responseOverrides["Content-Encoding"]), + "Content-Language": utils.ApplyOverride(res.ContentLanguage, responseOverrides["Content-Language"]), + "Cache-Control": utils.ApplyOverride(res.CacheControl, responseOverrides["Cache-Control"]), + "Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength), + "Content-Type": utils.ApplyOverride(res.ContentType, responseOverrides["Content-Type"]), + "Expires": utils.ApplyOverride(res.ExpiresString, responseOverrides["Expires"]), + "x-amz-website-redirect-location": res.WebsiteRedirectLocation, + "ETag": res.ETag, + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), + "x-amz-restore": res.Restore, + "accept-ranges": res.AcceptRanges, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-sha512": res.ChecksumSHA512, + "x-amz-checksum-md5": res.ChecksumMD5, + "x-amz-checksum-xxhash64": res.ChecksumXXHASH64, + "x-amz-checksum-xxhash3": res.ChecksumXXHASH3, + "x-amz-checksum-xxhash128": res.ChecksumXXHASH128, + "x-amz-version-id": res.VersionId, + "x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount), + "x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode), + "x-amz-object-lock-legal-hold": utils.ConvertToStringPtr(res.ObjectLockLegalHoldStatus), + "x-amz-storage-class": utils.ConvertToStringPtr(res.StorageClass), + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-object-lock-retain-until-date": utils.FormatDatePtrToString(res.ObjectLockRetainUntilDate, time.RFC3339), + "x-amz-tagging-count": utils.ConvertPtrToStringPtr(res.TagCount), + "x-amz-server-side-encryption-customer-algorithm": res.SSECustomerAlgorithm, + "x-amz-server-side-encryption-customer-key-MD5": res.SSECustomerKeyMD5, + }, + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + Status: status, + }, + }, nil +} diff --git a/s3api/controllers/object-head_test.go b/s3api/controllers/object-head_test.go new file mode 100644 index 0000000..ed168ce --- /dev/null +++ b/s3api/controllers/object-head_test.go @@ -0,0 +1,316 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +func TestS3ApiController_HeadObject(t *testing.T) { + tm := time.Now() + cLength := int64(100) + + failingBeRes := &s3.HeadObjectOutput{ + LastModified: &tm, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "empty versionId query param", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "versionId": "", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgEmptyVersionId, ""), + }, + }, + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "anonymous access with override params", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + utils.ContextKeyRegion: "us-east-1", + utils.ContextKeyPublicBucket: true, + }, + queries: map[string]string{ + "response-expires": "something", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders), + }, + }, + { + name: "invalid part number", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "partNumber": "-4", + "versionId": "01BX5ZZKBKACTAV9WEVGEMMVRZ", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, "-4"), + }, + }, + { + name: "both partNumber and Range", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "partNumber": "6", + }, + headers: map[string]string{ + "Range": "bytes=1-3", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrRangeAndPartNumber), + }, + }, + { + name: "invalid checksum mode", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "x-amz-checksum-mode": "invalid_checksum_mode", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + beRes: failingBeRes, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.GetStringPtr(tm.UTC().Format(timefmt)), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + }, + }, + { + name: "successful response", + input: testInput{ + queries: map[string]string{ + "partNumber": "4", + }, + locals: defaultLocals, + headers: map[string]string{ + "x-amz-checksum-mode": "enabled", + }, + beRes: &s3.HeadObjectOutput{ + ETag: utils.GetStringPtr("ETag"), + ContentType: utils.GetStringPtr("application/xml"), + ContentLength: &cLength, + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": utils.GetStringPtr("ETag"), + "x-amz-restore": nil, + "accept-ranges": nil, + "Content-Range": nil, + "Content-Disposition": nil, + "Content-Encoding": nil, + "Content-Language": nil, + "Cache-Control": nil, + "Expires": nil, + "x-amz-website-redirect-location": nil, + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, + "x-amz-version-id": nil, + "x-amz-mp-parts-count": nil, + "x-amz-object-lock-mode": nil, + "x-amz-object-lock-legal-hold": nil, + "x-amz-storage-class": nil, + "x-amz-checksum-type": nil, + "x-amz-object-lock-retain-until-date": nil, + "Last-Modified": nil, + "x-amz-tagging-count": nil, + "x-amz-server-side-encryption-customer-algorithm": nil, + "x-amz-server-side-encryption-customer-key-MD5": nil, + "Content-Type": utils.GetStringPtr("application/xml"), + "Content-Length": utils.GetStringPtr("100"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusOK, + }, + }, + }, + }, + { + name: "successful partial response", + input: testInput{ + locals: defaultLocals, + beRes: &s3.HeadObjectOutput{ + ContentRange: utils.GetStringPtr("bytes 10-20/100"), + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": nil, + "x-amz-restore": nil, + "accept-ranges": nil, + "Content-Range": utils.GetStringPtr("bytes 10-20/100"), + "Content-Disposition": nil, + "Content-Encoding": nil, + "Content-Language": nil, + "Cache-Control": nil, + "Expires": nil, + "x-amz-website-redirect-location": nil, + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, + "x-amz-version-id": nil, + "x-amz-mp-parts-count": nil, + "x-amz-object-lock-mode": nil, + "x-amz-object-lock-legal-hold": nil, + "x-amz-storage-class": nil, + "x-amz-checksum-type": nil, + "x-amz-object-lock-retain-until-date": nil, + "Last-Modified": nil, + "x-amz-tagging-count": nil, + "x-amz-server-side-encryption-customer-algorithm": nil, + "x-amz-server-side-encryption-customer-key-MD5": nil, + "Content-Type": nil, + "Content-Length": nil, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusPartialContent, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + HeadObjectFunc: func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + return tt.input.beRes.(*s3.HeadObjectOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.HeadObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + queries: tt.input.queries, + headers: tt.input.headers, + }) + }) + } +} diff --git a/s3err/invalid-argument.go b/s3err/invalid-argument.go index 0d0ac81..27f992f 100644 --- a/s3err/invalid-argument.go +++ b/s3err/invalid-argument.go @@ -26,7 +26,6 @@ type InvalidArgErrorCode int const ( InvalidArgMaxBuckets InvalidArgErrorCode = iota InvalidArgNegativeMaxKeys - InvalidArgStartAfter InvalidArgObjectAttributes InvalidArgPartNumber InvalidArgCompleteMpPartNumber @@ -91,10 +90,6 @@ var invalidArgErrResponses = map[InvalidArgErrorCode]InvalidArgumentError{ Description: "max-keys cannot be negative", ArgumentName: "maxKeys", }, - InvalidArgStartAfter: { - Description: "startAfter only supported in REST.GET.BUCKET with list-type=2", - ArgumentName: "start-after", - }, InvalidArgObjectAttributes: { Description: "Invalid attribute name specified.", ArgumentName: "x-amz-object-attributes", diff --git a/tests/integration/GetObject.go b/tests/integration/GetObject.go new file mode 100644 index 0000000..ab24187 --- /dev/null +++ b/tests/integration/GetObject.go @@ -0,0 +1,1773 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +func GetObject_non_existing_key(s *S3Conf) error { + testName := "GetObject_non_existing_key" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: getPtr("non-existing-key"), + }) + cancel() + var bae *types.NoSuchKey + if !errors.As(err, &bae) { + return err + } + return nil + }) +} + +func GetObject_directory_object_noslash(s *S3Conf) error { + testName := "GetObject_directory_object_noslash" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj/" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + obj = "my-obj" + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + var bae *types.NoSuchKey + if !errors.As(err, &bae) { + return err + } + return nil + }) +} + +func GetObject_with_range(s *S3Conf) error { + testName := "GetObject_with_range" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Match HeadObject_with_range: 100-byte object + obj, objLength := "my-obj", int64(100) + res, err := putObjectWithData(objLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + testGetObjectRange := func(rng, contentRange string, cLength int64, expData []byte, expErr error) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rng, + }) + if err == nil && expErr != nil { + return fmt.Errorf("expected err %v, instead got nil", expErr) + } + if err != nil { + if expErr == nil { + return err + } + parsedErr, ok := expErr.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type provided, expected s3err.APIError") + } + return checkApiErr(err, parsedErr) + } + + if out.ContentLength == nil { + return fmt.Errorf("expected non nil content-length") + } + if *out.ContentLength != cLength { + return fmt.Errorf("expected content-length to be %v, instead got %v", cLength, *out.ContentLength) + } + if getString(out.AcceptRanges) != "bytes" { + return fmt.Errorf("expected accept-ranges to be 'bytes', instead got %v", getString(out.AcceptRanges)) + } + if getString(out.ContentRange) != contentRange { + return fmt.Errorf("expected content-range to be %v, instead got %v", contentRange, getString(out.ContentRange)) + } + + outData, err := io.ReadAll(out.Body) + if err != nil { + return fmt.Errorf("read object data: %w", err) + } + out.Body.Close() + + if !isSameData(outData, expData) { + return fmt.Errorf("incorrect data retrieved") + } + return nil + } + + for _, el := range []struct { + rng string + contentRange string + cLength int64 + expData []byte + expErr error + }{ + // Invalid / ignored ranges (return full object, empty Content-Range) + {"bytes=,", "", objLength, res.data, nil}, + {"bytes= -1", "", objLength, res.data, nil}, + {"bytes=--1", "", objLength, res.data, nil}, + {"bytes=0 -1", "", objLength, res.data, nil}, + {"bytes=0--1", "", objLength, res.data, nil}, + {"bytes=10-5", "", objLength, res.data, nil}, + {"bytes=abc", "", objLength, res.data, nil}, + {"bytes=a-z", "", objLength, res.data, nil}, + {"foo=0-1", "", objLength, res.data, nil}, + {"bytes=abc-xyz", "", objLength, res.data, nil}, + {"bytes=100-x", "", objLength, res.data, nil}, + {"bytes=0-0,1-2", "", objLength, res.data, nil}, + {fmt.Sprintf("bytes=%v-%v", objLength+2, objLength-100), "", objLength, res.data, nil}, + + // Valid numeric with leading zeros + {"bytes=00-01", "bytes 0-1/100", 2, res.data[0:2], nil}, + + // Suffix ranges + {"bytes=-1", "bytes 99-99/100", 1, res.data[99:], nil}, + {"bytes=-2", "bytes 98-99/100", 2, res.data[98:], nil}, + {"bytes=-10", "bytes 90-99/100", 10, res.data[90:], nil}, + {"bytes=-100", "bytes 0-99/100", objLength, res.data, nil}, + {"bytes=-101", "bytes 0-99/100", objLength, res.data, nil}, + + // Standard byte ranges + {"bytes=0-0", "bytes 0-0/100", 1, res.data[0:1], nil}, + {"bytes=0-99", "bytes 0-99/100", objLength, res.data, nil}, + {"bytes=0-100", "bytes 0-99/100", objLength, res.data, nil}, + {"bytes=0-999999", "bytes 0-99/100", objLength, res.data, nil}, + {"bytes=1-99", "bytes 1-99/100", 99, res.data[1:], nil}, + {"bytes=50-99", "bytes 50-99/100", 50, res.data[50:], nil}, + {"bytes=50-", "bytes 50-99/100", 50, res.data[50:], nil}, + {"bytes=0-", "bytes 0-99/100", objLength, res.data, nil}, + {"bytes=99-99", "bytes 99-99/100", 1, res.data[99:], nil}, + + // Unsatisfiable -> error + {"bytes=-0", "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + {"bytes=100-100", "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + {"bytes=100-110", "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + } { + if err := testGetObjectRange(el.rng, el.contentRange, el.cLength, el.expData, el.expErr); err != nil { + return err + } + } + return nil + }) +} + +func GetObject_zero_len_with_range(s *S3Conf) error { + testName := "GetObject_zero_len_with_range" + return getObject_zero_len_with_range_helper(testName, "my-obj", s) +} + +func GetObject_dir_with_range(s *S3Conf) error { + testName := "GetObject_dir_with_range" + return getObject_zero_len_with_range_helper(testName, "my-dir/", s) +} + +func GetObject_invalid_parent(s *S3Conf) error { + testName := "GetObject_invalid_parent" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "not-a-dir" + + _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: getPtr("not-a-dir/bad-obj"), + }) + cancel() + var bae *types.NoSuchKey + if !errors.As(err, &bae) { + return err + } + return nil + }) +} + +func GetObject_checksums(s *S3Conf) error { + testName := "GetObject_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() { + key := fmt.Sprintf("obj-%v", i+1) + out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: checksumAlgo, + }, s3client, withPutObjectChecksumAlgo(checksumAlgo)) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", + key, types.ChecksumTypeFullObject, res.ChecksumType) + } + if got, expected := getString(getGetObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected { + return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got) + } + } + + return nil + }) +} + +func GetObject_dir_object_checksum(s *S3Conf) error { + testName := "GetObject_dir_object_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for i, test := range []struct { + checksumAlgo types.ChecksumAlgorithm + expectedSum string + }{ + {types.ChecksumAlgorithmCrc32, "AAAAAA=="}, + {types.ChecksumAlgorithmCrc32c, "AAAAAA=="}, + {types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="}, + {types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="}, + {types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="}, + {types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="}, + {types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="}, + {types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="}, + {types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="}, + {types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="}, + } { + key := fmt.Sprintf("obj-%v/", i+1) + input := &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: test.checksumAlgo, + } + setPutObjectChecksum(input, test.checksumAlgo, &test.expectedSum) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, input) + cancel() + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v", + i+1, key, types.ChecksumTypeFullObject, res.ChecksumType) + } + + gotSum := getGetObjectChecksum(res, test.checksumAlgo) + + if getString(gotSum) != test.expectedSum { + return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, test.checksumAlgo, test.expectedSum, getString(gotSum)) + } + } + + return nil + }) +} + +func GetObject_large_object(s *S3Conf) error { + testName := "GetObject_large_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + //FIXME: make the object size larger after + // resolving the context deadline exceeding issue + // in the github actions + dataLength, obj := int64(100*1024*1024), "my-obj" + ctype := defaultContentType + + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ContentType: &ctype, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), longTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + if out.ContentLength == nil { + return fmt.Errorf("expected non nil content length") + } + if *out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", + dataLength, out.ContentLength) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != r.csum { + return fmt.Errorf("expected the output data checksum to be %v, instead got %v", + r.csum, outCsum) + } + return nil + }) +} + +func GetObject_conditional_reads(s *S3Conf) error { + testName := "GetObject_conditional_reads" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + key := "my-obj" + obj, err := putObjectWithData(10, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + }, s3client) + if err != nil { + return err + } + + errMod := s3err.GetAPIError(s3err.ErrNotModified) + errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + // sleep one second to get dates before and after + // the object creation + time.Sleep(time.Second * 1) + + before := time.Now().AddDate(0, 0, -3) + after := time.Now() + etag := obj.res.ETag + etagTrimmed := strings.Trim(*etag, `"`) + + for i, test := range []struct { + ifmatch *string + ifnonematch *string + ifmodifiedsince *time.Time + ifunmodifiedsince *time.Time + err error + }{ + // all the cases when preconditions are either empty, true or false + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond}, + + {getPtr("invalid_etag"), etag, &before, &before, errCond}, + {getPtr("invalid_etag"), etag, &before, &after, errCond}, + {getPtr("invalid_etag"), etag, &before, nil, errCond}, + {getPtr("invalid_etag"), etag, &after, &before, errCond}, + {getPtr("invalid_etag"), etag, &after, &after, errCond}, + {getPtr("invalid_etag"), etag, &after, nil, errCond}, + {getPtr("invalid_etag"), etag, nil, &before, errCond}, + {getPtr("invalid_etag"), etag, nil, &after, errCond}, + {getPtr("invalid_etag"), etag, nil, nil, errCond}, + + {getPtr("invalid_etag"), nil, &before, &before, errCond}, + {getPtr("invalid_etag"), nil, &before, &after, errCond}, + {getPtr("invalid_etag"), nil, &before, nil, errCond}, + {getPtr("invalid_etag"), nil, &after, &before, errCond}, + {getPtr("invalid_etag"), nil, &after, &after, errCond}, + {getPtr("invalid_etag"), nil, &after, nil, errCond}, + {getPtr("invalid_etag"), nil, nil, &before, errCond}, + {getPtr("invalid_etag"), nil, nil, &after, errCond}, + {getPtr("invalid_etag"), nil, nil, nil, errCond}, + + {etag, getPtr("invalid_etag"), &before, &before, nil}, + {etag, getPtr("invalid_etag"), &before, &after, nil}, + {etag, getPtr("invalid_etag"), &before, nil, nil}, + {etag, getPtr("invalid_etag"), &after, &before, nil}, + {etag, getPtr("invalid_etag"), &after, &after, nil}, + {etag, getPtr("invalid_etag"), &after, nil, nil}, + {etag, getPtr("invalid_etag"), nil, &before, nil}, + {etag, getPtr("invalid_etag"), nil, &after, nil}, + {etag, getPtr("invalid_etag"), nil, nil, nil}, + + {etag, etag, &before, &before, errMod}, + {etag, etag, &before, &after, errMod}, + {etag, etag, &before, nil, errMod}, + {etag, etag, &after, &before, errMod}, + {etag, etag, &after, &after, errMod}, + {etag, etag, &after, nil, errMod}, + {etag, etag, nil, &before, errMod}, + {etag, etag, nil, &after, errMod}, + {etag, etag, nil, nil, errMod}, + + {etag, nil, &before, &before, nil}, + {etag, nil, &before, &after, nil}, + {etag, nil, &before, nil, nil}, + {etag, nil, &after, &before, errMod}, + {etag, nil, &after, &after, errMod}, + {etag, nil, &after, nil, errMod}, + {etag, nil, nil, &before, nil}, + {etag, nil, nil, &after, nil}, + {etag, nil, nil, nil, nil}, + + {nil, getPtr("invalid_etag"), &before, &before, errCond}, + {nil, getPtr("invalid_etag"), &before, &after, nil}, + {nil, getPtr("invalid_etag"), &before, nil, nil}, + {nil, getPtr("invalid_etag"), &after, &before, errCond}, + {nil, getPtr("invalid_etag"), &after, &after, nil}, + {nil, getPtr("invalid_etag"), &after, nil, nil}, + {nil, getPtr("invalid_etag"), nil, &before, errCond}, + {nil, getPtr("invalid_etag"), nil, &after, nil}, + {nil, getPtr("invalid_etag"), nil, nil, nil}, + + {nil, etag, &before, &before, errCond}, + {nil, etag, &before, &after, errMod}, + {nil, etag, &before, nil, errMod}, + {nil, etag, &after, &before, errCond}, + {nil, etag, &after, &after, errMod}, + {nil, etag, &after, nil, errMod}, + {nil, etag, nil, &before, errCond}, + {nil, etag, nil, &after, errMod}, + {nil, etag, nil, nil, errMod}, + + {nil, nil, &before, &before, errCond}, + {nil, nil, &before, &after, nil}, + {nil, nil, &before, nil, nil}, + {nil, nil, &after, &before, errCond}, + {nil, nil, &after, &after, errMod}, + {nil, nil, &after, nil, errMod}, + {nil, nil, nil, &before, errCond}, + {nil, nil, nil, &after, nil}, + {nil, nil, nil, nil, nil}, + + // if-match, if-non-match without quotes + {&etagTrimmed, getPtr("invalid_etag"), &before, &before, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &before, &after, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &before, nil, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &after, &before, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &after, &after, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &after, nil, nil}, + {&etagTrimmed, getPtr("invalid_etag"), nil, &before, nil}, + {&etagTrimmed, getPtr("invalid_etag"), nil, &after, nil}, + {&etagTrimmed, getPtr("invalid_etag"), nil, nil, nil}, + + {&etagTrimmed, &etagTrimmed, &before, &before, errMod}, + {&etagTrimmed, &etagTrimmed, &before, &after, errMod}, + {&etagTrimmed, &etagTrimmed, &before, nil, errMod}, + {&etagTrimmed, &etagTrimmed, &after, &before, errMod}, + {&etagTrimmed, &etagTrimmed, &after, &after, errMod}, + {&etagTrimmed, &etagTrimmed, &after, nil, errMod}, + {&etagTrimmed, &etagTrimmed, nil, &before, errMod}, + {&etagTrimmed, &etagTrimmed, nil, &after, errMod}, + {&etagTrimmed, &etagTrimmed, nil, nil, errMod}, + + {&etagTrimmed, nil, &before, &before, nil}, + {&etagTrimmed, nil, &before, &after, nil}, + {&etagTrimmed, nil, &before, nil, nil}, + {&etagTrimmed, nil, &after, &before, errMod}, + {&etagTrimmed, nil, &after, &after, errMod}, + {&etagTrimmed, nil, &after, nil, errMod}, + {&etagTrimmed, nil, nil, &before, nil}, + {&etagTrimmed, nil, nil, &after, nil}, + {&etagTrimmed, nil, nil, nil, nil}, + + {nil, &etagTrimmed, &before, &before, errCond}, + {nil, &etagTrimmed, &before, &after, errMod}, + {nil, &etagTrimmed, &before, nil, errMod}, + {nil, &etagTrimmed, &after, &before, errCond}, + {nil, &etagTrimmed, &after, &after, errMod}, + {nil, &etagTrimmed, &after, nil, errMod}, + {nil, &etagTrimmed, nil, &before, errCond}, + {nil, &etagTrimmed, nil, &after, errMod}, + {nil, &etagTrimmed, nil, nil, errMod}, + + // if-match and if-none-match with asterisk + {getPtr("*"), nil, nil, nil, nil}, + {getPtr("*"), nil, &after, nil, errMod}, + {getPtr("*"), getPtr("invalid_etag"), nil, nil, nil}, + {getPtr("*"), etag, nil, nil, errMod}, + {getPtr("*"), getPtr("*"), nil, nil, errMod}, + {getPtr("*"), getPtr("*"), nil, &before, errMod}, + {nil, getPtr("*"), nil, nil, errMod}, + {nil, getPtr("*"), &before, nil, errMod}, + {nil, getPtr("*"), nil, &after, errMod}, + {nil, getPtr("*"), nil, &before, errCond}, + {getPtr("invalid_etag"), getPtr("*"), nil, nil, errCond}, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + IfMatch: test.ifmatch, + IfNoneMatch: test.ifnonematch, + IfModifiedSince: test.ifmodifiedsince, + IfUnmodifiedSince: test.ifunmodifiedsince, + }) + cancel() + if test.err == nil && err != nil { + return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err) + } + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + +func GetObject_success(s *S3Conf) error { + testName := "GetObject_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + ctype, cDisp, cEnc, cLang := defaultContentType, "cont-desp", "json", "eng" + cacheControl, expires := "cache-ctrl", time.Now().Add(time.Hour*2) + redirectLocation := "/get-object-redirect" + meta := map[string]string{ + "foo": "bar", + "baz": "quxx", + } + + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ContentType: &ctype, + ContentDisposition: &cDisp, + ContentEncoding: &cEnc, + ContentLanguage: &cLang, + Expires: &expires, + CacheControl: &cacheControl, + WebsiteRedirectLocation: &redirectLocation, + Metadata: meta, + Tagging: getPtr("key=value&key1=val1"), + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + if *out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", + dataLength, out.ContentLength) + } + if getString(out.ContentType) != defaultContentType { + return fmt.Errorf("expected Content-Type %v, instead got %v", + defaultContentType, getString(out.ContentType)) + } + if getString(out.ContentDisposition) != cDisp { + return fmt.Errorf("expected Content-Disposition %v, instead got %v", + cDisp, getString(out.ContentDisposition)) + } + if getString(out.ContentEncoding) != cEnc { + return fmt.Errorf("expected Content-Encoding %v, instead got %v", + cEnc, getString(out.ContentEncoding)) + } + if getString(out.ContentLanguage) != cLang { + return fmt.Errorf("expected Content-Language %v, instead got %v", + cLang, getString(out.ContentLanguage)) + } + if getString(out.ExpiresString) != expires.UTC().Format(timefmt) { + return fmt.Errorf("expected Expiress %v, instead got %v", + expires.UTC().Format(timefmt), getString(out.ExpiresString)) + } + if getString(out.CacheControl) != cacheControl { + return fmt.Errorf("expected Cache-Control %v, instead got %v", + cacheControl, getString(out.CacheControl)) + } + if getString(out.WebsiteRedirectLocation) != redirectLocation { + return fmt.Errorf("expected WebsiteRedirectLocation %v, instead got %v", + redirectLocation, getString(out.WebsiteRedirectLocation)) + } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", + types.StorageClassStandard, out.StorageClass) + } + if !areMapsSame(out.Metadata, meta) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", + meta, out.Metadata) + } + var tagCount int32 + if out.TagCount != nil { + tagCount = *out.TagCount + } + if tagCount != 2 { + return fmt.Errorf("expected tag count to be 2, instead got %v", tagCount) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != r.csum { + return fmt.Errorf("invalid object data") + } + return nil + }) +} + +const directoryContentType = "application/x-directory" + +func GetObject_directory_success(s *S3Conf) error { + testName := "GetObject_directory_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(0), "my-dir/" + + _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + + if out.ContentLength == nil { + return fmt.Errorf("expected non nil content length") + } + if *out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", + dataLength, out.ContentLength) + } + if getString(out.ContentType) != directoryContentType { + return fmt.Errorf("expected content type %v, instead got %v", + directoryContentType, getString(out.ContentType)) + } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", + types.StorageClassStandard, out.StorageClass) + } + + out.Body.Close() + return nil + }) +} + +func GetObject_by_range_resp_status(s *S3Conf) error { + testName := "GetObject_by_range_resp_status" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + objLength := int64(100) + _, err := putObjectWithData(objLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + checkRangeStatus := func(rng string, expectedStatus int) error { + req, err := createSignedReq( + http.MethodGet, + s.endpoint, + fmt.Sprintf("%v/%v", bucket, obj), + s.awsID, + s.awsSecret, + "s3", + s.awsRegion, + "", + nil, + time.Now(), + map[string]string{"Range": rng}, + ) + if err != nil { + return err + } + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + if resp.StatusCode != expectedStatus { + return fmt.Errorf("range %q: expected status %d, instead got %d", rng, expectedStatus, resp.StatusCode) + } + return nil + } + + for _, tc := range []struct { + rng string + expectedStatus int + }{ + // Invalid/ignored ranges → full object, no Content-Range → 200 + {"bytes=,", http.StatusOK}, + {"bytes= -1", http.StatusOK}, + {"bytes=--1", http.StatusOK}, + {"bytes=0 -1", http.StatusOK}, + {"bytes=0--1", http.StatusOK}, + {"bytes=10-5", http.StatusOK}, + {"bytes=abc", http.StatusOK}, + {"bytes=a-z", http.StatusOK}, + {"foo=0-1", http.StatusOK}, + {"bytes=abc-xyz", http.StatusOK}, + {"bytes=100-x", http.StatusOK}, + {"bytes=0-0,1-2", http.StatusOK}, + {fmt.Sprintf("bytes=%v-%v", objLength+2, objLength-100), http.StatusOK}, + + // Valid ranges → partial content, non-empty Content-Range → 206 + {"bytes=00-01", http.StatusPartialContent}, + {"bytes=-1", http.StatusPartialContent}, + {"bytes=-2", http.StatusPartialContent}, + {"bytes=-10", http.StatusPartialContent}, + {"bytes=-100", http.StatusPartialContent}, + {"bytes=-101", http.StatusPartialContent}, + {"bytes=0-0", http.StatusPartialContent}, + {"bytes=0-99", http.StatusPartialContent}, + {"bytes=0-100", http.StatusPartialContent}, + {"bytes=0-999999", http.StatusPartialContent}, + {"bytes=1-99", http.StatusPartialContent}, + {"bytes=50-99", http.StatusPartialContent}, + {"bytes=50-", http.StatusPartialContent}, + {"bytes=0-", http.StatusPartialContent}, + {"bytes=99-99", http.StatusPartialContent}, + } { + if err := checkRangeStatus(tc.rng, tc.expectedStatus); err != nil { + return err + } + } + + return nil + }) +} + +func GetObject_not_enabled_checksum_mode(s *S3Conf) error { + testName := "GetObject_not_enabled_checksum_mode" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + _, err := putObjectWithData(500, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }, func(o *s3.Options) { + // config sdk to not automatically set the `x-amz-checksum-mode: ENABLED` header + o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset + o.ResponseChecksumValidation = aws.ResponseChecksumValidationUnset + }) + cancel() + if err != nil { + return err + } + + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getGetObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } + } + + return nil + }) +} + +// GetObject_incidental_dir_object verifies that a directory created incidentally +// as a parent during object upload (i.e. never explicitly PUT via S3 with a +// trailing-slash key) is not accessible via GetObject. +func GetObject_incidental_dir_object(s *S3Conf) error { + testName := "GetObject_incidental_dir_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Upload an object under a prefix; this creates the parent directory + // incidentally on posix but the directory was never PUT as an S3 object. + obj := "my-dir/my-obj" + _, err := putObjectWithData(int64(64), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + dir := "my-dir/" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + return checkSdkApiErr(err, "NoSuchKey") + }) +} + +func GetObject_non_existing_dir_object(s *S3Conf) error { + testName := "GetObject_non_existing_dir_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + obj = "my-obj/" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { + return err + } + return nil + }) +} + +func GetObject_overrides_success(s *S3Conf) error { + testName := "GetObject_overrides_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Test data + objKey := "test-object" + objContent := "test content for response overrides" + exp := time.Now() + + // Put an object first + _, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &objKey, + Body: strings.NewReader(objContent), + }) + if err != nil { + return fmt.Errorf("failed to put object: %v", err) + } + + for _, test := range []PublicBucketTestCase{ + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseCacheControl: getPtr("max-age=90"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentDisposition: getPtr("inline"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentEncoding: getPtr("txt"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentLanguage: getPtr("en"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentType: getPtr("application/json"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseExpires: &exp, + }) + return err + }, + ExpectedErr: nil, + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + err := test.Call(ctx) + cancel() + if err == nil && test.ExpectedErr != nil { + return fmt.Errorf("%v: expected err %v, instead got successful response", test.Action, test.ExpectedErr) + } + if err != nil { + if test.ExpectedErr == nil { + return fmt.Errorf("%v: expected no error, instead got %v", test.Action, err) + } + + apiErr, ok := test.ExpectedErr.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type provided in the test, expected s3err.APIError") + } + + if err := checkApiErr(err, apiErr); err != nil { + return err + } + } + } + + return nil + }) +} + +func GetObject_overrides_presign_success(s *S3Conf) error { + testName := "GetObject_overrides_presign_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Test data + objKey := "test-object" + objContent := "test content for response overrides" + + // Put an object first + _, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &objKey, + Body: strings.NewReader(objContent), + }) + if err != nil { + return fmt.Errorf("failed to put object: %v", err) + } + + // Test cases for each response override parameter + testCases := []struct { + name string + queryParam string + expectedHeader string + expectedValue string + }{ + { + name: "response-cache-control", + queryParam: "response-cache-control=no-cache", + expectedHeader: "Cache-Control", + expectedValue: "no-cache", + }, + { + name: "response-content-disposition", + queryParam: "response-content-disposition=attachment%3B%20filename%3D%22test.txt%22", + expectedHeader: "Content-Disposition", + expectedValue: "attachment; filename=\"test.txt\"", + }, + { + name: "response-content-encoding", + queryParam: "response-content-encoding=txt", + expectedHeader: "Content-Encoding", + expectedValue: "txt", + }, + { + name: "response-content-language", + queryParam: "response-content-language=en-US", + expectedHeader: "Content-Language", + expectedValue: "en-US", + }, + { + name: "response-content-type", + queryParam: "response-content-type=text%2Fplain", + expectedHeader: "Content-Type", + expectedValue: "text/plain", + }, + { + name: "response-expires", + queryParam: "response-expires=Thu%2C%2001%20Dec%202024%2016%3A00%3A00%20GMT", + expectedHeader: "Expires", + expectedValue: "Thu, 01 Dec 2024 16:00:00 GMT", + }, + } + + // Test each override parameter individually + for _, tc := range testCases { + // Create a signed request with the response override parameter + req, err := createSignedReq( + http.MethodGet, + s.endpoint, + fmt.Sprintf("%s/%s?%s", bucket, objKey, tc.queryParam), + s.awsID, + s.awsSecret, + "s3", + s.awsRegion, + "", + nil, + time.Now(), + nil, + ) + if err != nil { + return fmt.Errorf("failed to create signed request for %s: %v", tc.name, err) + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to execute request for %s: %v", tc.name, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected status 200 for %s, got %d", tc.name, resp.StatusCode) + } + + // Verify the response override header is set correctly + actualValue := resp.Header.Get(tc.expectedHeader) + if actualValue != tc.expectedValue { + return fmt.Errorf("expected %s header to be %q for %s, got %q", + tc.expectedHeader, tc.expectedValue, tc.name, actualValue) + } + + // Verify content is still correct + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body for %s: %v", tc.name, err) + } + + if string(body) != objContent { + return fmt.Errorf("expected content %q for %s, got %q", objContent, tc.name, string(body)) + } + } + + // Test multiple override parameters together + multiParam := "response-cache-control=max-age%3D3600&response-content-type=application%2Fjson&response-content-disposition=inline" + req, err := createSignedReq( + http.MethodGet, + s.endpoint, + fmt.Sprintf("%s/%s?%s", bucket, objKey, multiParam), + s.awsID, + s.awsSecret, + "s3", + s.awsRegion, + "", + nil, + time.Now(), + nil, + ) + if err != nil { + return fmt.Errorf("failed to create signed request for multiple overrides: %v", err) + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to execute request for multiple overrides: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected status 200 for multiple overrides, got %d", resp.StatusCode) + } + + // Verify all override headers are set correctly + expectedHeaders := map[string]string{ + "Cache-Control": "max-age=3600", + "Content-Type": "application/json", + "Content-Disposition": "inline", + } + + for headerName, expectedValue := range expectedHeaders { + actualValue := resp.Header.Get(headerName) + if actualValue != expectedValue { + return fmt.Errorf("expected %s header to be %q for multiple overrides, got %q", + headerName, expectedValue, actualValue) + } + } + + return nil + }) +} + +func GetObject_overrides_fail_public(s *S3Conf) error { + testName := "GetObject_overrides_fail_public" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + rootClient := s.GetClient() + // Grant public access to the bucket for bucket operations + err := grantPublicBucketPolicy(rootClient, bucket, policyTypeObject) + if err != nil { + return err + } + + // Test data + objKey := "test-object" + objContent := "test content for response overrides" + exp := time.Now() + + // Put an object first + _, err = rootClient.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &objKey, + Body: strings.NewReader(objContent), + }) + if err != nil { + return fmt.Errorf("failed to put object: %v", err) + } + + for _, test := range []PublicBucketTestCase{ + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseCacheControl: getPtr("max-age=90"), + }) + return err + }, + ExpectedErr: s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders), + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentDisposition: getPtr("inline"), + }) + return err + }, + ExpectedErr: s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders), + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentEncoding: getPtr("txt"), + }) + return err + }, + ExpectedErr: s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders), + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentLanguage: getPtr("en"), + }) + return err + }, + ExpectedErr: s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders), + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentType: getPtr("application/json"), + }) + return err + }, + ExpectedErr: s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders), + }, + { + Action: "GetObject", + Call: func(ctx context.Context) error { + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseExpires: &exp, + }) + return err + }, + ExpectedErr: s3err.GetAPIError(s3err.ErrAnonymousResponseHeaders), + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + err := test.Call(ctx) + cancel() + if err == nil && test.ExpectedErr != nil { + return fmt.Errorf("%v: expected err %v, instead got successful response", test.Action, test.ExpectedErr) + } + if err != nil { + if test.ExpectedErr == nil { + return fmt.Errorf("%v: expected no error, instead got %v", test.Action, err) + } + + apiErr, ok := test.ExpectedErr.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type provided in the test, expected s3err.APIError") + } + + if err := checkApiErr(err, apiErr); err != nil { + return err + } + } + } + + return nil + }, withAnonymousClient()) +} + +func GetObject_invalid_part_number(s *S3Conf) error { + testName := "GetObject_invalid_part_number" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: getPtr("obj"), + PartNumber: getPtr(int32(-3)), + }) + + return checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, "-3")) + }) +} + +func GetObject_range_and_part_number(s *S3Conf) error { + testName := "GetObject_range_and_part_number" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + _, err := putObjectWithData(100, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + pn := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=0-9"), + PartNumber: &pn, + }) + cancel() + return checkApiErr(err, s3err.GetAPIError(s3err.ErrRangeAndPartNumber)) + }) +} + +func GetObject_mp_part_number_exceeds_parts_count(s *S3Conf) error { + testName := "GetObject_mp_part_number_exceeds_parts_count" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + const partCount = int64(5) + parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := make([]types.CompletedPart, len(parts)) + for i, p := range parts { + compParts[i] = types.CompletedPart{ + ETag: p.ETag, + PartNumber: p.PartNumber, + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + // partNumber exceeds the number of parts in the completed upload + pn := int32(partCount + 1) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &pn, + }) + cancel() + return checkApiErr(err, s3err.GetInvalidPartNumberRangeErr(int32(partCount), pn)) + }) +} + +func GetObject_mp_part_number_success(s *S3Conf) error { + testName := "GetObject_mp_part_number_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + const partCount = 3 + const partSize = int64(5 * 1024 * 1024) + const totalSize = int64(partCount) * partSize + + // Upload parts manually to capture per-part data for integrity checks + partNumbers := make([]int32, partCount) + partChecksums := make([][32]byte, partCount) + compParts := make([]types.CompletedPart, partCount) + + for i := range partCount { + partNumbers[i] = int32(i + 1) + buf := make([]byte, partSize) + for j := range buf { + buf[j] = byte(i*17 + j%251) + } + partChecksums[i] = sha256.Sum256(buf) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + Body: bytes.NewReader(buf), + PartNumber: &partNumbers[i], + }) + cancel() + if err != nil { + return fmt.Errorf("upload part %d: %w", partNumbers[i], err) + } + compParts[i] = types.CompletedPart{ + ETag: res.ETag, + PartNumber: &partNumbers[i], + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + for i := range partCount { + pn := int32(i + 1) + startByte := int64(i) * partSize + endByte := startByte + partSize - 1 + expectedContentRange := fmt.Sprintf("bytes %d-%d/%d", startByte, endByte, totalSize) + + ctx, cancel := context.WithTimeout(context.Background(), longTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &pn, + }) + if err != nil { + cancel() + return fmt.Errorf("part %d: %w", pn, err) + } + + if res.PartsCount == nil { + cancel() + return fmt.Errorf("part %d: expected non-nil x-amz-mp-parts-count", pn) + } + if *res.PartsCount != int32(partCount) { + cancel() + return fmt.Errorf("part %d: expected PartsCount %d, got %d", pn, partCount, *res.PartsCount) + } + if getString(res.ContentRange) != expectedContentRange { + cancel() + return fmt.Errorf("part %d: expected Content-Range %q, got %q", pn, expectedContentRange, getString(res.ContentRange)) + } + if res.ContentLength == nil || *res.ContentLength != partSize { + cancel() + return fmt.Errorf("part %d: expected Content-Length %d, got %v", pn, partSize, res.ContentLength) + } + if getString(res.AcceptRanges) != "bytes" { + cancel() + return fmt.Errorf("part %d: expected Accept-Ranges 'bytes', got %q", pn, getString(res.AcceptRanges)) + } + + body, readErr := io.ReadAll(res.Body) + cancel() + res.Body.Close() + if readErr != nil { + return fmt.Errorf("part %d: read body: %w", pn, readErr) + } + gotSum := sha256.Sum256(body) + if gotSum != partChecksums[i] { + return fmt.Errorf("part %d: data integrity check failed: body checksum mismatch", pn) + } + } + + return nil + }) +} + +func GetObject_non_mp_part_number_1_success(s *S3Conf) error { + testName := "GetObject_non_mp_part_number_1_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "put-object-part1" + const objSize = int64(1234) + + out, err := putObjectWithData(objSize, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + pn := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), longTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &pn, + }) + if err != nil { + cancel() + return err + } + + if res.ContentLength == nil || *res.ContentLength != objSize { + cancel() + return fmt.Errorf("expected ContentLength %d, got %v", objSize, res.ContentLength) + } + expectedCRange := fmt.Sprintf("bytes 0-%d/%d", objSize-1, objSize) + if getString(res.ContentRange) != expectedCRange { + cancel() + return fmt.Errorf("expected Content-Range to be %s, instead got %s", expectedCRange, getString(res.ContentRange)) + } + if res.PartsCount != nil { + cancel() + return fmt.Errorf("expected nil PartsCount for non-multipart object, got %d", *res.PartsCount) + } + if getString(res.AcceptRanges) != "bytes" { + cancel() + return fmt.Errorf("expected Accept-Ranges 'bytes', got %q", getString(res.AcceptRanges)) + } + + body, readErr := io.ReadAll(res.Body) + cancel() + res.Body.Close() + if readErr != nil { + return fmt.Errorf("read body: %w", readErr) + } + if gotSum := sha256.Sum256(body); gotSum != out.csum { + return fmt.Errorf("data integrity check failed: body checksum mismatch") + } + + return nil + }) +} + +func GetObject_empty_object_part_number_1(s *S3Conf) error { + testName := "GetObject_empty_object_part_number_1" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "empty-obj" + + out, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }, s3client) + if err != nil { + return err + } + + pn := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &pn, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if res.ContentRange != nil { + return fmt.Errorf("expected nil Content-Range for empty object with partNumber=1, got %q", *res.ContentRange) + } + if res.ContentLength == nil || *res.ContentLength != 0 { + return fmt.Errorf("expected ContentLength 0, got %v", res.ContentLength) + } + if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { + return fmt.Errorf("expected sha256 checksum %v, got %v", + getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256)) + } + + return nil + }) +} + +func GetObject_mp_part_number_resp_status(s *S3Conf) error { + testName := "GetObject_mp_part_number_resp_status" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + const partCount = int64(2) + parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := make([]types.CompletedPart, len(parts)) + for i, p := range parts { + compParts[i] = types.CompletedPart{ + ETag: p.ETag, + PartNumber: p.PartNumber, + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + req, err := createSignedReq( + http.MethodGet, + s.endpoint, + fmt.Sprintf("%v/%v?partNumber=1", bucket, obj), + s.awsID, + s.awsSecret, + "s3", + s.awsRegion, + "", + nil, + time.Now(), + nil, + ) + if err != nil { + return err + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusPartialContent { + return fmt.Errorf("expected response status to be %v, instead got %v", + http.StatusPartialContent, resp.StatusCode) + } + + return nil + }) +} + +func GetObject_ranged_with_checksum_mode(s *S3Conf) error { + testName := "GetObject_ranged_with_checksum_mode" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + checkNoChecksums := func(res *s3.GetObjectOutput) error { + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getGetObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } + } + return nil + } + + // Sub-test 1: regular object with Range header and ChecksumMode enabled + regularObj := "regular-obj" + _, err := putObjectWithData(500, &s3.PutObjectInput{ + Bucket: &bucket, + Key: ®ularObj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: ®ularObj, + Range: getPtr("bytes=0-99"), + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return fmt.Errorf("ranged GET on regular object: %w", err) + } + if err := checkNoChecksums(res); err != nil { + return fmt.Errorf("ranged GET on regular object: %w", err) + } + + // Sub-test 2: multipart object with partNumber and ChecksumMode enabled + mpObj := "mp-obj" + mpOut, err := createMp(s3client, bucket, mpObj) + if err != nil { + return err + } + + const partCount = int64(2) + parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, mpObj, *mpOut.UploadId) + if err != nil { + return err + } + + compParts := make([]types.CompletedPart, len(parts)) + for i, p := range parts { + compParts[i] = types.CompletedPart{ + ETag: p.ETag, + PartNumber: p.PartNumber, + } + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &mpObj, + UploadId: mpOut.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + pn := int32(1) + ctx, cancel = context.WithTimeout(context.Background(), longTimeout) + mpRes, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &mpObj, + PartNumber: &pn, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return fmt.Errorf("partNumber GET on MP object: %w", err) + } + if err := checkNoChecksums(mpRes); err != nil { + return fmt.Errorf("partNumber GET on MP object: %w", err) + } + + return nil + }) +} + +func GetObject_empty_version_id(s *S3Conf) error { + return testEmptyVersionId(s, "GetObject_empty_version_id", http.MethodGet, "", nil) +} diff --git a/tests/integration/HeadObject.go b/tests/integration/HeadObject.go new file mode 100644 index 0000000..8a283fe --- /dev/null +++ b/tests/integration/HeadObject.go @@ -0,0 +1,1516 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/versity/versitygw/s3err" +) + +func HeadObject_non_existing_object(s *S3Conf) error { + testName := "HeadObject_non_existing_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil + }) +} + +func HeadObject_invalid_part_number(s *S3Conf) error { + testName := "HeadObject_invalid_part_number" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + partNumber := int32(-3) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + PartNumber: &partNumber, + }) + cancel() + if err := checkSdkApiErr(err, "BadRequest"); err != nil { + return err + } + return nil + }) +} + +// HeadObject_incidental_dir_object verifies that a directory created incidentally +// as a parent during object upload (i.e. never explicitly PUT via S3 with a +// trailing-slash key) is not accessible via HeadObject. +func HeadObject_incidental_dir_object(s *S3Conf) error { + testName := "HeadObject_incidental_dir_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Upload an object under a prefix; this creates the parent directory + // incidentally on posix but the directory was never PUT as an S3 object. + obj := "my-dir/my-obj" + _, err := putObjectWithData(int64(64), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + dir := "my-dir/" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dir, + }) + cancel() + return checkSdkApiErr(err, "NotFound") + }) +} + +func HeadObject_non_existing_dir_object(s *S3Conf) error { + testName := "HeadObject_non_existing_dir_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, dataLen := "my-obj", int64(1234567) + meta := map[string]string{ + "key1": "val1", + "key2": "val2", + } + + _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: meta, + }, s3client) + if err != nil { + return err + } + + obj = "my-obj/" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + return nil + }) +} + +func HeadObject_directory_object_noslash(s *S3Conf) error { + testName := "HeadObject_directory_object_noslash" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj/" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + obj = "my-obj" + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + return nil + }) +} + +const defaultContentType = "binary/octet-stream" + +func HeadObject_not_enabled_checksum_mode(s *S3Conf) error { + testName := "HeadObject_not_enabled_checksum_mode" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + _, err := putObjectWithData(500, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getHeadObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } + } + + return nil + }) +} + +func HeadObject_checksums(s *S3Conf) error { + testName := "HeadObject_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() { + key := fmt.Sprintf("obj-%v", i+1) + out, err := putObjectWithData(int64((i+1)*200), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: checksumAlgo, + }, s3client, withPutObjectChecksumAlgo(checksumAlgo)) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &key, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", key, types.ChecksumTypeFullObject, res.ChecksumType) + } + if got, expected := getString(getHeadObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected { + return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got) + } + } + + return nil + }) +} + +func HeadObject_invalid_parent_dir(s *S3Conf) error { + testName := "HeadObject_invalid_parent_dir" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, dataLen := "not-a-dir", int64(1) + + _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + obj = "not-a-dir/bad-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + + return nil + }) +} + +func HeadObject_with_range(s *S3Conf) error { + testName := "HeadObject_with_range" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, objLength := "my-obj", int64(100) + _, err := putObjectWithData(objLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + testRange := func(rg, contentRange string, cLength int64, expectErr bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rg, + }) + cancel() + if err == nil && expectErr { + return fmt.Errorf("%v: expected err 'RequestedRangeNotSatisfiable' error, instead got nil", rg) + } + if err != nil { + if !expectErr { + return err + } + + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() != "RequestedRangeNotSatisfiable" { + return fmt.Errorf("%v: expected RequestedRangeNotSatisfiable, instead got %v", rg, ae.ErrorCode()) + } + if ae.ErrorMessage() != "Requested Range Not Satisfiable" { + return fmt.Errorf("%v: expected the error message to be 'Requested Range Not Satisfiable', instead got %v", rg, ae.ErrorMessage()) + } + return nil + } + return fmt.Errorf("%v: invalid error got %w", rg, err) + } + + if getString(res.AcceptRanges) != "bytes" { + return fmt.Errorf("%v: expected accept ranges to be 'bytes', instead got %v", rg, getString(res.AcceptRanges)) + } + if res.ContentLength == nil { + return fmt.Errorf("%v: expected non nil content-length", rg) + } + if *res.ContentLength != cLength { + return fmt.Errorf("%v: expected content-length to be %v, instead got %v", rg, cLength, *res.ContentLength) + } + if getString(res.ContentRange) != contentRange { + return fmt.Errorf("%v: expected content-range to be %v, instead got %v", rg, contentRange, getString(res.ContentRange)) + } + return nil + } + + // Reference server expectations for a 100-byte object. + for _, el := range []struct { + objRange string + contentRange string + contentLength int64 + expectedErr bool + }{ + // The following inputs should NOT produce an error and return the full object with empty Content-Range. + {"bytes=,", "", objLength, false}, + {"bytes= -1", "", objLength, false}, + {"bytes=--1", "", objLength, false}, + {"bytes=0 -1", "", objLength, false}, + {"bytes=0--1", "", objLength, false}, + {"bytes=10-5", "", objLength, false}, // start > end treated as invalid + {"bytes=abc", "", objLength, false}, + {"bytes=a-z", "", objLength, false}, + {"foo=0-1", "", objLength, false}, // unsupported unit + {"bytes=00-01", "bytes 0-1/100", 2, false}, // valid numeric despite leading zeros + {"bytes=abc-xyz", "", objLength, false}, // retain legacy invalid pattern + {"bytes=100-x", "", objLength, false}, + {"bytes=0-0,1-2", "", objLength, false}, // multiple ranges unsupported -> ignore + + // Valid suffix ranges (negative forms) + {"bytes=-1", "bytes 99-99/100", 1, false}, + {"bytes=-2", "bytes 98-99/100", 2, false}, + {"bytes=-10", "bytes 90-99/100", 10, false}, + {"bytes=-100", "bytes 0-99/100", objLength, false}, + {"bytes=-101", "bytes 0-99/100", objLength, false}, // larger than object -> entire object + + // Standard byte ranges + {"bytes=0-0", "bytes 0-0/100", 1, false}, + {"bytes=0-99", "bytes 0-99/100", objLength, false}, + {"bytes=0-100", "bytes 0-99/100", objLength, false}, // end past object -> trimmed + {"bytes=0-999999", "bytes 0-99/100", objLength, false}, + {"bytes=1-99", "bytes 1-99/100", objLength - 1, false}, + {"bytes=50-99", "bytes 50-99/100", 50, false}, + {"bytes=50-", "bytes 50-99/100", 50, false}, + {"bytes=0-", "bytes 0-99/100", objLength, false}, + {"bytes=99-99", "bytes 99-99/100", 1, false}, + + // Ranges expected to produce RequestedRangeNotSatisfiable + {"bytes=-0", "", 0, true}, + {"bytes=100-100", "", 0, true}, + {"bytes=100-110", "", 0, true}, + } { + if err := testRange(el.objRange, el.contentRange, el.contentLength, el.expectedErr); err != nil { + return err + } + } + return nil + }) +} + +func HeadObject_zero_len_with_range(s *S3Conf) error { + testName := "HeadObject_zero_len_with_range" + return headObject_zero_len_with_range_helper(testName, "my-obj", s) +} + +func HeadObject_dir_with_range(s *S3Conf) error { + testName := "HeadObject_dir_with_range" + return headObject_zero_len_with_range_helper(testName, "my-dir/", s) +} + +func HeadObject_conditional_reads(s *S3Conf) error { + testName := "HeadObject_conditional_reads" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + key := "my-obj" + obj, err := putObjectWithData(10, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + }, s3client) + if err != nil { + return err + } + + errMod := getPtr("NotModified") + errCond := getPtr("PreconditionFailed") + + // sleep one second to get dates before and after + // the object creation + time.Sleep(time.Second * 1) + + before := time.Now().AddDate(0, 0, -3) + after := time.Now() + etag := obj.res.ETag + etagTrimmed := strings.Trim(*etag, `"`) + + for i, test := range []struct { + ifmatch *string + ifnonematch *string + ifmodifiedsince *time.Time + ifunmodifiedsince *time.Time + err *string + }{ + // all the cases when preconditions are either empty, true or false + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond}, + + {getPtr("invalid_etag"), etag, &before, &before, errCond}, + {getPtr("invalid_etag"), etag, &before, &after, errCond}, + {getPtr("invalid_etag"), etag, &before, nil, errCond}, + {getPtr("invalid_etag"), etag, &after, &before, errCond}, + {getPtr("invalid_etag"), etag, &after, &after, errCond}, + {getPtr("invalid_etag"), etag, &after, nil, errCond}, + {getPtr("invalid_etag"), etag, nil, &before, errCond}, + {getPtr("invalid_etag"), etag, nil, &after, errCond}, + {getPtr("invalid_etag"), etag, nil, nil, errCond}, + + {getPtr("invalid_etag"), nil, &before, &before, errCond}, + {getPtr("invalid_etag"), nil, &before, &after, errCond}, + {getPtr("invalid_etag"), nil, &before, nil, errCond}, + {getPtr("invalid_etag"), nil, &after, &before, errCond}, + {getPtr("invalid_etag"), nil, &after, &after, errCond}, + {getPtr("invalid_etag"), nil, &after, nil, errCond}, + {getPtr("invalid_etag"), nil, nil, &before, errCond}, + {getPtr("invalid_etag"), nil, nil, &after, errCond}, + {getPtr("invalid_etag"), nil, nil, nil, errCond}, + + {etag, getPtr("invalid_etag"), &before, &before, nil}, + {etag, getPtr("invalid_etag"), &before, &after, nil}, + {etag, getPtr("invalid_etag"), &before, nil, nil}, + {etag, getPtr("invalid_etag"), &after, &before, nil}, + {etag, getPtr("invalid_etag"), &after, &after, nil}, + {etag, getPtr("invalid_etag"), &after, nil, nil}, + {etag, getPtr("invalid_etag"), nil, &before, nil}, + {etag, getPtr("invalid_etag"), nil, &after, nil}, + {etag, getPtr("invalid_etag"), nil, nil, nil}, + + {etag, etag, &before, &before, errMod}, + {etag, etag, &before, &after, errMod}, + {etag, etag, &before, nil, errMod}, + {etag, etag, &after, &before, errMod}, + {etag, etag, &after, &after, errMod}, + {etag, etag, &after, nil, errMod}, + {etag, etag, nil, &before, errMod}, + {etag, etag, nil, &after, errMod}, + {etag, etag, nil, nil, errMod}, + + {etag, nil, &before, &before, nil}, + {etag, nil, &before, &after, nil}, + {etag, nil, &before, nil, nil}, + {etag, nil, &after, &before, errMod}, + {etag, nil, &after, &after, errMod}, + {etag, nil, &after, nil, errMod}, + {etag, nil, nil, &before, nil}, + {etag, nil, nil, &after, nil}, + {etag, nil, nil, nil, nil}, + + {nil, getPtr("invalid_etag"), &before, &before, errCond}, + {nil, getPtr("invalid_etag"), &before, &after, nil}, + {nil, getPtr("invalid_etag"), &before, nil, nil}, + {nil, getPtr("invalid_etag"), &after, &before, errCond}, + {nil, getPtr("invalid_etag"), &after, &after, nil}, + {nil, getPtr("invalid_etag"), &after, nil, nil}, + {nil, getPtr("invalid_etag"), nil, &before, errCond}, + {nil, getPtr("invalid_etag"), nil, &after, nil}, + {nil, getPtr("invalid_etag"), nil, nil, nil}, + + {nil, etag, &before, &before, errCond}, + {nil, etag, &before, &after, errMod}, + {nil, etag, &before, nil, errMod}, + {nil, etag, &after, &before, errCond}, + {nil, etag, &after, &after, errMod}, + {nil, etag, &after, nil, errMod}, + {nil, etag, nil, &before, errCond}, + {nil, etag, nil, &after, errMod}, + {nil, etag, nil, nil, errMod}, + + {nil, nil, &before, &before, errCond}, + {nil, nil, &before, &after, nil}, + {nil, nil, &before, nil, nil}, + {nil, nil, &after, &before, errCond}, + {nil, nil, &after, &after, errMod}, + {nil, nil, &after, nil, errMod}, + {nil, nil, nil, &before, errCond}, + {nil, nil, nil, &after, nil}, + {nil, nil, nil, nil, nil}, + + // if-match, if-non-match without quotes + {&etagTrimmed, getPtr("invalid_etag"), &before, &before, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &before, &after, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &before, nil, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &after, &before, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &after, &after, nil}, + {&etagTrimmed, getPtr("invalid_etag"), &after, nil, nil}, + {&etagTrimmed, getPtr("invalid_etag"), nil, &before, nil}, + {&etagTrimmed, getPtr("invalid_etag"), nil, &after, nil}, + {&etagTrimmed, getPtr("invalid_etag"), nil, nil, nil}, + + {&etagTrimmed, &etagTrimmed, &before, &before, errMod}, + {&etagTrimmed, &etagTrimmed, &before, &after, errMod}, + {&etagTrimmed, &etagTrimmed, &before, nil, errMod}, + {&etagTrimmed, &etagTrimmed, &after, &before, errMod}, + {&etagTrimmed, &etagTrimmed, &after, &after, errMod}, + {&etagTrimmed, &etagTrimmed, &after, nil, errMod}, + {&etagTrimmed, &etagTrimmed, nil, &before, errMod}, + {&etagTrimmed, &etagTrimmed, nil, &after, errMod}, + {&etagTrimmed, &etagTrimmed, nil, nil, errMod}, + + {&etagTrimmed, nil, &before, &before, nil}, + {&etagTrimmed, nil, &before, &after, nil}, + {&etagTrimmed, nil, &before, nil, nil}, + {&etagTrimmed, nil, &after, &before, errMod}, + {&etagTrimmed, nil, &after, &after, errMod}, + {&etagTrimmed, nil, &after, nil, errMod}, + {&etagTrimmed, nil, nil, &before, nil}, + {&etagTrimmed, nil, nil, &after, nil}, + {&etagTrimmed, nil, nil, nil, nil}, + + {nil, &etagTrimmed, &before, &before, errCond}, + {nil, &etagTrimmed, &before, &after, errMod}, + {nil, &etagTrimmed, &before, nil, errMod}, + {nil, &etagTrimmed, &after, &before, errCond}, + {nil, &etagTrimmed, &after, &after, errMod}, + {nil, &etagTrimmed, &after, nil, errMod}, + {nil, &etagTrimmed, nil, &before, errCond}, + {nil, &etagTrimmed, nil, &after, errMod}, + {nil, &etagTrimmed, nil, nil, errMod}, + + // if-match and if-none-match with asterisk + {getPtr("*"), nil, nil, nil, nil}, + {getPtr("*"), nil, &after, nil, errMod}, + {getPtr("*"), getPtr("invalid_etag"), nil, nil, nil}, + {getPtr("*"), etag, nil, nil, errMod}, + {getPtr("*"), getPtr("*"), nil, nil, errMod}, + {getPtr("*"), getPtr("*"), nil, &before, errMod}, + {nil, getPtr("*"), nil, nil, errMod}, + {nil, getPtr("*"), &before, nil, errMod}, + {nil, getPtr("*"), nil, &after, errMod}, + {nil, getPtr("*"), nil, &before, errCond}, + {getPtr("invalid_etag"), getPtr("*"), nil, nil, errCond}, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &key, + IfMatch: test.ifmatch, + IfNoneMatch: test.ifnonematch, + IfModifiedSince: test.ifmodifiedsince, + IfUnmodifiedSince: test.ifunmodifiedsince, + }) + cancel() + if test.err == nil && err != nil { + return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err) + } + if test.err != nil { + if err := checkSdkApiErr(err, *test.err); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + +func HeadObject_success(s *S3Conf) error { + testName := "HeadObject_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, dataLen := "my-obj", int64(1234567) + meta := map[string]string{ + "key1": "val1", + "key2": "val2", + } + ctype, cDisp, cEnc, cLang := defaultContentType, "cont-desp", "json", "eng" + cacheControl, expires := "cache-ctrl", time.Now().Add(time.Hour*2) + redirectLocation := "/head-object-redirect" + + _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Metadata: meta, + ContentType: &ctype, + ContentDisposition: &cDisp, + ContentEncoding: &cEnc, + ContentLanguage: &cLang, + CacheControl: &cacheControl, + Expires: &expires, + WebsiteRedirectLocation: &redirectLocation, + Tagging: getPtr("key=value"), + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + + if !areMapsSame(out.Metadata, meta) { + return fmt.Errorf("incorrect object metadata") + } + contentLength := int64(0) + if out.ContentLength != nil { + contentLength = *out.ContentLength + } + if contentLength != dataLen { + return fmt.Errorf("expected data length %v, instead got %v", + dataLen, contentLength) + } + if getString(out.ContentType) != defaultContentType { + return fmt.Errorf("expected Content-Type %v, instead got %v", + defaultContentType, getString(out.ContentType)) + } + if getString(out.ContentDisposition) != cDisp { + return fmt.Errorf("expected Content-Disposition %v, instead got %v", + cDisp, getString(out.ContentDisposition)) + } + if getString(out.ContentEncoding) != cEnc { + return fmt.Errorf("expected Content-Encoding %v, instead got %v", + cEnc, getString(out.ContentEncoding)) + } + if getString(out.ContentLanguage) != cLang { + return fmt.Errorf("expected Content-Language %v, instead got %v", + cLang, getString(out.ContentLanguage)) + } + if getString(out.ExpiresString) != expires.UTC().Format(timefmt) { + return fmt.Errorf("expected Expiress %v, instead got %v", + expires.UTC().Format(timefmt), getString(out.ExpiresString)) + } + if getString(out.CacheControl) != cacheControl { + return fmt.Errorf("expected Cache-Control %v, instead got %v", + cacheControl, getString(out.CacheControl)) + } + if getString(out.WebsiteRedirectLocation) != redirectLocation { + return fmt.Errorf("expected WebsiteRedirectLocation %v, instead got %v", + redirectLocation, getString(out.WebsiteRedirectLocation)) + } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", + types.StorageClassStandard, out.StorageClass) + } + tagCount := int32(0) + if out.TagCount != nil { + tagCount = *out.TagCount + } + + if tagCount != 1 { + return fmt.Errorf("expected the tagcount to be 1, instead got %v", tagCount) + } + + return nil + }) +} + +func HeadObject_overrides_success(s *S3Conf) error { + testName := "HeadObject_overrides_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objKey := "test-object" + exp := time.Now() + + _, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &objKey, + }) + if err != nil { + return fmt.Errorf("failed to put object: %v", err) + } + + for _, test := range []PublicBucketTestCase{ + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseCacheControl: getPtr("max-age=90"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentDisposition: getPtr("inline"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentEncoding: getPtr("txt"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentLanguage: getPtr("en"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentType: getPtr("application/json"), + }) + return err + }, + ExpectedErr: nil, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseExpires: &exp, + }) + return err + }, + ExpectedErr: nil, + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + err := test.Call(ctx) + cancel() + if err == nil && test.ExpectedErr != nil { + return fmt.Errorf("%v: expected err %v, instead got successful response", test.Action, test.ExpectedErr) + } + if err != nil { + if test.ExpectedErr == nil { + return fmt.Errorf("%v: expected no error, instead got %v", test.Action, err) + } + + apiErr, ok := test.ExpectedErr.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type provided in the test, expected s3err.APIError") + } + + if err := checkApiErr(err, apiErr); err != nil { + return err + } + } + } + + return nil + }) +} + +func HeadObject_overrides_presign_success(s *S3Conf) error { + testName := "HeadObject_overrides_presign_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objKey := "test-object" + + _, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &objKey, + }) + if err != nil { + return fmt.Errorf("failed to put object: %v", err) + } + + testCases := []struct { + name string + queryParam string + expectedHeader string + expectedValue string + }{ + { + name: "response-cache-control", + queryParam: "response-cache-control=no-cache", + expectedHeader: "Cache-Control", + expectedValue: "no-cache", + }, + { + name: "response-content-disposition", + queryParam: "response-content-disposition=attachment%3B%20filename%3D%22test.txt%22", + expectedHeader: "Content-Disposition", + expectedValue: "attachment; filename=\"test.txt\"", + }, + { + name: "response-content-encoding", + queryParam: "response-content-encoding=txt", + expectedHeader: "Content-Encoding", + expectedValue: "txt", + }, + { + name: "response-content-language", + queryParam: "response-content-language=en-US", + expectedHeader: "Content-Language", + expectedValue: "en-US", + }, + { + name: "response-content-type", + queryParam: "response-content-type=text%2Fplain", + expectedHeader: "Content-Type", + expectedValue: "text/plain", + }, + { + name: "response-expires", + queryParam: "response-expires=Thu%2C%2001%20Dec%202024%2016%3A00%3A00%20GMT", + expectedHeader: "Expires", + expectedValue: "Thu, 01 Dec 2024 16:00:00 GMT", + }, + } + + for _, tc := range testCases { + req, err := createSignedReq( + http.MethodHead, + s.endpoint, + fmt.Sprintf("%s/%s?%s", bucket, objKey, tc.queryParam), + s.awsID, + s.awsSecret, + "s3", + s.awsRegion, + "", + nil, + time.Now(), + nil, + ) + if err != nil { + return fmt.Errorf("failed to create signed request for %s: %v", tc.name, err) + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to execute request for %s: %v", tc.name, err) + } + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected status 200 for %s, got %d", tc.name, resp.StatusCode) + } + + actualValue := resp.Header.Get(tc.expectedHeader) + if actualValue != tc.expectedValue { + return fmt.Errorf("expected %s header to be %q for %s, got %q", + tc.expectedHeader, tc.expectedValue, tc.name, actualValue) + } + } + + // Test multiple override parameters together + multiParam := "response-cache-control=max-age%3D3600&response-content-type=application%2Fjson&response-content-disposition=inline" + req, err := createSignedReq( + http.MethodHead, + s.endpoint, + fmt.Sprintf("%s/%s?%s", bucket, objKey, multiParam), + s.awsID, + s.awsSecret, + "s3", + s.awsRegion, + "", + nil, + time.Now(), + nil, + ) + if err != nil { + return fmt.Errorf("failed to create signed request for multiple overrides: %v", err) + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to execute request for multiple overrides: %v", err) + } + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected status 200 for multiple overrides, got %d", resp.StatusCode) + } + + expectedHeaders := map[string]string{ + "Cache-Control": "max-age=3600", + "Content-Type": "application/json", + "Content-Disposition": "inline", + } + + for headerName, expectedValue := range expectedHeaders { + actualValue := resp.Header.Get(headerName) + if actualValue != expectedValue { + return fmt.Errorf("expected %s header to be %q for multiple overrides, got %q", + headerName, expectedValue, actualValue) + } + } + + return nil + }) +} + +func HeadObject_range_and_part_number(s *S3Conf) error { + testName := "HeadObject_range_and_part_number" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + _, err := putObjectWithData(100, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + pn := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=0-9"), + PartNumber: &pn, + }) + cancel() + return checkSdkApiErr(err, "BadRequest") + }) +} + +func HeadObject_mp_part_number_exceeds_parts_count(s *S3Conf) error { + testName := "HeadObject_mp_part_number_exceeds_parts_count" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + const partCount = int64(5) + parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := make([]types.CompletedPart, len(parts)) + for i, p := range parts { + compParts[i] = types.CompletedPart{ + ETag: p.ETag, + PartNumber: p.PartNumber, + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + // partNumber exceeds the number of parts in the completed upload + pn := int32(partCount + 1) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &pn, + }) + cancel() + return checkSdkApiErr(err, "RequestedRangeNotSatisfiable") + }) +} + +func HeadObject_mp_part_number_success(s *S3Conf) error { + testName := "HeadObject_mp_part_number_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + const partCount = int64(3) + const partSize = int64(5 * 1024 * 1024) + const totalSize = partCount * partSize + + parts, _, err := uploadParts(s3client, totalSize, partCount, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := make([]types.CompletedPart, len(parts)) + for i, p := range parts { + compParts[i] = types.CompletedPart{ + ETag: p.ETag, + PartNumber: p.PartNumber, + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + for i := range partCount { + pn := int32(i + 1) + startByte := i * partSize + endByte := startByte + partSize - 1 + expectedContentRange := fmt.Sprintf("bytes %d-%d/%d", startByte, endByte, totalSize) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &pn, + }) + cancel() + if err != nil { + return fmt.Errorf("part %d: %w", pn, err) + } + + if res.PartsCount == nil { + return fmt.Errorf("part %d: expected non-nil x-amz-mp-parts-count", pn) + } + if *res.PartsCount != int32(partCount) { + return fmt.Errorf("part %d: expected PartsCount %d, got %d", pn, partCount, *res.PartsCount) + } + if getString(res.ContentRange) != expectedContentRange { + return fmt.Errorf("part %d: expected Content-Range %q, got %q", pn, expectedContentRange, getString(res.ContentRange)) + } + if res.ContentLength == nil || *res.ContentLength != partSize { + return fmt.Errorf("part %d: expected Content-Length %d, got %v", pn, partSize, res.ContentLength) + } + if getString(res.AcceptRanges) != "bytes" { + return fmt.Errorf("part %d: expected Accept-Ranges 'bytes', got %q", pn, getString(res.AcceptRanges)) + } + } + + return nil + }) +} + +func HeadObject_non_mp_part_number_1_success(s *S3Conf) error { + testName := "HeadObject_non_mp_part_number_1_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "put-object-part1" + const objSize = int64(1234) + + _, err := putObjectWithData(objSize, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + pn := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &pn, + }) + cancel() + if err != nil { + return err + } + + if res.ContentLength == nil || *res.ContentLength != objSize { + return fmt.Errorf("expected ContentLength %d, got %v", objSize, res.ContentLength) + } + expectedCRange := fmt.Sprintf("bytes 0-%d/%d", objSize-1, objSize) + if getString(res.ContentRange) != expectedCRange { + return fmt.Errorf("expected Content-Range to be %s, instead got %s", expectedCRange, getString(res.ContentRange)) + } + if res.PartsCount != nil { + return fmt.Errorf("expected nil PartsCount for non-multipart object, got %d", *res.PartsCount) + } + if getString(res.AcceptRanges) != "bytes" { + return fmt.Errorf("expected Accept-Ranges 'bytes', got %q", getString(res.AcceptRanges)) + } + + return nil + }) +} + +func HeadObject_by_range_resp_status(s *S3Conf) error { + testName := "HeadObject_by_range_resp_status" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + objLength := int64(100) + _, err := putObjectWithData(objLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + checkRangeStatus := func(rng string, expectedStatus int) error { + req, err := createSignedReq( + http.MethodHead, + s.endpoint, + fmt.Sprintf("%v/%v", bucket, obj), + s.awsID, + s.awsSecret, + "s3", + s.awsRegion, + "", + nil, + time.Now(), + map[string]string{"Range": rng}, + ) + if err != nil { + return err + } + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + resp.Body.Close() + if resp.StatusCode != expectedStatus { + return fmt.Errorf("range %q: expected status %d, instead got %d", rng, expectedStatus, resp.StatusCode) + } + return nil + } + + for _, tc := range []struct { + rng string + expectedStatus int + }{ + // Invalid/ignored ranges → full object, no Content-Range → 200 + {"bytes=,", http.StatusOK}, + {"bytes= -1", http.StatusOK}, + {"bytes=--1", http.StatusOK}, + {"bytes=0 -1", http.StatusOK}, + {"bytes=0--1", http.StatusOK}, + {"bytes=10-5", http.StatusOK}, + {"bytes=abc", http.StatusOK}, + {"bytes=a-z", http.StatusOK}, + {"foo=0-1", http.StatusOK}, + {"bytes=abc-xyz", http.StatusOK}, + {"bytes=100-x", http.StatusOK}, + {"bytes=0-0,1-2", http.StatusOK}, + {fmt.Sprintf("bytes=%v-%v", objLength+2, objLength-100), http.StatusOK}, + + // Valid ranges → partial content, non-empty Content-Range → 206 + {"bytes=00-01", http.StatusPartialContent}, + {"bytes=-1", http.StatusPartialContent}, + {"bytes=-2", http.StatusPartialContent}, + {"bytes=-10", http.StatusPartialContent}, + {"bytes=-100", http.StatusPartialContent}, + {"bytes=-101", http.StatusPartialContent}, + {"bytes=0-0", http.StatusPartialContent}, + {"bytes=0-99", http.StatusPartialContent}, + {"bytes=0-100", http.StatusPartialContent}, + {"bytes=0-999999", http.StatusPartialContent}, + {"bytes=1-99", http.StatusPartialContent}, + {"bytes=50-99", http.StatusPartialContent}, + {"bytes=50-", http.StatusPartialContent}, + {"bytes=0-", http.StatusPartialContent}, + {"bytes=99-99", http.StatusPartialContent}, + } { + if err := checkRangeStatus(tc.rng, tc.expectedStatus); err != nil { + return err + } + } + + return nil + }) +} + +func HeadObject_empty_object_part_number_1(s *S3Conf) error { + testName := "HeadObject_empty_object_part_number_1" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "empty-obj" + + out, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }, s3client) + if err != nil { + return err + } + + pn := int32(1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &pn, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return err + } + + if res.ContentRange != nil { + return fmt.Errorf("expected nil Content-Range for empty object with partNumber=1, got %q", *res.ContentRange) + } + if res.ContentLength == nil || *res.ContentLength != 0 { + return fmt.Errorf("expected ContentLength 0, got %v", res.ContentLength) + } + if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { + return fmt.Errorf("expected sha256 checksum %v, got %v", + getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256)) + } + + return nil + }) +} + +func HeadObject_mp_part_number_resp_status(s *S3Conf) error { + testName := "HeadObject_mp_part_number_resp_status" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + out, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + const partCount = int64(2) + parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := make([]types.CompletedPart, len(parts)) + for i, p := range parts { + compParts[i] = types.CompletedPart{ + ETag: p.ETag, + PartNumber: p.PartNumber, + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + req, err := createSignedReq( + http.MethodHead, + s.endpoint, + fmt.Sprintf("%v/%v?partNumber=1", bucket, obj), + s.awsID, + s.awsSecret, + "s3", + s.awsRegion, + "", + nil, + time.Now(), + nil, + ) + if err != nil { + return err + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + resp.Body.Close() + + if resp.StatusCode != http.StatusPartialContent { + return fmt.Errorf("expected response status to be %v, instead got %v", + http.StatusPartialContent, resp.StatusCode) + } + + return nil + }) +} + +func HeadObject_ranged_with_checksum_mode(s *S3Conf) error { + testName := "HeadObject_ranged_with_checksum_mode" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + checkNoChecksums := func(res *s3.HeadObjectOutput) error { + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getHeadObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } + } + return nil + } + + // Sub-test 1: regular object with Range header and ChecksumMode enabled + regularObj := "regular-obj" + _, err := putObjectWithData(500, &s3.PutObjectInput{ + Bucket: &bucket, + Key: ®ularObj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: ®ularObj, + Range: getPtr("bytes=0-99"), + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return fmt.Errorf("ranged HEAD on regular object: %w", err) + } + if err := checkNoChecksums(res); err != nil { + return fmt.Errorf("ranged HEAD on regular object: %w", err) + } + + // Sub-test 2: multipart object with partNumber and ChecksumMode enabled + mpObj := "mp-obj" + mpOut, err := createMp(s3client, bucket, mpObj) + if err != nil { + return err + } + + const partCount = int64(2) + parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, mpObj, *mpOut.UploadId) + if err != nil { + return err + } + + compParts := make([]types.CompletedPart, len(parts)) + for i, p := range parts { + compParts[i] = types.CompletedPart{ + ETag: p.ETag, + PartNumber: p.PartNumber, + } + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &mpObj, + UploadId: mpOut.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + pn := int32(1) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + mpRes, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &mpObj, + PartNumber: &pn, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return fmt.Errorf("partNumber HEAD on MP object: %w", err) + } + if err := checkNoChecksums(mpRes); err != nil { + return fmt.Errorf("partNumber HEAD on MP object: %w", err) + } + + return nil + }) +} + +func HeadObject_overrides_fail_public(s *S3Conf) error { + testName := "HeadObject_overrides_fail_public" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + rootClient := s.GetClient() + err := grantPublicBucketPolicy(rootClient, bucket, policyTypeObject) + if err != nil { + return err + } + + objKey := "test-object" + exp := time.Now() + + _, err = rootClient.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &objKey, + }) + if err != nil { + return fmt.Errorf("failed to put object: %v", err) + } + + for _, test := range []PublicBucketTestCase{ + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseCacheControl: getPtr("max-age=90"), + }) + return err + }, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentDisposition: getPtr("inline"), + }) + return err + }, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentEncoding: getPtr("txt"), + }) + return err + }, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentLanguage: getPtr("en"), + }) + return err + }, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseContentType: getPtr("application/json"), + }) + return err + }, + }, + { + Action: "HeadObject", + Call: func(ctx context.Context) error { + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &objKey, + ResponseExpires: &exp, + }) + return err + }, + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + err := test.Call(ctx) + cancel() + if err == nil { + return fmt.Errorf("%v: expected a BadRequest error, instead got successful response", test.Action) + } + + if err := checkSdkApiErr(err, "BadRequest"); err != nil { + return err + } + } + + return nil + }, withAnonymousClient()) +} + +func HeadObject_empty_version_id(s *S3Conf) error { + return testEmptyVersionId(s, "HeadObject_empty_version_id", http.MethodHead, "", nil) +} diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index f1994d8..328683b 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -315,7 +315,6 @@ func TestListObjects(ts *TestState) { ts.Run(ListObjects_delimiter) ts.Run(ListObjects_max_keys_none) ts.Run(ListObjects_marker_not_from_obj_list) - ts.Run(ListObjects_start_after_not_supported) ts.Run(ListObjects_list_all_objs) ts.Run(ListObjects_nested_dir_file_objs) ts.Run(ListObjects_check_owner) @@ -3118,7 +3117,6 @@ func GetIntTests() IntTests { "ListObjects_delimiter": ListObjects_delimiter, "ListObjects_max_keys_none": ListObjects_max_keys_none, "ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list, - "ListObjects_start_after_not_supported": ListObjects_start_after_not_supported, "ListObjects_list_all_objs": ListObjects_list_all_objs, "ListObjects_nested_dir_file_objs": ListObjects_nested_dir_file_objs, "ListObjects_check_owner": ListObjects_check_owner, diff --git a/tests/integration/utils.go b/tests/integration/utils.go new file mode 100644 index 0000000..429f1a8 --- /dev/null +++ b/tests/integration/utils.go @@ -0,0 +1,4307 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/md5" + "crypto/rand" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "hash" + "hash/crc32" + "hash/crc64" + "io" + "math/big" + "math/bits" + rnd "math/rand" + "mime/multipart" + "net/http" + "net/url" + "os/exec" + "slices" + "sort" + "strings" + "sync/atomic" + "time" + "unicode" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/cespare/xxhash/v2" + "github.com/versity/versitygw/iamapi/iamerr" + "github.com/versity/versitygw/s3err" + "github.com/zeebo/xxh3" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" +) + +const emptySHA256Hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +var ( + bcktCount atomic.Uint64 + adminErrorPrefix = "XAdmin" +) + +type user struct { + access string + secret string + role string +} + +func getBucketName() string { + val := bcktCount.Add(1) + return fmt.Sprintf("test-bucket-%v", val) +} + +func getUser(role string) user { + return user{ + access: fmt.Sprintf("test-user-%v", genRandString(16)), + secret: fmt.Sprintf("test-secret-%v", genRandString(16)), + role: role, + } +} + +func setup(s *S3Conf, bucket string, opts ...setupOpt) error { + s3client := s.GetClient() + + cfg := new(setupCfg) + for _, opt := range opts { + opt(cfg) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: &bucket, + ObjectLockEnabledForBucket: &cfg.LockEnabled, + ObjectOwnership: cfg.Ownership, + }) + cancel() + if err != nil { + return err + } + + if cfg.VersioningStatus != "" { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: &bucket, + VersioningConfiguration: &types.VersioningConfiguration{ + Status: cfg.VersioningStatus, + }, + }) + cancel() + if err != nil { + return err + } + } + + return nil +} + +func teardown(s *S3Conf, bucket string) error { + s3client := s.GetClient() + + deleteObject := func(bucket, key, versionId *string) error { + var attempts int + var err error + for attempts < maxRetryAttempts { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: bucket, + Key: key, + VersionId: versionId, + BypassGovernanceRetention: aws.Bool(true), + }) + cancel() + if err == nil { + return nil + } + + attempts++ + time.Sleep(time.Second) + } + + return fmt.Errorf("delete object %s: %w", *key, err) + } + + if s.versioningEnabled { + in := &s3.ListObjectVersionsInput{Bucket: &bucket} + for { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectVersions(ctx, in) + cancel() + if err != nil { + return fmt.Errorf("failed to list objects: %w", err) + } + + for _, item := range out.Versions { + err = deleteObject(&bucket, item.Key, item.VersionId) + if err != nil { + return err + } + } + for _, item := range out.DeleteMarkers { + err = deleteObject(&bucket, item.Key, item.VersionId) + if err != nil { + return err + } + } + + if out.IsTruncated != nil && *out.IsTruncated { + in.KeyMarker = out.KeyMarker + in.VersionIdMarker = out.NextVersionIdMarker + } else { + break + } + } + } else { + for { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return fmt.Errorf("failed to list objects: %w", err) + } + + for _, item := range out.Contents { + err = deleteObject(&bucket, item.Key, nil) + if err != nil { + return err + } + } + + if out.IsTruncated != nil && *out.IsTruncated { + continue + } + break + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + return err +} + +type setupCfg struct { + LockEnabled bool + VersioningStatus types.BucketVersioningStatus + Ownership types.ObjectOwnership + Anonymous bool + SkipTearDown bool +} + +type setupOpt func(*setupCfg) + +func withLock() setupOpt { + return func(s *setupCfg) { s.LockEnabled = true } +} +func withOwnership(o types.ObjectOwnership) setupOpt { + return func(s *setupCfg) { s.Ownership = o } +} +func withVersioning(v types.BucketVersioningStatus) setupOpt { + return func(s *setupCfg) { s.VersioningStatus = v } +} +func withAnonymousClient() setupOpt { + return func(s *setupCfg) { s.Anonymous = true } +} +func withSkipTearDown() setupOpt { + return func(s *setupCfg) { s.SkipTearDown = true } +} + +func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error, opts ...setupOpt) error { + runF(testName) + bucketName := getBucketName() + + cfg := new(setupCfg) + for _, opt := range opts { + opt(cfg) + } + + err := setup(s, bucketName, opts...) + if err != nil { + failF("%v: failed to create a bucket: %v", testName, err) + return fmt.Errorf("%v: failed to create a bucket: %w", testName, err) + } + + var client *s3.Client + if cfg.Anonymous { + client = s.GetAnonymousClient() + } else { + client = s.GetClient() + } + + handlerErr := handler(client, bucketName) + if handlerErr != nil { + failF("%v: %v", testName, handlerErr) + } + + if !cfg.SkipTearDown { + err = teardown(s, bucketName) + if err != nil { + fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err) + if handlerErr == nil { + return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err) + } + } + } + if handlerErr == nil { + passF(testName) + } + + return handlerErr +} + +func actionHandlerNoSetup(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error, _ ...setupOpt) error { + runF(testName) + client := s.GetClient() + bucket := getBucketName() + handlerErr := handler(client, bucket) + if handlerErr != nil { + failF("%v: %v", testName, handlerErr) + } + + if handlerErr == nil { + passF(testName) + } + + return handlerErr +} + +func iamActionHandler(s *S3Conf, testName string, handler func(client *iam.Client) error) error { + runF(testName) + + err := handler(s.GetIAMClient()) + if err != nil { + failF("%v: %v", testName, err) + return fmt.Errorf("%v: %w", testName, err) + } + + passF(testName) + return nil +} + +type authConfig struct { + testName string + path string + method string + overrideSha256 string + body []byte + service string + access string + secret string + region string + date time.Time + headers map[string]string +} + +func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error { + runF(cfg.testName) + access, secret, region := s.awsID, s.awsSecret, s.awsRegion + if cfg.access != "" { + access = cfg.access + } + if cfg.secret != "" { + secret = cfg.secret + } + if cfg.region != "" { + region = cfg.region + } + req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, access, secret, cfg.service, region, cfg.overrideSha256, cfg.body, cfg.date, cfg.headers) + if err != nil { + failF("%v: %v", cfg.testName, err) + return fmt.Errorf("%v: %w", cfg.testName, err) + } + + err = handler(req) + if err != nil { + failF("%v: %v", cfg.testName, err) + return fmt.Errorf("%v: %w", cfg.testName, err) + } + passF(cfg.testName) + return nil +} + +func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient, bucket string) error) error { + runF(testName) + bucket := getBucketName() + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testName, err) + return fmt.Errorf("%v: %w", testName, err) + } + clt := s.GetPresignClient() + + err = handler(clt, bucket) + if err != nil { + failF("%v: %v", testName, err) + return fmt.Errorf("%v: %w", testName, err) + } + + err = teardown(s, bucket) + if err != nil { + failF("%v: %v", testName, err) + return fmt.Errorf("%v: %w", testName, err) + } + + passF(testName) + return nil +} + +func createSignedReq(method, endpoint, path, access, secret, service, region, overrideSha256 string, body []byte, date time.Time, headers map[string]string) (*http.Request, error) { + req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("failed to send the request: %w", err) + } + + signer := v4.NewSigner() + + hexPayload := overrideSha256 + if hexPayload == "" { + hashedPayload := sha256.Sum256(body) + hexPayload = hex.EncodeToString(hashedPayload[:]) + } + + // x-amz-content-sha256 is an S3 signing header. Other services still use + // hexPayload in the canonical request, but should not send or sign this + // header unless the caller explicitly supplies it. + if service == "s3" { + req.Header.Set("X-Amz-Content-Sha256", hexPayload) + } + for key, val := range headers { + req.Header.Add(key, val) + } + + signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, req, hexPayload, service, region, date) + if signErr != nil { + return nil, fmt.Errorf("failed to sign the request: %w", signErr) + } + + return req, nil +} + +// createPresignedReq presigns an S3 request for an operation the SDK's +// PresignClient doesn't cover. Every header passed is signed, and the returned +// request carries back exactly what the signer decided to sign. +func createPresignedReq(method, endpoint, path, access, secret, region string, date time.Time, headers map[string]string) (*http.Request, error) { + req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), nil) + if err != nil { + return nil, fmt.Errorf("failed to create the request: %w", err) + } + // the signer doesn't add the expiration - the SDK's PresignClient puts it in + // the query before signing - and S3 rejects a presigned URL without it + query := req.URL.Query() + query.Set("X-Amz-Expires", "900") + req.URL.RawQuery = query.Encode() + + for key, val := range headers { + req.Header.Set(key, val) + } + + uri, signedHeaders, err := v4.NewSigner().PresignHTTP(req.Context(), + aws.Credentials{AccessKeyID: access, SecretAccessKey: secret}, + req, "UNSIGNED-PAYLOAD", "s3", region, date) + if err != nil { + return nil, fmt.Errorf("failed to presign the request: %w", err) + } + + presigned, err := http.NewRequest(method, uri, nil) + if err != nil { + return nil, fmt.Errorf("failed to create the presigned request: %w", err) + } + presigned.Header = signedHeaders + + return presigned, nil +} + +type APIErrorResponse struct { + XMLName xml.Name `xml:"Error"` + Code string + Message string + ArgumentName string `xml:"ArgumentName,omitempty"` + ArgumentValue string `xml:"ArgumentValue,omitempty"` + Method string `xml:"Method,omitempty"` + Resource string `xml:"Resource,omitempty"` + ResourceType s3err.ResourceType + CalculatedDigest string `xml:"CalculatedDigest,omitempty"` + ExpectedDigest string `xml:"ExpectedDigest,omitempty"` + BucketName string `xml:"BucketName,omitempty"` + ClientComputedContentSHA256 string `xml:"ClientComputedContentSHA256,omitempty"` + S3ComputedContentSHA256 string `xml:"S3ComputedContentSHA256,omitempty"` + ProposedSize int64 `xml:"ProposedSize,omitempty"` + MaxSizeAllowed int64 `xml:"MaxSizeAllowed,omitempty"` + MinSizeAllowed int64 `xml:"MinSizeAllowed,omitempty"` + ServerTime string `xml:"ServerTime,omitempty"` + XAmzExpires int `xml:"X-Amz-Expires,omitempty"` + Expires string `xml:"Expires,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Chunk int `xml:"Chunk,omitempty"` + BadChunkSize int64 `xml:"BadChunkSize,omitempty"` + ContentMD5 string `xml:"Content-MD5,omitempty"` + LocationConstraint string `xml:"LocationConstraint,omitempty"` + UploadId string `xml:"UploadId,omitempty"` + PartNumber int32 `xml:"PartNumber,omitempty"` + ETag string `xml:"ETag,omitempty"` + ActualPartCount int32 `xml:"ActualPartCount,omitempty"` + PartNumberRequested int32 `xml:"PartNumberRequested,omitempty"` + RangeRequested string `xml:"RangeRequested,omitempty"` + ActualObjectSize int64 `xml:"ActualObjectSize,omitempty"` + TagKey string `xml:"TagKey,omitempty"` + TagValue string `xml:"TagValue,omitempty"` + Size int64 `xml:"Size,omitempty"` + Header string `xml:"Header,omitempty"` + AdditionalMessage s3err.NmpAdditionalMessage `xml:"additionalMessage,omitempty"` + Key string `xml:"Key,omitempty"` + VersionId string `xml:"VersionId,omitempty"` + Condition s3err.Condition `xml:"Condition,omitempty"` + RequestTime string `xml:"RequestTime,omitempty"` + MaxAllowedSkewMilliseconds int `xml:"MaxAllowedSkewMilliseconds,omitempty"` + Region string `xml:"Region,omitempty"` + StringToSign string `xml:"StringToSign,omitempty"` + SignatureProvided string `xml:"SignatureProvided,omitempty"` + StringToSignBytes string `xml:"StringToSignBytes,omitempty"` + CanonicalRequest string `xml:"CanonicalRequest,omitempty"` + CanonicalRequestBytes string `xml:"CanonicalRequestBytes,omitempty"` + HeadersNotSigned string `xml:"HeadersNotSigned,omitempty"` + RequestID string `xml:"RequestId,omitempty"` + HostID string `xml:"HostId,omitempty"` +} + +type IAMErrorResponse struct { + XMLName xml.Name `xml:"ErrorResponse"` + Error struct { + Type string + Code string + Message string + } + RequestID string `xml:"RequestId"` +} + +func checkHTTPResponseApiErr(resp *http.Response, expected s3err.S3Error) error { + apiErr := expected.BaseError() + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + resp.Body.Close() + + var errResp APIErrorResponse + err = xml.Unmarshal(body, &errResp) + if err != nil { + return err + } + + if resp.StatusCode != apiErr.HTTPStatusCode { + return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode) + } + + if err := checkRegionMismatchHeader(resp, expected); err != nil { + return err + } + + return compareS3ApiError(expected, &errResp) +} + +// checkRegionMismatchHeader verifies that a signing region mismatch reports the +// gateway region in the x-amz-bucket-region response header. HEAD requests +// carry no response body, so the header is the only channel through which a +// client can discover the region and retry against it. +func checkRegionMismatchHeader(resp *http.Response, expected s3err.S3Error) error { + rerr, ok := expected.(s3err.RegionMismatchError) + if !ok { + return nil + } + + region := rerr.ExpectedRegion() + if region == "" { + return nil + } + + if got := resp.Header.Get("x-amz-bucket-region"); got != region { + return fmt.Errorf("expected x-amz-bucket-region response header to be %v, instead got %v", region, got) + } + + return nil +} + +// testEmptyVersionId verifies an action rejects an empty versionId query +// parameter. The SDK drops empty query parameters, so the request has to be +// signed and sent by hand. subresource is the action's query flag, e.g. +// "tagging", and is empty for the actions addressed by the bare object path. +// body is required for the actions whose routes are guarded by the checksum +// middleware, which rejects empty bodies before the controller runs. +func testEmptyVersionId(s *S3Conf, testName, method, subresource string, body []byte) error { + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + _, err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + var headers map[string]string + if len(body) != 0 { + sum := md5.Sum(body) + headers = map[string]string{ + "Content-Md5": base64.StdEncoding.EncodeToString(sum[:]), + } + } + + query := "versionId=" + if subresource != "" { + query = fmt.Sprintf("%v&%v", subresource, query) + } + + req, err := createSignedReq(method, s.endpoint, + fmt.Sprintf("%v/%v?%v", bucket, obj, query), s.awsID, s.awsSecret, + "s3", s.awsRegion, "", body, time.Now(), headers) + if err != nil { + return err + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + + expected := s3err.GetInvalidArgumentErr(s3err.InvalidArgEmptyVersionId, "") + // HEAD responses carry no body: only the status code is verifiable + if method == http.MethodHead { + resp.Body.Close() + if resp.StatusCode != expected.StatusCode() { + return fmt.Errorf("expected response status code to be %v, instead got %v", + expected.StatusCode(), resp.StatusCode) + } + + return nil + } + + return checkHTTPResponseApiErr(resp, expected) + }) +} + +func checkIAMAuthRequest(s *S3Conf, req *http.Request, expected iamerr.APIError) error { + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + + return checkHTTPResponseIAMErr(resp, expected) +} + +func checkHTTPResponseIAMErr(resp *http.Response, expected iamerr.APIError) error { + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + resp.Body.Close() + + var errResp IAMErrorResponse + err = xml.Unmarshal(body, &errResp) + if err != nil { + return err + } + + if resp.StatusCode != expected.StatusCode() { + return fmt.Errorf("expected response status code to be %v, instead got %v", expected.StatusCode(), resp.StatusCode) + } + if errResp.XMLName.Space != iamerr.Namespace { + return fmt.Errorf("expected IAM error namespace, instead got %q", errResp.XMLName.Space) + } + if errResp.RequestID == "" { + return fmt.Errorf("expected IAM error response request id") + } + + expectedBody := expected.XMLBody(errResp.RequestID) + if string(body) != string(expectedBody) { + return fmt.Errorf("expected IAM error response body to be %q, instead got %q", expectedBody, body) + } + + return nil +} + +// isSuccessStatus returns true for 2xx HTTP status codes. +func isSuccessStatus(statusCode int) bool { + return statusCode >= http.StatusOK && statusCode < http.StatusMultipleChoices +} + +func checkIAMSuccess(resp *http.Response) error { + defer resp.Body.Close() + if !isSuccessStatus(resp.StatusCode) { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("expected response status code to be %v, instead got %v: %s", http.StatusOK, resp.StatusCode, body) + } + + return nil +} + +// websiteGet issues a plain HTTP GET to the dedicated website endpoint. +// The bucket is resolved from the request URL host. No S3 signing is applied. +func websiteGet(s *S3Conf, bucket, path string, headers map[string]string) (*http.Response, error) { + return websiteRequest(s, http.MethodGet, bucket, path, headers) +} + +func websiteHead(s *S3Conf, bucket, path string, headers map[string]string) (*http.Response, error) { + return websiteRequest(s, http.MethodHead, bucket, path, headers) +} + +func websiteOptions(s *S3Conf, bucket, path string, headers map[string]string) (*http.Response, error) { + return websiteRequest(s, http.MethodOptions, bucket, path, headers) +} + +func websiteRequest(s *S3Conf, method, bucket, path string, headers map[string]string) (*http.Response, error) { + reqURL, err := websiteURL(s, bucket, path) + if err != nil { + return nil, err + } + req, err := http.NewRequest(method, reqURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create website request: %w", err) + } + for key, val := range headers { + req.Header.Set(key, val) + } + + return s.httpClient.Do(req) +} + +func websiteHost(s *S3Conf, bucket string) string { + _, domain, port := websiteEndpointParts(s) + + host := fmt.Sprintf("%s.%s", bucket, domain) + if port != "" { + host = fmt.Sprintf("%s:%s", host, port) + } + return host +} + +func websiteURL(s *S3Conf, bucket, path string) (string, error) { + return websiteAbsoluteURL(s, websiteHost(s, bucket), path) +} + +func websiteAbsoluteURL(s *S3Conf, host, path string) (string, error) { + scheme, _, _ := websiteEndpointParts(s) + + rel, err := url.Parse("/" + strings.TrimLeft(path, "/")) + if err != nil { + return "", fmt.Errorf("parse website request path: %w", err) + } + + return (&url.URL{ + Scheme: scheme, + Host: host, + Path: rel.Path, + RawQuery: rel.RawQuery, + }).String(), nil +} + +func websiteEndpointParts(s *S3Conf) (scheme, domain, port string) { + scheme = strings.ToLower(strings.TrimSpace(s.websiteScheme)) + domain = strings.TrimSpace(s.websiteDomain) + port = strings.TrimPrefix(strings.TrimSpace(s.websitePort), ":") + + return scheme, domain, port +} + +func putBucketWebsiteConfig(client *s3.Client, bucket string, config *types.WebsiteConfiguration) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutBucketWebsite(ctx, &s3.PutBucketWebsiteInput{ + Bucket: &bucket, + WebsiteConfiguration: config, + }) + cancel() + return err +} + +func checkWebsiteResponse(resp *http.Response, expectedStatus int, expectedBody []byte) error { + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read website response body: %w", err) + } + + if resp.StatusCode != expectedStatus { + return fmt.Errorf("expected status %v, got %v; body: %s", expectedStatus, resp.StatusCode, body) + } + + return compareBodySHA256(expectedBody, body) +} + +func checkWebsiteErrorResponse(resp *http.Response, expected s3err.S3Error) error { + apiErr := expected.BaseError() + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read website error body: %w", err) + } + + if resp.StatusCode != apiErr.HTTPStatusCode { + return fmt.Errorf("expected status %v, got %v; body: %s", apiErr.HTTPStatusCode, resp.StatusCode, body) + } + if got := resp.Header.Get("x-amz-error-code"); got != apiErr.Code { + return fmt.Errorf("expected x-amz-error-code %q, got %q", apiErr.Code, got) + } + if got := resp.Header.Get("x-amz-error-message"); got != apiErr.Description { + return fmt.Errorf("expected x-amz-error-message %q, got %q", apiErr.Description, got) + } + requestID := resp.Header.Get("x-amz-request-id") + if requestID == "" { + return fmt.Errorf("expected x-amz-request-id header") + } + hostID := resp.Header.Get("x-amz-id-2") + if hostID == "" { + return fmt.Errorf("expected x-amz-id-2 header") + } + if got := resp.Header.Get("Content-Type"); !strings.Contains(got, "text/html") { + return fmt.Errorf("expected html Content-Type, got %q", got) + } + if methodErr, ok := expected.(s3err.MethodNotAllowedError); ok && len(methodErr.AllowedMethods) != 0 { + if got, want := resp.Header.Get("Allow"), methodErr.AllowedMethodsString(); got != want { + return fmt.Errorf("expected Allow header %q, got %q", want, got) + } + } + + expectedBody := expected.HTMLBody(requestID, hostID) + return compareBodySHA256(expectedBody, body) +} + +func compareBodySHA256(expected, actual []byte) error { + expectedSum := sha256.Sum256(expected) + actualSum := sha256.Sum256(actual) + if expectedSum != actualSum { + return fmt.Errorf("body checksum mismatch: expected sha256 %x, got %x; expected body %q, got %q", + expectedSum, actualSum, string(expected), string(actual)) + } + + return nil +} + +func compareS3ApiError(expected s3err.S3Error, received *APIErrorResponse) error { + apiErr := expected.BaseError() + if received == nil { + return fmt.Errorf("expected %w, received nil", apiErr) + } + + if received.Code != apiErr.Code { + return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, received.Code) + } + if received.Message != apiErr.Description { + return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, received.Message) + } + + return compareS3ApiErr(expected, received) +} + +func compareS3ApiErr(expected s3err.S3Error, received *APIErrorResponse) error { + switch err := expected.(type) { + case s3err.APIError: + return nil + case s3err.AccessForbiddenError: + return compareS3ApiErrFields( + compareErrField("Method", err.Method, received.Method), + compareErrField("ResourceType", err.ResourceType, received.ResourceType), + ) + case s3err.BadDigestError: + return compareS3ApiErrFields( + compareErrField("CalculatedDigest", err.CalculatedDigest, received.CalculatedDigest), + compareErrField("ExpectedDigest", err.ExpectedDigest, received.ExpectedDigest), + ) + case s3err.BucketError: + return compareErrField("BucketName", err.BucketName, received.BucketName) + case s3err.ContentSHA256MismatchError: + return compareS3ApiErrFields( + compareErrField("ClientComputedContentSHA256", err.ClientComputedContentSHA256, received.ClientComputedContentSHA256), + compareErrField("S3ComputedContentSHA256", err.S3ComputedContentSHA256, received.S3ComputedContentSHA256), + ) + case s3err.EntityTooLargeError: + return compareS3ApiErrFields( + compareErrField("ProposedSize", err.ProposedSize, received.ProposedSize), + compareErrField("MaxSizeAllowed", err.MaxSizeAllowed, received.MaxSizeAllowed), + ) + case s3err.EntityTooSmallError: + return compareS3ApiErrFields( + compareErrField("ProposedSize", err.ProposedSize, received.ProposedSize), + compareErrField("MinSizeAllowed", err.MinSizeAllowed, received.MinSizeAllowed), + ) + case s3err.ExpiredPresignedURLError: + return compareS3ApiErrFields( + checkErrFieldEmptiness("ServerTime", received.ServerTime, true), + compareErrField("X-Amz-Expires", err.XAmzExpires, received.XAmzExpires), + checkErrFieldEmptiness("Expires", received.Expires, true), + ) + case s3err.InvalidAccessKeyIdError: + return compareErrField("AWSAccessKeyId", err.AWSAccessKeyId, received.AWSAccessKeyId) + case s3err.InvalidArgumentError: + return compareS3ApiErrFields( + compareErrField("ArgumentName", err.ArgumentName, received.ArgumentName), + compareErrField("ArgumentValue", err.ArgumentValue, received.ArgumentValue), + compareErrField("Region", err.Region, received.Region), + ) + case s3err.InvalidChunkSizeError: + return compareS3ApiErrFields( + compareErrField("Chunk", err.Chunk, received.Chunk), + compareErrField("BadChunkSize", err.BadChunkSize, received.BadChunkSize), + ) + case s3err.InvalidDigestError: + return compareErrField("Content-MD5", err.ContentMD5, received.ContentMD5) + case s3err.InvalidLocationConstraintError: + return compareErrField("LocationConstraint", err.LocationConstraint, received.LocationConstraint) + case s3err.InvalidPartError: + return compareS3ApiErrFields( + compareErrField("UploadId", err.UploadId, received.UploadId), + compareErrField("PartNumber", err.PartNumber, received.PartNumber), + compareErrField("ETag", err.ETag, received.ETag), + ) + case s3err.InvalidPartNumberRangeError: + return compareS3ApiErrFields( + compareErrField("ActualPartCount", err.ActualPartCount, received.ActualPartCount), + compareErrField("PartNumberRequested", err.PartNumberRequested, received.PartNumberRequested), + ) + case s3err.InvalidRangeError: + return compareS3ApiErrFields( + compareErrField("RangeRequested", err.RangeRequested, received.RangeRequested), + compareErrField("ActualObjectSize", err.ActualObjectSize, received.ActualObjectSize), + ) + case s3err.InvalidTagError: + return compareS3ApiErrFields( + compareErrField("TagKey", err.TagKey, received.TagKey), + compareErrField("TagValue", err.TagValue, received.TagValue), + ) + case s3err.KeyTooLongError: + return compareS3ApiErrFields( + compareErrField("Size", err.Size, received.Size), + compareErrField("MaxSizeAllowed", err.MaxSizeAllowed, received.MaxSizeAllowed), + ) + case s3err.MetadataTooLargeError: + return compareS3ApiErrFields( + compareErrField("Size", int64(err.Size), received.Size), + compareErrField("MaxSizeAllowed", int64(err.MaxSizeAllowed), received.MaxSizeAllowed), + ) + case s3err.RequestHeaderSectionTooLargeError: + return compareErrField("MaxSizeAllowed", int64(err.MaxSizeAllowed), received.MaxSizeAllowed) + case s3err.MethodNotAllowedError: + return compareS3ApiErrFields( + compareErrField("Method", err.Method, received.Method), + compareErrField("ResourceType", err.ResourceType, received.ResourceType), + ) + case s3err.MalformedAuthError: + return compareErrField("Region", err.Region, received.Region) + case s3err.AuthQueryParamError: + return compareErrField("Region", err.Region, received.Region) + case s3err.HeadersNotSignedError: + return compareErrField("HeadersNotSigned", err.HeadersNotSigned, received.HeadersNotSigned) + case s3err.NoSuchUploadError: + return compareErrField("UploadId", err.UploadId, received.UploadId) + case s3err.NoSuchVersionError: + return compareS3ApiErrFields( + compareErrField("Key", err.Key, received.Key), + compareErrField("VersionId", err.VersionId, received.VersionId), + ) + case s3err.NotImplementedError: + return compareS3ApiErrFields( + compareErrField("Header", err.Header, received.Header), + compareErrField("additionalMessage", err.AdditionalMessage, received.AdditionalMessage), + ) + case s3err.PreconditionFailedError: + return compareErrField("Condition", err.Condition, received.Condition) + case s3err.RequestTimeTooSkewedError: + return compareS3ApiErrFields( + checkErrFieldEmptiness("RequestTime", received.RequestTime, true), + checkErrFieldEmptiness("ServerTime", received.ServerTime, true), + compareErrField("MaxAllowedSkewMilliseconds", err.MaxAllowedSkewMilliseconds, received.MaxAllowedSkewMilliseconds), + ) + case s3err.SignatureDoesNotMatchError: + return compareS3ApiErrFields( + compareErrField("AWSAccessKeyId", err.AWSAccessKeyId, received.AWSAccessKeyId), + checkErrFieldEmptiness("StringToSign", err.StringToSign, true), + compareErrField("SignatureProvided", err.SignatureProvided, received.SignatureProvided), + checkErrFieldEmptiness("StringToSignBytes", err.StringToSignBytes, true), + checkErrFieldEmptiness("CanonicalRequest", err.CanonicalRequest, true), + checkErrFieldEmptiness("CanonicalRequestBytes", err.CanonicalRequestBytes, true), + ) + } + + return nil +} + +func compareS3ApiErrFields(errs ...error) error { + for _, err := range errs { + if err != nil { + return err + } + } + + return nil +} + +func compareErrField[T comparable](field string, expected, received T) error { + if received != expected { + return fmt.Errorf("expected error %s to be %v, instead got %v", field, expected, received) + } + + return nil +} + +func checkErrFieldEmptiness(field, received string, require bool) error { + if require && received == "" { + return fmt.Errorf("expected error %s to be non-empty", field) + } + if !require && received != "" { + return fmt.Errorf("expected error %s to be empty, instead got %s", field, received) + } + + return nil +} + +func checkApiErr(err error, expected s3err.S3Error) error { + apiErr := expected.BaseError() + if err == nil { + return fmt.Errorf("expected %v, instead got nil", apiErr.Code) + } + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() != apiErr.Code { + return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, ae.ErrorCode()) + } + + if ae.ErrorMessage() != apiErr.Description { + return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, ae.ErrorMessage()) + } + + return nil + } + + return fmt.Errorf("expected aws api error, instead got: %w", err) +} + +func checkSdkApiErr(err error, code string) error { + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() != code { + return fmt.Errorf("expected %v, instead got %v", code, ae.ErrorCode()) + } + return nil + } + return err +} + +func checkIAMApiErr(err error, expected iamerr.APIError) error { + if err == nil { + return fmt.Errorf("expected IAM API error, instead got nil") + } + + var apiErr smithy.APIError + if !errors.As(err, &apiErr) { + return fmt.Errorf("expected IAM API error, instead got: %w", err) + } + + expectedErr, ok := expected.(iamerr.Error) + if !ok { + return fmt.Errorf("expected concrete IAM error, got %T", expected) + } + if apiErr.ErrorCode() != expectedErr.Code { + return fmt.Errorf("expected IAM error code to be %q, instead got %q", expectedErr.Code, apiErr.ErrorCode()) + } + if apiErr.ErrorMessage() != expectedErr.Message { + return fmt.Errorf("expected IAM error message to be %q, instead got %q", expectedErr.Message, apiErr.ErrorMessage()) + } + + var responseErr *awshttp.ResponseError + if !errors.As(err, &responseErr) { + return fmt.Errorf("expected IAM HTTP response error, instead got: %w", err) + } + if responseErr.HTTPStatusCode() != expected.StatusCode() { + return fmt.Errorf("expected IAM response status code to be %v, instead got %v", expected.StatusCode(), responseErr.HTTPStatusCode()) + } + if responseErr.ServiceRequestID() == "" { + return fmt.Errorf("expected IAM error response request id") + } + + return nil +} + +type trustPolicyGrammarCase struct { + name string + doc string + wantErr iamerr.APIError // nil means the document must be accepted +} + +// trustPolicyGrammarCases covers the role trust-policy grammar +var trustPolicyGrammarCases = []trustPolicyGrammarCase{ + {"valid AWS principal", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::123456789012:root"},"Action":"sts:AssumeRole"}]}`, nil}, + {"valid without version", `{"Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`, nil}, + {"valid Service principal", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"s3.amazonaws.com"},"Action":"sts:AssumeRole"}]}`, nil}, + {"valid multiple principal type keys together", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*","Service":"sts.amazonaws.com"},"Action":"sts:AssumeRole"}]}`, nil}, + {"valid Federated non-cognito provider (looks suspicious, is valid)", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"bogus.example.com"},"Action":"sts:AssumeRole"}]}`, nil}, + {"valid non-AssumeRole sts action", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:TagSession"}]}`, nil}, + {"valid NotAction with sts prefix", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":{"AWS":"*"},"NotAction":"sts:AssumeRole"}]}`, nil}, + {"valid action array all sts prefixed", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["sts:AssumeRole","sts:TagSession"]}]}`, nil}, + {"valid multiple unique sids", `{"Version":"2012-10-17","Statement":[{"Sid":"A","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"},{"Sid":"B","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`, nil}, + {"cognito federated with condition", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"cognito-identity.amazonaws.com"},"Action":"sts:AssumeRole","Condition":{"StringEquals":{"cognito-identity.amazonaws.com:aud":"us-east-1:abc"}}}]}`, nil}, + {"unrelated condition block ignored (looks suspicious, is valid)", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole","Condition":{"StringEquals":{"aws:SourceAccount":"123456789012"}}}]}`, nil}, + + {"invalid json syntax", `{invalid json`, iamerr.MalformedPolicyDocument("This policy contains invalid Json")}, + {"invalid version", `{"Version":"2020-01-01","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("The policy must contain a valid version string")}, + {"empty statement array", `{"Version":"2012-10-17","Statement":[]}`, iamerr.MalformedPolicyDocument("Could not parse the policy: Statement is empty!")}, + {"missing statement", `{"Version":"2012-10-17"}`, iamerr.MalformedPolicyDocument("Could not parse the policy: Statement is empty!")}, + + {"invalid effect value", `{"Version":"2012-10-17","Statement":[{"Effect":"Maybe","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("Invalid effect: Maybe")}, + {"missing effect field", `{"Version":"2012-10-17","Statement":[{"Principal":{"Service":"s3.amazonaws.com"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("Missing required field Effect")}, + + {"missing principal (opposite of an identity policy, which forbids it)", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("Missing required field Principal")}, + {"empty principal object", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("Missing required field Principal cannot be empty!")}, + {"principal as bare string", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("Principal must be a JSON object.")}, + {"principal as array", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":["a"],"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("Syntax error in policy.")}, + {"principal has invalid key", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"CanonicalUser":"abc"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument(`Invalid principal in policy: "CanonicalUser"`)}, + {"principal key wrong case (looks like it should work, key match is case-sensitive)", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"service":"s3.amazonaws.com"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument(`Invalid principal in policy: "service"`)}, + {"principal has unrecognized service", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"invalid.amazonaws.com"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument(`Invalid principal in policy: "SERVICE":"invalid.amazonaws.com"`)}, + {"principal has ec2 service (valid on real AWS, unsupported by this gateway)", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument(`Invalid principal in policy: "SERVICE":"ec2.amazonaws.com"`)}, + + {"allow with notprincipal", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","NotPrincipal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("Allow with NotPrincipal is not allowed.")}, + {"deny with notprincipal", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","NotPrincipal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("AssumeRole policy must not contain NotPrincipal field.")}, + + {"missing action and notaction", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"}}]}`, iamerr.MalformedPolicyDocument("Missing required field Action")}, + {"bare wildcard action rejected (legal in an identity policy, not here)", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"*"}]}`, iamerr.MalformedPolicyDocument("AssumeRole policy may only specify STS AssumeRole actions.")}, + {"non-sts vendor action rejected", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObject"}]}`, iamerr.MalformedPolicyDocument("AssumeRole policy may only specify STS AssumeRole actions.")}, + {"non-sts notaction rejected even on deny", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":{"AWS":"*"},"NotAction":"s3:GetObject"}]}`, iamerr.MalformedPolicyDocument("AssumeRole policy may only specify STS AssumeRole actions.")}, + {"one non-sts action in an otherwise-valid array rejected", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["sts:AssumeRole","s3:GetObject"]}]}`, iamerr.MalformedPolicyDocument("AssumeRole policy may only specify STS AssumeRole actions.")}, + + {"resource forbidden", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole","Resource":"*"}]}`, iamerr.MalformedPolicyDocument("Has prohibited field Resource")}, + {"notresource forbidden", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole","NotResource":"*"}]}`, iamerr.MalformedPolicyDocument("AssumeRole policy must not contain resources.")}, + + {"duplicate sid across statements", `{"Version":"2012-10-17","Statement":[{"Sid":"Dup","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"},{"Sid":"Dup","Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("The Statement Ids in the policy are not unique")}, + + {"cognito federated without condition (looks valid, Cognito needs a Condition)", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"cognito-identity.amazonaws.com"},"Action":"sts:AssumeRole"}]}`, iamerr.MalformedPolicyDocument("A condition block must be present for the Cognito provider")}, +} + +func putObjects(client *s3.Client, objs []string, bucket string) ([]types.Object, error) { + var contents []types.Object + var size int64 + for _, key := range objs { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := client.PutObject(ctx, &s3.PutObjectInput{ + Key: &key, + Bucket: &bucket, + }) + cancel() + if err != nil { + return nil, err + } + k := key + contents = append(contents, types.Object{ + Key: &k, + ETag: res.ETag, + StorageClass: types.ObjectStorageClassStandard, + Size: &size, + }) + } + + sort.SliceStable(contents, func(i, j int) bool { + return *contents[i].Key < *contents[j].Key + }) + + return contents, nil +} + +// putObjectAndGetETag uploads an object and returns its ETag as the wire +// carries it, quotes included, for the conditional-write tests that have to +// name a real ETag in an If-Match header or a policy condition. +func putObjectAndGetETag(client *s3.Client, bucket, key string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + + res, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + Body: bytes.NewReader([]byte("data")), + }) + if err != nil { + return "", err + } + return getString(res.ETag), nil +} + +// withRequestHeader sets a raw header on an SDK request, for the cases no +// input field reaches — an If-Match on a bucket-level write, say. The +// middleware runs right before signing so the header is signed like any +// other. +func withRequestHeader(key, value string) func(*s3.Options) { + return func(o *s3.Options) { + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + return stack.Finalize.Insert( + middleware.FinalizeMiddlewareFunc("SetRequestHeader", + func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) ( + out middleware.FinalizeOutput, md middleware.Metadata, err error, + ) { + if req, ok := in.Request.(*smithyhttp.Request); ok { + req.Header.Set(key, value) + } + return next.HandleFinalize(ctx, in) + }), + "Signing", + middleware.Before, + ) + }) + } +} + +// headObjectETag re-reads an object's current ETag, which an overwrite may +// have changed. +func headObjectETag(client *s3.Client, bucket, key string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + + res, err := client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &bucket, Key: &key}) + if err != nil { + return "", err + } + return getString(res.ETag), nil +} + +func listObjects(client *s3.Client, bucket, prefix, delimiter string, maxKeys int32) ([]types.Object, []types.CommonPrefix, error) { + var contents []types.Object + var commonPrefixes []types.CommonPrefix + + var continuationToken *string + + for { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + ContinuationToken: continuationToken, + Prefix: &prefix, + Delimiter: &delimiter, + MaxKeys: &maxKeys, + }) + cancel() + if err != nil { + return nil, nil, err + } + contents = append(contents, res.Contents...) + commonPrefixes = append(commonPrefixes, res.CommonPrefixes...) + continuationToken = res.NextContinuationToken + + if !*res.IsTruncated { + break + } + } + + return contents, commonPrefixes, nil +} + +func constructObjectLocation(endpoint, bucket, object string, hostStyle bool) string { + // Normalize endpoint (no trailing slash) + endpoint = strings.TrimRight(endpoint, "/") + + if !hostStyle { + // Path-style: http://endpoint/bucket/object + return fmt.Sprintf("%s/%s/%s", endpoint, bucket, object) + } + + // Host-style: http://bucket.endpoint/object + u, err := url.Parse(endpoint) + if err != nil || u.Host == "" { + // Fallback for raw host:port endpoints (e.g. "127.0.0.1:7070") + return fmt.Sprintf("http://%s.%s/%s", bucket, endpoint, object) + } + + host := u.Host + u.Host = fmt.Sprintf("%s.%s", bucket, host) + + return fmt.Sprintf("%s/%s", u.String(), object) +} + +func hasObjNames(objs []types.Object, names []string) bool { + if len(objs) != len(names) { + return false + } + + for _, obj := range objs { + if slices.Contains(names, *obj.Key) { + continue + } + return false + } + + return true +} + +func hasPrefixName(prefixes []types.CommonPrefix, names []string) bool { + if len(prefixes) != len(names) { + return false + } + + for _, prefix := range prefixes { + if slices.Contains(names, *prefix.Prefix) { + continue + } + return false + } + + return true +} + +type putObjectCfg struct { + checksumAlgorithm types.ChecksumAlgorithm +} + +type putObjectOpt func(*putObjectCfg) + +func withPutObjectChecksumAlgo(algo types.ChecksumAlgorithm) putObjectOpt { + return func(poc *putObjectCfg) { poc.checksumAlgorithm = algo } +} + +type putObjectOutput struct { + csum [32]byte + data []byte + res *s3.PutObjectOutput +} + +func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client, opts ...putObjectOpt) (*putObjectOutput, error) { + cfg := &putObjectCfg{} + for _, opt := range opts { + opt(cfg) + } + + var csum [32]byte + var data []byte + if input.Body == nil { + if lgth != 0 { + data = make([]byte, lgth) + rand.Read(data) + input.Body = bytes.NewReader(data) + } + + // an empty body has a checksum too + if cfg.checksumAlgorithm != "" { + hasher, err := NewHasher(cfg.checksumAlgorithm) + if err != nil { + return nil, err + } + + hasher.Write(data) + sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum) + } + csum = sha256.Sum256(data) + } + + ctx, cancel := context.WithTimeout(context.Background(), longTimeout) + res, err := client.PutObject(ctx, input, func(o *s3.Options) { + // if input.Body is not nil, aws sdk hardcodes Content-Type: application/octet-stream + // this adds a new middleware in the stack to remove the Content-Type header, if + // it isn't explicitly provided as 'PutObject' input. Place the middleware + // right before "Signing" middleware to avoid incorrect request signature calculation + if input.ContentType == nil { + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + return stack.Finalize.Insert( + middleware.FinalizeMiddlewareFunc("UnsetContentType", + func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) ( + out middleware.FinalizeOutput, md middleware.Metadata, err error, + ) { + if req, ok := in.Request.(*smithyhttp.Request); ok { + req.Header.Del("Content-Type") + } + return next.HandleFinalize(ctx, in) + }), + "Signing", + middleware.Before, + ) + }) + } + }) + cancel() + if err != nil { + return nil, err + } + + return &putObjectOutput{ + csum: csum, + data: data, + res: res, + }, nil +} + +type mpCfg struct { + checksumAlgorithm types.ChecksumAlgorithm + checksumType types.ChecksumType + metadata map[string]string +} + +type mpOpt func(*mpCfg) + +func withChecksum(algo types.ChecksumAlgorithm) mpOpt { + return func(mc *mpCfg) { mc.checksumAlgorithm = algo } +} +func withChecksumType(t types.ChecksumType) mpOpt { + return func(mc *mpCfg) { mc.checksumType = t } +} +func withMetadata(m map[string]string) mpOpt { + return func(mc *mpCfg) { mc.metadata = m } +} + +func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) { + cfg := new(mpCfg) + for _, opt := range opts { + opt(cfg) + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &key, + ChecksumAlgorithm: cfg.checksumAlgorithm, + ChecksumType: cfg.checksumType, + Metadata: cfg.metadata, + }) + cancel() + return out, err +} + +func isSameData(a, b []byte) bool { + return bytes.Equal(a, b) +} + +func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { + if len(list1) != len(list2) { + return false + } + for i, item := range list1 { + if *item.Key != *list2[i].Key { + return false + } + if *item.UploadId != *list2[i].UploadId { + return false + } + if item.StorageClass != list2[i].StorageClass { + return false + } + if item.ChecksumAlgorithm != list2[i].ChecksumAlgorithm { + return false + } + if item.ChecksumType != list2[i].ChecksumType { + return false + } + } + + return true +} + +func compareParts(parts1, parts2 []types.Part) bool { + if len(parts1) != len(parts2) { + fmt.Printf("list length are not equal: %v != %v\n", len(parts1), len(parts2)) + return false + } + + for i, prt := range parts1 { + if *prt.PartNumber != *parts2[i].PartNumber { + fmt.Printf("partNumbers are not equal, %v != %v\n", *prt.PartNumber, *parts2[i].PartNumber) + return false + } + if *prt.ETag != *parts2[i].ETag { + fmt.Printf("etags are not equal, %v != %v\n", *prt.ETag, *parts2[i].ETag) + return false + } + if *prt.Size != *parts2[i].Size { + fmt.Printf("sizes are not equal, %v != %v\n", *prt.Size, *parts2[i].Size) + return false + } + if prt.ChecksumCRC32 != nil { + if *prt.ChecksumCRC32 != getString(parts2[i].ChecksumCRC32) { + fmt.Printf("crc32 checksums are not equal, %v != %v\n", *prt.ChecksumCRC32, getString(parts2[i].ChecksumCRC32)) + return false + } + } + if prt.ChecksumCRC32C != nil { + if *prt.ChecksumCRC32C != getString(parts2[i].ChecksumCRC32C) { + fmt.Printf("crc32c checksums are not equal, %v != %v\n", *prt.ChecksumCRC32C, getString(parts2[i].ChecksumCRC32C)) + return false + } + } + if prt.ChecksumSHA1 != nil { + if *prt.ChecksumSHA1 != getString(parts2[i].ChecksumSHA1) { + fmt.Printf("sha1 checksums are not equal, %v != %v\n", *prt.ChecksumSHA1, getString(parts2[i].ChecksumSHA1)) + return false + } + } + if prt.ChecksumSHA256 != nil { + if *prt.ChecksumSHA256 != getString(parts2[i].ChecksumSHA256) { + fmt.Printf("sha256 checksums are not equal, %v != %v\n", *prt.ChecksumSHA256, getString(parts2[i].ChecksumSHA256)) + return false + } + } + if prt.ChecksumCRC64NVME != nil { + if *prt.ChecksumCRC64NVME != getString(parts2[i].ChecksumCRC64NVME) { + fmt.Printf("crc64nvme checksums are not equal, %v != %v\n", *prt.ChecksumCRC64NVME, getString(parts2[i].ChecksumCRC64NVME)) + return false + } + } + if prt.ChecksumSHA512 != nil { + if *prt.ChecksumSHA512 != getString(parts2[i].ChecksumSHA512) { + fmt.Printf("sha512 checksums are not equal, %v != %v\n", *prt.ChecksumSHA512, getString(parts2[i].ChecksumSHA512)) + return false + } + } + if prt.ChecksumMD5 != nil { + if *prt.ChecksumMD5 != getString(parts2[i].ChecksumMD5) { + fmt.Printf("md5 checksums are not equal, %v != %v\n", *prt.ChecksumMD5, getString(parts2[i].ChecksumMD5)) + return false + } + } + if prt.ChecksumXXHASH64 != nil { + if *prt.ChecksumXXHASH64 != getString(parts2[i].ChecksumXXHASH64) { + fmt.Printf("xxhash64 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH64, getString(parts2[i].ChecksumXXHASH64)) + return false + } + } + if prt.ChecksumXXHASH3 != nil { + if *prt.ChecksumXXHASH3 != getString(parts2[i].ChecksumXXHASH3) { + fmt.Printf("xxhash3 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH3, getString(parts2[i].ChecksumXXHASH3)) + return false + } + } + if prt.ChecksumXXHASH128 != nil { + if *prt.ChecksumXXHASH128 != getString(parts2[i].ChecksumXXHASH128) { + fmt.Printf("xxhash128 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH128, getString(parts2[i].ChecksumXXHASH128)) + return false + } + } + } + return true +} + +func areTagsSame(tags1, tags2 []types.Tag) bool { + if len(tags1) != len(tags2) { + return false + } + + for _, tag := range tags1 { + if !containsTag(tag, tags2) { + return false + } + } + return true +} + +func containsTag(tag types.Tag, list []types.Tag) bool { + for _, item := range list { + if *item.Key == *tag.Key && *item.Value == *tag.Value { + return true + } + } + return false +} + +func compareGrants(grts1, grts2 []types.Grant) bool { + if len(grts1) != len(grts2) { + return false + } + + for i, grt := range grts1 { + if grt.Permission != grts2[i].Permission { + return false + } + if *grt.Grantee.ID != *grts2[i].Grantee.ID { + return false + } + if grt.Grantee.Type != grts2[i].Grantee.Type { + return false + } + } + return true +} + +func execCommand(args ...string) ([]byte, error) { + cmd := exec.Command("./versitygw", args...) + + return cmd.CombinedOutput() +} + +func getString(str *string) string { + if str == nil { + return "" + } + return *str +} + +func getPtr[T any](str T) *T { + return &str +} + +// getNonEmptyPtr returns nil for an empty string, rather than a pointer to +// the empty string. +func getNonEmptyPtr(s string) *string { + if s == "" { + return nil + } + return &s +} + +func checksumHeaderName(algo types.ChecksumAlgorithm) string { + return fmt.Sprintf("x-amz-checksum-%s", strings.ToLower(string(algo))) +} + +type checksumFields struct { + CRC32 **string + CRC32C **string + SHA1 **string + SHA256 **string + CRC64NVME **string + SHA512 **string + MD5 **string + XXHASH64 **string + XXHASH3 **string + XXHASH128 **string +} + +func selectChecksum(algo types.ChecksumAlgorithm, fields checksumFields) **string { + switch algo { + case types.ChecksumAlgorithmCrc32: + return fields.CRC32 + case types.ChecksumAlgorithmCrc32c: + return fields.CRC32C + case types.ChecksumAlgorithmSha1: + return fields.SHA1 + case types.ChecksumAlgorithmSha256: + return fields.SHA256 + case types.ChecksumAlgorithmCrc64nvme: + return fields.CRC64NVME + case types.ChecksumAlgorithmSha512: + return fields.SHA512 + case types.ChecksumAlgorithmMd5: + return fields.MD5 + case types.ChecksumAlgorithmXxhash64: + return fields.XXHASH64 + case types.ChecksumAlgorithmXxhash3: + return fields.XXHASH3 + case types.ChecksumAlgorithmXxhash128: + return fields.XXHASH128 + default: + return nil + } +} + +func getChecksum(algo types.ChecksumAlgorithm, fields checksumFields) *string { + if checksum := selectChecksum(algo, fields); checksum != nil { + return *checksum + } + return nil +} + +func setChecksum(algo types.ChecksumAlgorithm, fields checksumFields, checksum *string) { + if selected := selectChecksum(algo, fields); selected != nil { + *selected = checksum + } +} + +func getPartChecksum(part types.Part, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &part.ChecksumCRC32, + CRC32C: &part.ChecksumCRC32C, + SHA1: &part.ChecksumSHA1, + SHA256: &part.ChecksumSHA256, + CRC64NVME: &part.ChecksumCRC64NVME, + SHA512: &part.ChecksumSHA512, + MD5: &part.ChecksumMD5, + XXHASH64: &part.ChecksumXXHASH64, + XXHASH3: &part.ChecksumXXHASH3, + XXHASH128: &part.ChecksumXXHASH128, + }) +} + +func setPartChecksum(part *types.Part, algo types.ChecksumAlgorithm, checksum *string) { + setChecksum(algo, checksumFields{ + CRC32: &part.ChecksumCRC32, + CRC32C: &part.ChecksumCRC32C, + SHA1: &part.ChecksumSHA1, + SHA256: &part.ChecksumSHA256, + CRC64NVME: &part.ChecksumCRC64NVME, + SHA512: &part.ChecksumSHA512, + MD5: &part.ChecksumMD5, + XXHASH64: &part.ChecksumXXHASH64, + XXHASH3: &part.ChecksumXXHASH3, + XXHASH128: &part.ChecksumXXHASH128, + }, checksum) +} + +func getCompletedPartChecksum(part types.CompletedPart, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &part.ChecksumCRC32, + CRC32C: &part.ChecksumCRC32C, + SHA1: &part.ChecksumSHA1, + SHA256: &part.ChecksumSHA256, + CRC64NVME: &part.ChecksumCRC64NVME, + SHA512: &part.ChecksumSHA512, + MD5: &part.ChecksumMD5, + XXHASH64: &part.ChecksumXXHASH64, + XXHASH3: &part.ChecksumXXHASH3, + XXHASH128: &part.ChecksumXXHASH128, + }) +} + +func completedPartFromPart(part types.Part) types.CompletedPart { + return types.CompletedPart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + ChecksumCRC32: part.ChecksumCRC32, + ChecksumCRC32C: part.ChecksumCRC32C, + ChecksumSHA1: part.ChecksumSHA1, + ChecksumSHA256: part.ChecksumSHA256, + ChecksumCRC64NVME: part.ChecksumCRC64NVME, + ChecksumSHA512: part.ChecksumSHA512, + ChecksumMD5: part.ChecksumMD5, + ChecksumXXHASH64: part.ChecksumXXHASH64, + ChecksumXXHASH3: part.ChecksumXXHASH3, + ChecksumXXHASH128: part.ChecksumXXHASH128, + } +} + +func getPutObjectChecksum(out *s3.PutObjectOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func setPutObjectChecksum(in *s3.PutObjectInput, algo types.ChecksumAlgorithm, checksum *string) { + setChecksum(algo, checksumFields{ + CRC32: &in.ChecksumCRC32, + CRC32C: &in.ChecksumCRC32C, + SHA1: &in.ChecksumSHA1, + SHA256: &in.ChecksumSHA256, + CRC64NVME: &in.ChecksumCRC64NVME, + SHA512: &in.ChecksumSHA512, + MD5: &in.ChecksumMD5, + XXHASH64: &in.ChecksumXXHASH64, + XXHASH3: &in.ChecksumXXHASH3, + XXHASH128: &in.ChecksumXXHASH128, + }, checksum) +} + +func getGetObjectChecksum(out *s3.GetObjectOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func getHeadObjectChecksum(out *s3.HeadObjectOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func getObjectAttributesChecksum(out *types.Checksum, algo types.ChecksumAlgorithm) *string { + if out == nil { + return nil + } + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func getUploadPartChecksum(out *s3.UploadPartOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func setUploadPartChecksum(in *s3.UploadPartInput, algo types.ChecksumAlgorithm, checksum *string) { + setChecksum(algo, checksumFields{ + CRC32: &in.ChecksumCRC32, + CRC32C: &in.ChecksumCRC32C, + SHA1: &in.ChecksumSHA1, + SHA256: &in.ChecksumSHA256, + CRC64NVME: &in.ChecksumCRC64NVME, + SHA512: &in.ChecksumSHA512, + MD5: &in.ChecksumMD5, + XXHASH64: &in.ChecksumXXHASH64, + XXHASH3: &in.ChecksumXXHASH3, + XXHASH128: &in.ChecksumXXHASH128, + }, checksum) +} + +func getCompleteMultipartUploadChecksum(out *s3.CompleteMultipartUploadOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func setCompleteMultipartUploadChecksum(in *s3.CompleteMultipartUploadInput, algo types.ChecksumAlgorithm, checksum *string) { + setChecksum(algo, checksumFields{ + CRC32: &in.ChecksumCRC32, + CRC32C: &in.ChecksumCRC32C, + SHA1: &in.ChecksumSHA1, + SHA256: &in.ChecksumSHA256, + CRC64NVME: &in.ChecksumCRC64NVME, + SHA512: &in.ChecksumSHA512, + MD5: &in.ChecksumMD5, + XXHASH64: &in.ChecksumXXHASH64, + XXHASH3: &in.ChecksumXXHASH3, + XXHASH128: &in.ChecksumXXHASH128, + }, checksum) +} + +func getCopyObjectChecksum(result *types.CopyObjectResult, algo types.ChecksumAlgorithm) *string { + if result == nil { + return nil + } + return getChecksum(algo, checksumFields{ + CRC32: &result.ChecksumCRC32, + CRC32C: &result.ChecksumCRC32C, + SHA1: &result.ChecksumSHA1, + SHA256: &result.ChecksumSHA256, + CRC64NVME: &result.ChecksumCRC64NVME, + SHA512: &result.ChecksumSHA512, + MD5: &result.ChecksumMD5, + XXHASH64: &result.ChecksumXXHASH64, + XXHASH3: &result.ChecksumXXHASH3, + XXHASH128: &result.ChecksumXXHASH128, + }) +} + +func getUploadPartCopyChecksum(result *types.CopyPartResult, algo types.ChecksumAlgorithm) *string { + if result == nil { + return nil + } + return getChecksum(algo, checksumFields{ + CRC32: &result.ChecksumCRC32, + CRC32C: &result.ChecksumCRC32C, + SHA1: &result.ChecksumSHA1, + SHA256: &result.ChecksumSHA256, + CRC64NVME: &result.ChecksumCRC64NVME, + SHA512: &result.ChecksumSHA512, + MD5: &result.ChecksumMD5, + XXHASH64: &result.ChecksumXXHASH64, + XXHASH3: &result.ChecksumXXHASH3, + XXHASH128: &result.ChecksumXXHASH128, + }) +} + +// mp1 needs to be the response from the server +// mp2 needs to be the expected values +// The keys from the server are always converted to lowercase +func areMapsSame(mp1, mp2 map[string]string) bool { + if len(mp1) != len(mp2) { + return false + } + for key, val := range mp2 { + if mp1[key] != val { + return false + } + } + return true +} + +func compareBuckets(list1 []types.Bucket, list2 []types.Bucket, ignore ...string) bool { + if len(ignore) > 0 { + ignoreSet := make(map[string]struct{}, len(ignore)) + for _, name := range ignore { + ignoreSet[name] = struct{}{} + } + filtered := make([]types.Bucket, 0, len(list1)) + for _, b := range list1 { + if _, skip := ignoreSet[*b.Name]; !skip { + filtered = append(filtered, b) + } + } + list1 = filtered + } + + if len(list1) != len(list2) { + return false + } + + for i, elem := range list1 { + if *elem.Name != *list2[i].Name { + fmt.Printf("bucket names are not equal: %s != %s\n", *elem.Name, *list2[i].Name) + return false + } + if *elem.BucketRegion != *list2[i].BucketRegion { + fmt.Printf("bucket regions are not equal: %s != %s\n", *elem.BucketRegion, *list2[i].BucketRegion) + return false + } + } + + return true +} + +func compareObjects(list1, list2 []types.Object) bool { + if len(list1) != len(list2) { + fmt.Println("list lengths are not equal") + return false + } + + for i, obj := range list1 { + if *obj.Key != *list2[i].Key { + fmt.Printf("keys are not equal: %q != %q\n", *obj.Key, *list2[i].Key) + return false + } + if *obj.ETag != *list2[i].ETag { + fmt.Printf("etags are not equal: (%q %q) %q != %q\n", + *obj.Key, *list2[i].Key, *obj.ETag, *list2[i].ETag) + return false + } + if *obj.Size != *list2[i].Size { + fmt.Printf("sizes are not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, *obj.Size, *list2[i].Size) + return false + } + if obj.StorageClass != list2[i].StorageClass { + fmt.Printf("storage classes are not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, obj.StorageClass, list2[i].StorageClass) + return false + } + if len(obj.ChecksumAlgorithm) != 0 { + if obj.ChecksumAlgorithm[0] != list2[i].ChecksumAlgorithm[0] { + fmt.Printf("checksum algorithms are not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, obj.ChecksumAlgorithm[0], list2[i].ChecksumAlgorithm[0]) + return false + } + } + if obj.ChecksumType != "" { + if obj.ChecksumType[0] != list2[i].ChecksumType[0] { + fmt.Printf("checksum types are not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, obj.ChecksumType, list2[i].ChecksumType) + return false + } + } + if obj.Owner != nil { + if *obj.Owner.ID != *list2[i].Owner.ID { + fmt.Printf("object owner IDs not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, *obj.Owner.ID, *list2[i].Owner.ID) + } + } + } + + return true +} + +func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool { + if len(list1) != len(list2) { + return false + } + + for i, prefix := range list1 { + if list2[i].Prefix == nil { + fmt.Printf("unexpected nil prefix on index %v", i) + return false + } + if *list2[i].Prefix != prefix { + fmt.Printf("prefix mismatch on index %v: expected %s, got %v", i, prefix, *list2[i].Prefix) + return false + } + } + + return true +} + +func compareDelObjects(list1, list2 []types.DeletedObject) bool { + if len(list1) != len(list2) { + return false + } + + for i, obj := range list1 { + if *obj.Key != *list2[i].Key { + return false + } + + if obj.VersionId != nil { + if list2[i].VersionId == nil { + return false + } + if *obj.VersionId != *list2[i].VersionId { + return false + } + } + if obj.DeleteMarkerVersionId != nil { + if list2[i].DeleteMarkerVersionId == nil { + return false + } + if *obj.DeleteMarkerVersionId != *list2[i].DeleteMarkerVersionId { + return false + } + } + if obj.DeleteMarker != nil { + if list2[i].DeleteMarker == nil { + return false + } + if *obj.DeleteMarker != *list2[i].DeleteMarker { + return false + } + } + } + + return true +} + +func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) { + partSize := size / partCount + + var objHasher hash.Hash + var partHasher hash.Hash + + cfg := new(mpCfg) + for _, opt := range opts { + opt(cfg) + } + + switch cfg.checksumAlgorithm { + case types.ChecksumAlgorithmCrc32: + objHasher = crc32.NewIEEE() + case types.ChecksumAlgorithmCrc32c: + objHasher = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case types.ChecksumAlgorithmMd5: + objHasher = md5.New() + case types.ChecksumAlgorithmSha1: + objHasher = sha1.New() + case types.ChecksumAlgorithmSha256: + objHasher = sha256.New() + case types.ChecksumAlgorithmSha512: + objHasher = sha512.New() + case types.ChecksumAlgorithmCrc64nvme: + objHasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))) + case types.ChecksumAlgorithmXxhash64: + objHasher = xxhash.New() + case types.ChecksumAlgorithmXxhash3: + objHasher = xxh3.New() + case types.ChecksumAlgorithmXxhash128: + objHasher = xxh3.New128() + default: + objHasher = sha256.New() + } + + if cfg.checksumAlgorithm != "" { + partHasher, err = NewHasher(cfg.checksumAlgorithm) + if err != nil { + return nil, "", err + } + } + + for partNumber := int64(1); partNumber <= partCount; partNumber++ { + partStart := (partNumber - 1) * partSize + partEnd := partStart + partSize - 1 + if partEnd > size-1 { + partEnd = size - 1 + } + + partBuffer := make([]byte, partEnd-partStart+1) + rand.Read(partBuffer) + objHasher.Write(partBuffer) + if partHasher != nil { + partHasher.Write(partBuffer) + } + + pn := int32(partNumber) + input := &s3.UploadPartInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + Body: bytes.NewReader(partBuffer), + PartNumber: &pn, + ChecksumAlgorithm: cfg.checksumAlgorithm, + } + if partHasher != nil { + partChecksum := base64.StdEncoding.EncodeToString(partHasher.Sum(nil)) + setUploadPartChecksum(input, cfg.checksumAlgorithm, &partChecksum) + partHasher.Reset() + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.UploadPart(ctx, input) + cancel() + if err != nil { + return parts, "", err + } + + part := types.Part{ + ETag: out.ETag, + PartNumber: &pn, + Size: &partSize, + } + + setPartChecksum(&part, cfg.checksumAlgorithm, getUploadPartChecksum(out, cfg.checksumAlgorithm)) + + parts = append(parts, part) + } + sum := objHasher.Sum(nil) + + if cfg.checksumAlgorithm == "" { + csum = hex.EncodeToString(sum[:]) + } else { + csum = base64.StdEncoding.EncodeToString(sum[:]) + } + + return parts, csum, err +} + +func createUsers(s *S3Conf, users []user) error { + for _, usr := range users { + out, err := execCommand(s.getAdminCommand("-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)...) + if err != nil { + return err + } + if strings.Contains(string(out), adminErrorPrefix) { + return fmt.Errorf("failed to create user account: %s", out) + } + } + return nil +} + +func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error { + for _, bucket := range buckets { + out, err := execCommand(s.getAdminCommand("-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)...) + if err != nil { + return err + } + if strings.Contains(string(out), adminErrorPrefix) { + return fmt.Errorf("failed to change the bucket owner: %s", out) + } + } + + return nil +} + +func listBuckets(s *S3Conf) error { + out, err := execCommand(s.getAdminCommand("-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "list-buckets")...) + if err != nil { + return err + } + if strings.Contains(string(out), adminErrorPrefix) { + return fmt.Errorf("failed to list buckets, %s", out) + } + + return nil +} + +const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +// randCounter is an atomic counter used by genRandString. It is seeded once +// with time.Now().UnixNano() so that values are unique even when many goroutines +// call genRandString concurrently on systems with low-resolution clocks. +var randCounter = atomic.Uint64{} + +func init() { + randCounter.Store(uint64(time.Now().UnixNano())) +} + +func genRandString(length int) string { + source := rnd.NewSource(int64(randCounter.Add(1))) + random := rnd.New(source) + result := make([]byte, length) + for i := range result { + result[i] = charset[random.Intn(len(charset))] + } + return string(result) +} + +const ( + credAccess int = iota + credDate + credRegion + credService + credTerminator +) + +func changeAuthCred(uri, newVal string, index int) (string, error) { + urlParsed, err := url.Parse(uri) + if err != nil { + return "", err + } + + queries := urlParsed.Query() + creds := strings.Split(queries.Get("X-Amz-Credential"), "/") + creds[index] = newVal + queries.Set("X-Amz-Credential", strings.Join(creds, "/")) + urlParsed.RawQuery = queries.Encode() + + return urlParsed.String(), nil +} + +func genPolicyDoc(effect, principal, action, resource string) string { + jsonTemplate := `{ + "Statement": [ + { + "Effect": "%s", + "Principal": %s, + "Action": %s, + "Resource": %s + } + ] + } + ` + + return fmt.Sprintf(jsonTemplate, effect, principal, action, resource) +} + +type policyType string + +const ( + policyTypeBucket policyType = "bucket" + policyTypeObject policyType = "object" + policyTypeFull policyType = "full" +) + +func grantPublicBucketPolicy(client *s3.Client, bucket string, tp policyType) error { + var doc string + + switch tp { + case policyTypeBucket: + doc = genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%s"`, bucket)) + case policyTypeObject: + doc = genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket)) + case policyTypeFull: + template := `{ + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": "arn:aws:s3:::%s" + }, + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": "arn:aws:s3:::%s/*" + } + ] + } + ` + doc = fmt.Sprintf(template, bucket, bucket) + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &doc, + }) + cancel() + return err +} + +func getMalformedPolicyError(msg string) s3err.APIError { + return s3err.APIError{ + Code: "MalformedPolicy", + Description: msg, + HTTPStatusCode: http.StatusBadRequest, + } +} + +func putBucketVersioningStatus(client *s3.Client, bucket string, status types.BucketVersioningStatus) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: &bucket, + VersioningConfiguration: &types.VersioningConfiguration{ + Status: status, + }, + }) + cancel() + + return err +} + +func checkWORMProtection(s *S3Conf, client *s3.Client, bucket, object string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &object, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } + + resp, err := sendPostObject(PostRequestConfig{ + bucket: bucket, + key: object, + s3Conf: s, + fileContent: []byte("overwrite"), + }) + if err != nil { + return err + } + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return fmt.Errorf("POST object overwrite: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &object, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{ + { + Key: &object, + }, + }, + }, + }) + cancel() + if err != nil { + return fmt.Errorf("expected DeleteObjects to succeed with a per-object denial, not fail outright: %w", err) + } + if len(out.Errors) != 1 { + return fmt.Errorf("expected exactly 1 per-object error, got %+v", out.Errors) + } + if err := checkDeleteObjectsErr(out.Errors[0], object, s3err.GetAPIError(s3err.ErrObjectLocked)); err != nil { + return err + } + + return nil +} + +func objStrings(objs []types.Object) []string { + objStrs := make([]string, len(objs)) + for i, obj := range objs { + objStrs[i] = *obj.Key + } + return objStrs +} + +func pfxStrings(pfxs []types.CommonPrefix) []string { + pfxStrs := make([]string, len(pfxs)) + for i, pfx := range pfxs { + pfxStrs[i] = *pfx.Prefix + } + return pfxStrs +} + +type versCfg struct { + checksumAlgorithm types.ChecksumAlgorithm +} + +type versOpt func(*versCfg) + +func withChecksumAlgo(algo types.ChecksumAlgorithm) versOpt { + return func(vc *versCfg) { vc.checksumAlgorithm = algo } +} + +func createObjVersions(client *s3.Client, bucket, object string, count int, opts ...versOpt) ([]types.ObjectVersion, error) { + cfg := new(versCfg) + for _, o := range opts { + o(cfg) + } + + versions := []types.ObjectVersion{} + for i := range count { + rNumber, err := rand.Int(rand.Reader, big.NewInt(100000)) + if err != nil { + return nil, err + } + dataLength := objDataLen(object, rNumber.Int64()) + + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &object, + }, client, withPutObjectChecksumAlgo(cfg.checksumAlgorithm)) + if err != nil { + return nil, err + } + + isLatest := i == count-1 + version := types.ObjectVersion{ + ETag: r.res.ETag, + IsLatest: &isLatest, + Key: &object, + Size: &dataLength, + VersionId: r.res.VersionId, + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: r.res.ChecksumType, + } + + switch { + case r.res.ChecksumCRC32 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc32, + } + case r.res.ChecksumCRC32C != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc32c, + } + case r.res.ChecksumCRC64NVME != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc64nvme, + } + case r.res.ChecksumSHA1 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmSha1, + } + case r.res.ChecksumSHA256 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmSha256, + } + case r.res.ChecksumSHA512 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmSha512, + } + case r.res.ChecksumMD5 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmMd5, + } + case r.res.ChecksumXXHASH64 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmXxhash64, + } + case r.res.ChecksumXXHASH3 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmXxhash3, + } + case r.res.ChecksumXXHASH128 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmXxhash128, + } + } + + versions = append(versions, version) + } + + versions = reverseSlice(versions) + + return versions, nil +} + +// createDeleteMarker deletes object without a version id, making the +// resulting delete marker the current version, and returns its version id. +func createDeleteMarker(client *s3.Client, bucket, object string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &object, + }) + cancel() + if err != nil { + return "", err + } + if out.DeleteMarker == nil || !*out.DeleteMarker { + return "", fmt.Errorf("expected a delete marker to be created for %v", object) + } + if getString(out.VersionId) == "" { + return "", fmt.Errorf("expected non empty delete marker versionId for %v", object) + } + + return *out.VersionId, nil +} + +// objDataLen returns the data length to upload for key: a directory +// object can't hold data +func objDataLen(key string, lgth int64) int64 { + if strings.HasSuffix(key, "/") { + return 0 + } + return lgth +} + +// forEachKey runs fn for each of the keys and prefixes a returned error +// with the key it failed for. Tests use it to run the same scenario for +// a regular object and a directory object. +func forEachKey(keys []string, fn func(key string) error) error { + for _, key := range keys { + if err := fn(key); err != nil { + return fmt.Errorf("%v: %w", key, err) + } + } + return nil +} + +// ReverseSlice reverses a slice of any type +func reverseSlice[T any](s []T) []T { + for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } + return s +} + +func compareVersions(v1, v2 []types.ObjectVersion) bool { + if len(v1) != len(v2) { + return false + } + + for i, version := range v1 { + if version.Key == nil || v2[i].Key == nil { + return false + } + if *version.Key != *v2[i].Key { + return false + } + + if version.VersionId == nil || v2[i].VersionId == nil { + return false + } + if *version.VersionId != *v2[i].VersionId { + return false + } + + if version.IsLatest == nil || v2[i].IsLatest == nil { + return false + } + if *version.IsLatest != *v2[i].IsLatest { + return false + } + + if version.Size == nil || v2[i].Size == nil { + return false + } + if *version.Size != *v2[i].Size { + return false + } + + if version.ETag == nil || v2[i].ETag == nil { + return false + } + if *version.ETag != *v2[i].ETag { + return false + } + + if version.StorageClass != v2[i].StorageClass { + return false + } + if version.ChecksumType != "" { + if version.ChecksumType != v2[i].ChecksumType { + return false + } + } + if len(version.ChecksumAlgorithm) != 0 { + if len(v2[i].ChecksumAlgorithm) == 0 { + return false + } + if version.ChecksumAlgorithm[0] != v2[i].ChecksumAlgorithm[0] { + return false + } + } + } + + return true +} + +func compareDelMarkers(d1, d2 []types.DeleteMarkerEntry) bool { + if len(d1) != len(d2) { + return false + } + + for i, dEntry := range d1 { + if dEntry.Key == nil || d2[i].Key == nil { + return false + } + if *dEntry.Key != *d2[i].Key { + return false + } + + if dEntry.IsLatest == nil || d2[i].IsLatest == nil { + return false + } + if *dEntry.IsLatest != *d2[i].IsLatest { + return false + } + + if dEntry.VersionId == nil || d2[i].VersionId == nil { + return false + } + if *dEntry.VersionId != *d2[i].VersionId { + return false + } + } + + return true +} + +type ObjectMetaProps struct { + ContentLength int64 + ContentType string + ContentEncoding string + ContentDisposition string + ContentLanguage string + CacheControl string + ExpiresString string + WebsiteRedirectLocation string + Metadata map[string]string +} + +func checkObjectMetaProps(client *s3.Client, bucket, object string, o ObjectMetaProps) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &object, + }) + cancel() + if err != nil { + return err + } + + if o.Metadata != nil { + if !areMapsSame(out.Metadata, o.Metadata) { + return fmt.Errorf("expected the object metadata to be %v, instead got %v", o.Metadata, out.Metadata) + } + } + if out.ContentLength == nil { + return fmt.Errorf("expected Content-Length %v, instead got nil", o.ContentLength) + } + if *out.ContentLength != o.ContentLength { + return fmt.Errorf("expected Content-Length %v, instead got %v", o.ContentLength, *out.ContentLength) + } + if o.ContentType != "" && getString(out.ContentType) != o.ContentType { + return fmt.Errorf("expected Content-Type %v, instead got %v", o.ContentType, getString(out.ContentType)) + } + if o.ContentDisposition != "" && getString(out.ContentDisposition) != o.ContentDisposition { + return fmt.Errorf("expected Content-Disposition %v, instead got %v", o.ContentDisposition, getString(out.ContentDisposition)) + } + if o.ContentEncoding != "" && getString(out.ContentEncoding) != o.ContentEncoding { + return fmt.Errorf("expected Content-Encoding %v, instead got %v", o.ContentEncoding, getString(out.ContentEncoding)) + } + if o.ContentLanguage != "" && getString(out.ContentLanguage) != o.ContentLanguage { + return fmt.Errorf("expected Content-Language %v, instead got %v", o.ContentLanguage, getString(out.ContentLanguage)) + } + if o.CacheControl != "" && getString(out.CacheControl) != o.CacheControl { + return fmt.Errorf("expected Cache-Control %v, instead got %v", o.CacheControl, getString(out.CacheControl)) + } + if o.ExpiresString != "" && getString(out.ExpiresString) != o.ExpiresString { + return fmt.Errorf("expected Expires %v, instead got %v", o.ExpiresString, getString(out.ExpiresString)) + } + if o.WebsiteRedirectLocation != "" && getString(out.WebsiteRedirectLocation) != o.WebsiteRedirectLocation { + return fmt.Errorf("expected WebsiteRedirectLocation %v, instead got %v", o.WebsiteRedirectLocation, getString(out.WebsiteRedirectLocation)) + } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) + } + + return nil +} + +func getBoolPtr(b bool) *bool { + return &b +} + +type PublicBucketTestCase struct { + Action string + Call func(ctx context.Context) error + ExpectedErr error +} + +// randomizeCase randomizes the provided string latters case +func randomizeCase(s string) string { + var b strings.Builder + + for _, ch := range s { + if rnd.Intn(2) == 0 { + b.WriteRune(unicode.ToLower(ch)) + } else { + b.WriteRune(unicode.ToUpper(ch)) + } + } + + return b.String() +} + +func headObject_zero_len_with_range_helper(testName, obj string, s *S3Conf) error { + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objLength := int64(0) + _, err := putObjectWithData(objLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + testRange := func(rg, contentRange string, cLength int64, expectErr bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rg, + }) + cancel() + if err == nil && expectErr { + return fmt.Errorf("%v: expected err 'RequestedRangeNotSatisfiable' error, instead got nil", rg) + } + if err != nil { + if !expectErr { + return err + } + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() != "RequestedRangeNotSatisfiable" { + return fmt.Errorf("%v: expected RequestedRangeNotSatisfiable, instead got %v", rg, ae.ErrorCode()) + } + if ae.ErrorMessage() != "Requested Range Not Satisfiable" { + return fmt.Errorf("%v: expected the error message to be 'Requested Range Not Satisfiable', instead got %v", rg, ae.ErrorMessage()) + } + return nil + } + return fmt.Errorf("%v: invalid error got %w", rg, err) + } + + if getString(res.AcceptRanges) != "bytes" { + return fmt.Errorf("%v: expected accept ranges to be 'bytes', instead got %v", rg, getString(res.AcceptRanges)) + } + if res.ContentLength == nil { + return fmt.Errorf("%v: expected non nil content-length", rg) + } + if *res.ContentLength != cLength { + return fmt.Errorf("%v: expected content-length to be %v, instead got %v", rg, cLength, *res.ContentLength) + } + if getString(res.ContentRange) != contentRange { + return fmt.Errorf("%v: expected content-range to be %v, instead got %v", rg, contentRange, getString(res.ContentRange)) + } + return nil + } + + // Reference server expectations for a 0-byte object. + for _, el := range []struct { + objRange string + contentRange string + contentLength int64 + expectedErr bool + }{ + {"bytes=abc", "", objLength, false}, + {"bytes=a-z", "", objLength, false}, + {"bytes=,", "", objLength, false}, + {"bytes=0-0,1-2", "", objLength, false}, + {"foo=0-1", "", objLength, false}, + {"bytes=--1", "", objLength, false}, + {"bytes=0--1", "", objLength, false}, + {"bytes= -1", "", objLength, false}, + {"bytes=0 -1", "", objLength, false}, + {"bytes=-1", "", objLength, false}, // reference server returns no error, empty Content-Range + {"bytes=00-01", "", objLength, true}, // RequestedRangeNotSatisfiable + {"bytes=-0", "", 0, true}, + {"bytes=0-0", "", 0, true}, + {"bytes=0-", "", 0, true}, + } { + if err := testRange(el.objRange, el.contentRange, el.contentLength, el.expectedErr); err != nil { + return err + } + } + return nil + }) +} + +func getObject_zero_len_with_range_helper(testName, obj string, s *S3Conf) error { + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objLength := int64(0) + res, err := putObjectWithData(objLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + testGetObjectRange := func(rng, contentRange string, cLength int64, expData []byte, expErr error) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rng, + }) + if err == nil && expErr != nil { + return fmt.Errorf("%v: expected err %v, instead got nil", rng, expErr) + } + if err != nil { + if expErr == nil { + return err + } + parsedErr, ok := expErr.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type provided, expected s3err.APIError") + } + return checkApiErr(err, parsedErr) + } + + if out.ContentLength == nil { + return fmt.Errorf("%v: expected non nil content-length", rng) + } + if *out.ContentLength != cLength { + return fmt.Errorf("%v: expected content-length to be %v, instead got %v", rng, cLength, *out.ContentLength) + } + if getString(out.AcceptRanges) != "bytes" { + return fmt.Errorf("%v: expected accept-ranges to be 'bytes', instead got %v", rng, getString(out.AcceptRanges)) + } + if getString(out.ContentRange) != contentRange { + return fmt.Errorf("%v: expected content-range to be %v, instead got %v", rng, contentRange, getString(out.ContentRange)) + } + + data, err := io.ReadAll(out.Body) + if err != nil { + return fmt.Errorf("%v: read object data: %w", rng, err) + } + out.Body.Close() + if !isSameData(data, expData) { + return fmt.Errorf("%v: incorrect data retrieved", rng) + } + return nil + } + + for _, el := range []struct { + rng string + contentRange string + cLength int64 + expData []byte + expErr error + }{ + {"bytes=abc", "", objLength, res.data, nil}, + {"bytes=a-z", "", objLength, res.data, nil}, + {"bytes=,", "", objLength, res.data, nil}, + {"bytes=0-0,1-2", "", objLength, res.data, nil}, + {"foo=0-1", "", objLength, res.data, nil}, + {"bytes=--1", "", objLength, res.data, nil}, + {"bytes=0--1", "", objLength, res.data, nil}, + {"bytes= -1", "", objLength, res.data, nil}, + {"bytes=0 -1", "", objLength, res.data, nil}, + {"bytes=-1", "", objLength, res.data, nil}, + // error (RequestedRangeNotSatisfiable) + {"bytes=00-01", "", objLength, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + {"bytes=-0", "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + {"bytes=0-0", "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + {"bytes=0-", "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + } { + if err := testGetObjectRange(el.rng, el.contentRange, el.cLength, el.expData, el.expErr); err != nil { + return err + } + } + return nil + }) +} + +func getInt32(ptr *int32) int32 { + if ptr == nil { + return 0 + } + + return *ptr +} + +func putBucketCors(client *s3.Client, input *s3.PutBucketCorsInput) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutBucketCors(ctx, input) + cancel() + return err +} + +func compareCorsConfig(expected, got []types.CORSRule) error { + if expected == nil && got == nil { + return nil + } + if got == nil { + return errors.New("nil CORS config") + } + + if len(expected) != len(got) { + return fmt.Errorf("expected CORS rules length to be %v, instead got %v", len(expected), len(got)) + } + + for i, r := range expected { + rule := got[i] + if !slices.Equal(r.AllowedOrigins, rule.AllowedOrigins) { + return fmt.Errorf("expected the allowed origins to be %v, instead got %v", r.AllowedOrigins, rule.AllowedOrigins) + } + if !slices.Equal(r.AllowedMethods, rule.AllowedMethods) { + return fmt.Errorf("expected the allowed methods to be %v, instead got %v", r.AllowedMethods, rule.AllowedMethods) + } + if !slices.Equal(r.AllowedHeaders, rule.AllowedHeaders) { + return fmt.Errorf("expected the allowed headers to be %v, instead got %v", r.AllowedHeaders, rule.AllowedHeaders) + } + if !slices.Equal(r.ExposeHeaders, rule.ExposeHeaders) { + return fmt.Errorf("expected the allowed origins to be %v, instead got %v", r.ExposeHeaders, rule.ExposeHeaders) + } + if getInt32(r.MaxAgeSeconds) != getInt32(rule.MaxAgeSeconds) { + return fmt.Errorf("expected the max age seconds to be %v, instead got %v", getInt32(r.MaxAgeSeconds), getInt32(rule.MaxAgeSeconds)) + } + if getString(r.ID) != getString(rule.ID) { + return fmt.Errorf("expected ID to be %v, instead got %v", getString(r.ID), getString(rule.ID)) + } + } + + return nil +} + +type PreflightResult struct { + Origin string + Methods string + AllowHeaders string + ExposeHeaders string + MaxAge string + AllowCredentials string + Vary string + err error +} + +func extractCORSHeaders(resp *http.Response) (*PreflightResult, error) { + if resp.StatusCode >= 400 { + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + var errResp smithy.GenericAPIError + err = xml.Unmarshal(body, &errResp) + if err != nil { + return nil, fmt.Errorf("unmarshal response body: %w", err) + } + + return &PreflightResult{ + err: &errResp, + }, nil + } + + return &PreflightResult{ + Origin: resp.Header.Get("Access-Control-Allow-Origin"), + Methods: resp.Header.Get("Access-Control-Allow-Methods"), + ExposeHeaders: resp.Header.Get("Access-Control-Expose-Headers"), + MaxAge: resp.Header.Get("Access-Control-Max-Age"), + AllowHeaders: resp.Header.Get("Access-Control-Allow-Headers"), + AllowCredentials: resp.Header.Get("Access-Control-Allow-Credentials"), + Vary: resp.Header.Get("Vary"), + }, nil +} + +func makeOPTIONSRequest(s *S3Conf, bucket, origin, method string, headers string) (*PreflightResult, error) { + req, err := http.NewRequest(http.MethodOptions, fmt.Sprintf("%s/%s/object", s.endpoint, bucket), nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + req.Header.Add("Origin", origin) + req.Header.Add("Access-Control-Request-Method", method) + req.Header.Add("Access-Control-Request-Headers", headers) + + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("send request: %w", err) + } + + return extractCORSHeaders(resp) +} + +func comparePreflightResult(expected, got *PreflightResult) error { + if expected == nil { + return fmt.Errorf("nil expected preflight request result") + } + if got == nil { + return fmt.Errorf("expected the preflights result to be %v, instead got nil", *expected) + } + + if expected.err != nil { + if got.err == nil { + return fmt.Errorf("expected %w error, instaed got nil", expected.err) + } + + apiErr, ok := expected.err.(s3err.APIError) + if !ok { + return fmt.Errorf("expected s3err.APIError, instead got %w", expected.err) + } + + return checkApiErr(got.err, apiErr) + } + + if got.err != nil { + return fmt.Errorf("expected no error, instaed got %w", got.err) + } + + if expected.Origin != got.Origin { + return fmt.Errorf("expected the origin to be %v, instead got %v", expected.Origin, got.Origin) + } + if expected.Methods != got.Methods { + return fmt.Errorf("expected the allowed methods to be %v, instead got %v", expected.Methods, got.Methods) + } + if expected.AllowHeaders != got.AllowHeaders { + return fmt.Errorf("expected the allow headers to be %v, instead got %v", expected.AllowHeaders, got.AllowHeaders) + } + if expected.ExposeHeaders != got.ExposeHeaders { + return fmt.Errorf("expected the expose headers to be %v, instead got %v", expected.ExposeHeaders, got.ExposeHeaders) + } + if expected.MaxAge != got.MaxAge { + return fmt.Errorf("expected the max age to be %v, instead got %v", expected.MaxAge, got.MaxAge) + } + if expected.AllowCredentials != got.AllowCredentials { + return fmt.Errorf("expected the allow credentials to be %v, instead got %v", expected.AllowCredentials, got.AllowCredentials) + } + if expected.Vary != got.Vary { + return fmt.Errorf("expected the Vary header to be %v, instead got %v", expected.Vary, got.Vary) + } + + return nil +} + +func testOPTIONSEdnpoint(s *S3Conf, bucket, origin, method string, headers string, expected *PreflightResult) error { + result, err := makeOPTIONSRequest(s, bucket, origin, method, headers) + if err != nil { + return err + } + + return comparePreflightResult(expected, result) +} + +func calculateEtag(data []byte) (string, error) { + h := md5.New() + _, err := h.Write(data) + if err != nil { + return "", err + } + dataSum := h.Sum(nil) + return fmt.Sprintf("\"%s\"", hex.EncodeToString(dataSum[:])), nil +} + +func sprintBuckets(buckets []types.Bucket) string { + if len(buckets) == 0 { + return "" + } + + names := make([]string, len(buckets)) + for i, bucket := range buckets { + names[i] = *bucket.Name + } + + return strings.Join(names, ",") +} + +func sprintPrefixes(cpfx []types.CommonPrefix) string { + if len(cpfx) == 0 { + return "" + } + + names := make([]string, len(cpfx)) + for i, pfx := range cpfx { + names[i] = *pfx.Prefix + } + + return strings.Join(names, ",") +} + +func sprintVersions(objects []types.ObjectVersion) string { + if len(objects) == 0 { + return "" + } + + names := make([]string, len(objects)) + for i, obj := range objects { + names[i] = fmt.Sprintf("%v/%v", *obj.Key, obj.VersionId) + } + + return strings.Join(names, ",") +} + +// objToDelete represents the metadata of an object that needs to be deleted. +// It holds details like the key, version, and legal/compliance lock flags. +type objToDelete struct { + key string // Object key (name) in the bucket + versionId string // Specific object version ID + removeLegalHold bool // Whether to remove legal hold before deletion + removeOnlyLeglHold bool // Whether to only remove legal hold, without deletion + isCompliance bool // Whether the object is under Compliance mode retention +} + +// Worker and retry configuration for deleting locked objects +const ( + maxDelObjWorkers int64 = 20 // Maximum number of concurrent delete workers + maxRetryAttempts int = 3 // Maximum retries for object deletion + lockWaitTime time.Duration = time.Second * 3 // Wait time for lock expiration before retrying delete + + // complianceTestRetention is how far out a test should set a COMPLIANCE + // retention it means to clean up afterwards. A COMPLIANCE retention can + // never be shortened or removed — by anyone, with any permission, which + // is the whole point of the mode — so the only way to release the object + // is to outlast it, which cleanupLockedObjects does. Long enough for the + // test body to run against a genuinely locked object, short enough to + // wait out. + complianceTestRetention time.Duration = time.Second * 10 + // maxComplianceCleanupWait bounds that wait, so a test that locks an + // object for hours fails quickly and clearly instead of hanging. + maxComplianceCleanupWait time.Duration = time.Second * 30 +) + +// cleanupLockedObjects removes objects from a bucket that may be protected by +// Object Lock (legal hold or retention). +// It handles both Governance and Compliance retention modes and retries deletions +// when necessary. +func cleanupLockedObjects(client *s3.Client, bucket string, objs []objToDelete) error { + eg, ctx := errgroup.WithContext(context.Background()) + + // Semaphore to limit the number of concurrent workers + sem := semaphore.NewWeighted(maxDelObjWorkers) + + for _, obj := range objs { + + // Acquire worker slot before processing an object + if err := sem.Acquire(ctx, 1); err != nil { + return fmt.Errorf("failed to acquire worker space: %w", err) + } + + defer sem.Release(1) + + eg.Go(func() error { + // Remove legal hold if required + if obj.removeLegalHold || obj.removeOnlyLeglHold { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &obj.key, + VersionId: getNonEmptyPtr(obj.versionId), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOff, // Disable legal hold + }, + }) + cancel() + // If object was already deleted, ignore the error + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return nil + } + if err != nil { + return err + } + + // If only the legal hold needs to be removed, stop here + if obj.removeOnlyLeglHold { + return nil + } + } + + // A COMPLIANCE retention can only be waited out — it cannot be + // shortened or removed by anyone. Tests that mean to clean up + // therefore lock for complianceTestRetention, and this sleeps + // until that has passed. + if obj.isCompliance { + return waitOutComplianceRetention(client, bucket, obj) + } + + // A GOVERNANCE retention can be weakened with the bypass + // permission and header, so shorten it to a few seconds out + // rather than waiting for the original date. The margin absorbs + // network delay. + retDate := time.Now().Add(lockWaitTime) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj.key, + VersionId: getNonEmptyPtr(obj.versionId), + BypassGovernanceRetention: getBoolPtr(true), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &retDate, + }, + }) + cancel() + + // If object was already deleted, ignore the error + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return nil + } + + if err != nil { + return err + } + + // Wait until retention lock expires before attempting delete + time.Sleep(lockWaitTime) + + // Return last error if all retries failed + return nil + }) + } + + // Wait for all goroutines to finish, return any error encountered + return eg.Wait() +} + +// waitOutComplianceRetention blocks until obj's retention has passed, the +// only way to release a COMPLIANCE-locked object. A retention further out +// than maxComplianceCleanupWait is reported as an error rather than waited +// on: such an object cannot be cleaned up within a test run at all, and the +// test that locked it should either use complianceTestRetention or skip +// teardown. +func waitOutComplianceRetention(client *s3.Client, bucket string, obj objToDelete) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &obj.key, + VersionId: getNonEmptyPtr(obj.versionId), + }) + cancel() + + // Already deleted: nothing to wait for. + if err != nil && checkSdkApiErr(err, "NoSuchKey") == nil { + return nil + } + + // No retention of its own means the object is protected only by the + // bucket's default retention. Nothing forbids giving it a short one of + // its own — an object-level retention supersedes the bucket default, and + // there is no existing retention here to weaken — so that is how such an + // object gets released. + noRetention := err != nil && checkSdkApiErr(err, "NoSuchObjectLockConfiguration") == nil + if err != nil && !noRetention { + return err + } + if !noRetention && (out.Retention == nil || out.Retention.RetainUntilDate == nil) { + noRetention = true + } + if noRetention { + retDate := time.Now().Add(lockWaitTime) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &obj.key, + VersionId: getNonEmptyPtr(obj.versionId), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: &retDate, + }, + }) + cancel() + if err != nil && checkSdkApiErr(err, "NoSuchKey") != nil { + return err + } + time.Sleep(lockWaitTime) + return nil + } + + // The extra second absorbs clock skew between this process and the + // gateway, which compares the retention against its own clock. + wait := time.Until(*out.Retention.RetainUntilDate) + time.Second + if wait > maxComplianceCleanupWait { + return fmt.Errorf("object %q is under COMPLIANCE retention until %v, too far out to wait for: use complianceTestRetention, or skip teardown", + obj.key, out.Retention.RetainUntilDate) + } + if wait > 0 { + time.Sleep(wait) + } + return nil +} + +type objectLockMode string + +const ( + objectLockModeLegalHold = "legal-hold" + objectLockModeGovernance = "governance" + objectLockModeCompliance = "compliance" +) + +func lockObject(client *s3.Client, mode objectLockMode, bucket, object, versionId string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + defer cancel() + var m types.ObjectLockRetentionMode + switch mode { + case objectLockModeLegalHold: + _, err := client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &object, + VersionId: getNonEmptyPtr(versionId), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + }) + return err + case objectLockModeCompliance: + m = types.ObjectLockRetentionModeCompliance + case objectLockModeGovernance: + m = types.ObjectLockRetentionModeGovernance + default: + return fmt.Errorf("invalid object lock mode: %s", mode) + } + + date := time.Now().Add(time.Hour * 3) + _, err := client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &object, + VersionId: getNonEmptyPtr(versionId), + Retention: &types.ObjectLockRetention{ + Mode: m, + RetainUntilDate: &date, + }, + }) + return err +} + +func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) { + var hasher hash.Hash + switch algo { + case types.ChecksumAlgorithmMd5: + hasher = md5.New() + case types.ChecksumAlgorithmSha256: + hasher = sha256.New() + case types.ChecksumAlgorithmSha512: + hasher = sha512.New() + case types.ChecksumAlgorithmSha1: + hasher = sha1.New() + case types.ChecksumAlgorithmCrc32: + hasher = crc32.NewIEEE() + case types.ChecksumAlgorithmCrc32c: + hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case types.ChecksumAlgorithmCrc64nvme: + hasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))) + case types.ChecksumAlgorithmXxhash64: + hasher = xxhash.New() + case types.ChecksumAlgorithmXxhash3: + hasher = xxh3.New() + case types.ChecksumAlgorithmXxhash128: + hasher = xxh3.New128() + default: + return nil, fmt.Errorf("unsupported hash algorithm: %s", algo) + } + + return hasher, nil +} + +func wrongChecksumForAlgorithm(algo types.ChecksumAlgorithm) (string, error) { + var size int + switch algo { + case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c: + size = 4 + case types.ChecksumAlgorithmCrc64nvme, types.ChecksumAlgorithmXxhash64, types.ChecksumAlgorithmXxhash3: + size = 8 + case types.ChecksumAlgorithmMd5, types.ChecksumAlgorithmXxhash128: + size = 16 + case types.ChecksumAlgorithmSha1: + size = 20 + case types.ChecksumAlgorithmSha256: + size = 32 + case types.ChecksumAlgorithmSha512: + size = 64 + default: + return "", fmt.Errorf("unsupported hash algorithm: %s", algo) + } + + return base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0xff}, size)), nil +} + +func processCompositeChecksum(hasher hash.Hash, checksum string) error { + data, err := base64.StdEncoding.DecodeString(checksum) + if err != nil { + return fmt.Errorf("base64 decode: %w", err) + } + + _, err = hasher.Write(data) + if err != nil { + return fmt.Errorf("hash write: %w", err) + } + + return nil +} + +type mpinfo struct { + uploadId *string + parts []types.CompletedPart +} + +func putBucketPolicy(client *s3.Client, bucket, policy string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: &bucket, + Policy: &policy, + }) + cancel() + return err +} + +func sendSignedRequest(s *S3Conf, req *http.Request, cancel context.CancelFunc) (map[string]string, *APIErrorResponse, error) { + signer := v4.NewSigner() + signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, req, "STREAMING-UNSIGNED-PAYLOAD-TRAILER", "s3", s.awsRegion, time.Now()) + if signErr != nil { + cancel() + return nil, nil, fmt.Errorf("failed to sign the request: %w", signErr) + } + + resp, err := s.httpClient.Do(req) + cancel() + if err != nil { + return nil, nil, fmt.Errorf("failed to send the request: %w", err) + } + + if resp.StatusCode >= 300 { + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("failed to read the request body: %w", err) + } + + var errResp APIErrorResponse + err = xml.Unmarshal(bodyBytes, &errResp) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal response body: %w", err) + } + + return nil, &errResp, nil + } + + headers := map[string]string{} + for key, val := range resp.Header { + headers[strings.ToLower(key)] = val[0] + } + + return headers, nil, nil +} + +func testUnsignedStreamingPayloadTrailerObjectPut(s *S3Conf, bucket, object string, body []byte, reqHeaders map[string]string) (map[string]string, *APIErrorResponse, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, s.endpoint+"/"+bucket+"/"+object, bytes.NewReader(body)) + if err != nil { + cancel() + return nil, nil, fmt.Errorf("failed to create a request: %w", err) + } + + req.Header.Add("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER") + for key, val := range reqHeaders { + req.Header.Add(key, val) + } + + return sendSignedRequest(s, req, cancel) +} + +func testUnsignedStreamingPayloadTrailerUploadPart(s *S3Conf, bucket, object string, uploadId *string, body []byte, reqHeaders map[string]string) (map[string]string, *APIErrorResponse, error) { + if uploadId == nil { + return nil, nil, fmt.Errorf("empty upload id") + } + + uri := fmt.Sprintf("%s/%s/%s?uploadId=%s&partNumber=%v", s.endpoint, bucket, object, *uploadId, 1) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uri, bytes.NewReader(body)) + if err != nil { + cancel() + return nil, nil, fmt.Errorf("failed to create a request: %w", err) + } + + req.Header.Add("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER") + for key, val := range reqHeaders { + req.Header.Add(key, val) + } + + return sendSignedRequest(s, req, cancel) +} + +// constructUnsignedPaylod constructs an unsigned streaming upload payload +// and returns the decoded content length and the payload +func constructUnsignedPaylod(chunkSizes ...int64) (int64, []byte, error) { + var cLength int64 + buffer := bytes.NewBuffer([]byte{}) + + for _, chunkSize := range chunkSizes { + cLength += chunkSize + _, err := buffer.WriteString(fmt.Sprintf("%x\r\n", chunkSize)) + if err != nil { + return 0, nil, err + } + _, err = buffer.WriteString(strings.Repeat("a", int(chunkSize))) + if err != nil { + return 0, nil, err + } + _, err = buffer.WriteString("\r\n") + if err != nil { + return 0, nil, err + } + } + + return cLength, buffer.Bytes(), nil +} + +type signedReqCfg struct { + headers map[string]string + chunkSize int64 + modifFrom *int + modifTo *int + modifPayload []byte + trailingChecksum *string + isTrailer bool +} + +type signedReqOpt func(*signedReqCfg) + +func withCustomHeaders(h map[string]string) signedReqOpt { + return func(src *signedReqCfg) { src.headers = h } +} + +func withChunkSize(s int64) signedReqOpt { + return func(src *signedReqCfg) { src.chunkSize = s } +} + +func withModifyPayload(from int, to int, p []byte) signedReqOpt { + return func(src *signedReqCfg) { + src.modifPayload = p + src.modifFrom = &from + src.modifTo = &to + } +} + +func withTrailingChecksum(checksum string) signedReqOpt { + return func(src *signedReqCfg) { + src.trailingChecksum = &checksum + src.isTrailer = true + } +} + +func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []byte, opts ...signedReqOpt) (map[string]string, *APIErrorResponse, error) { + cfg := &signedReqCfg{ + chunkSize: 8192, // minimal valid chunk size + } + + for _, opt := range opts { + opt(cfg) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + // create a request with no body + req, err := http.NewRequestWithContext(ctx, http.MethodPut, fmt.Sprintf("%s/%s/%s", s.endpoint, bucket, object), nil) + if err != nil { + return nil, nil, cancelAndError(fmt.Errorf("failed to create a request: %w", err), cancel) + } + + var payloadOffset int64 + var trailerLength int + + // any planned modification which is going to affect the + // Content-Length header value + if cfg.modifFrom != nil && cfg.modifTo != nil { + diff := len(cfg.modifPayload) - *cfg.modifTo + *cfg.modifFrom + payloadOffset = int64(diff) + } + if cfg.isTrailer { + trailerLength = len(*cfg.trailingChecksum) + } + // precalculated the Content-Length header to correctly sign the request + req.ContentLength = calculateSignedReqContentLength(int64(len(payload)), cfg.chunkSize, payloadOffset, cfg.isTrailer, int64(trailerLength)) + sha256Header := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + if cfg.isTrailer { + sha256Header = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + } + req.Header.Set("x-amz-decoded-content-length", fmt.Sprint(len(payload))) + req.Header.Set("x-amz-content-sha256", sha256Header) + + // set custom request headers + for key, val := range cfg.headers { + req.Header.Set(key, val) + } + + signer := v4.NewSigner() + signingTime := time.Now().UTC() + + // sign the request + err = signer.SignHTTP(ctx, aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, req, sha256Header, "s3", s.awsRegion, signingTime) + if err != nil { + return nil, nil, cancelAndError(fmt.Errorf("failed to sign the request: %w", err), cancel) + } + + // extract the seed signature + seedSignature, err := extractSignature(req) + if err != nil { + return nil, nil, cancelAndError(fmt.Errorf("failed to extract seed signature: %w", err), cancel) + } + + // initialize v4 stream signed + streamSigner := v4.NewStreamSigner(aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, "s3", s.awsRegion, seedSignature) + // create the signed payload + body, err := constructSignedStreamingPayload(ctx, streamSigner, signingTime, payload, cfg.chunkSize, cfg.trailingChecksum, s.awsRegion, s.awsSecret) + if err != nil { + return nil, nil, cancelAndError(fmt.Errorf("failed to encode req body: %w", err), cancel) + } + + // overwrite body bytes by configuration + if cfg.modifFrom != nil && cfg.modifTo != nil { + body, err = replaceRange(body, cfg.modifPayload, *cfg.modifFrom, *cfg.modifTo) + if err != nil { + return nil, nil, cancelAndError(fmt.Errorf("failed replace body bytes: %w", err), cancel) + } + } + + // assign req.Body and req.GetBody for the http client + // to handle the request + req.Body = io.NopCloser(bytes.NewReader(body)) + req.GetBody = func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(body)), nil + } + + // send the request + resp, err := s.httpClient.Do(req) + cancel() + if err != nil { + return nil, nil, fmt.Errorf("failed to send the request: %w", err) + } + + if resp.StatusCode >= 300 { + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("failed to read the response body: %w", err) + } + + var errResp APIErrorResponse + err = xml.Unmarshal(bodyBytes, &errResp) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal response body: %w", err) + } + return nil, &errResp, nil + } + + headers := map[string]string{} + for key, val := range resp.Header { + headers[strings.ToLower(key)] = val[0] + } + + return headers, nil, nil +} + +func cancelAndError(err error, cancel context.CancelFunc) error { + cancel() + return err +} + +const ( + chunkSigHdrLength int64 = 81 + trailerSigLength int64 = 88 +) + +// calculateSignedReqContentLength calculates the value of `Content-Length` header +// sizeOffset marks any planned changes on the body, which will affect the size +func calculateSignedReqContentLength(decPayloadSize int64, chunkSize int64, sizeOffset int64, withTrailer bool, trailerLength int64) int64 { + payloadSize := decPayloadSize + var chunkHeadersLength int64 + + if withTrailer { + chunkHeadersLength += trailerLength + 4 + trailerSigLength + } + + // special case when chunk size is greater or equal than decoded content length + if chunkSize >= decPayloadSize { + chSizeLgth := len(fmt.Sprintf("%x", decPayloadSize)) + return decPayloadSize + sizeOffset + int64(chSizeLgth) + 2*chunkSigHdrLength + 9 + chunkHeadersLength + } + + for { + if payloadSize == 0 { + chunkHeadersLength += chunkSigHdrLength + 5 + break + } + if payloadSize < chunkSize { + chunkHeadersLength += 2*chunkSigHdrLength + 9 + int64(len(fmt.Sprintf("%x", payloadSize))) + break + } + chSizeLgth := len(fmt.Sprintf("%x", chunkSize)) + chunkHeadersLength += int64(chSizeLgth) + chunkSigHdrLength + 4 + + payloadSize -= chunkSize + } + + return chunkHeadersLength + decPayloadSize + sizeOffset +} + +// constructSignedStreamingPayload creates chunk encoded payload with signatures. +func constructSignedStreamingPayload(ctx context.Context, signer *v4.StreamSigner, signingTime time.Time, payload []byte, chunkSize int64, trailer *string, region, secret string) ([]byte, error) { + buf := bytes.NewBuffer(nil) + payloadLen := int64(len(payload)) + + if chunkSize > payloadLen { + chunkSize = payloadLen + } + + for i := int64(0); i < payloadLen; i += chunkSize { + if i+chunkSize > payloadLen { + offset := payloadLen - i + sig, err := signer.GetSignature(ctx, nil, payload[i:i+offset], signingTime) + if err != nil { + return nil, err + } + + _, err = buf.WriteString(fmt.Sprintf("%x;chunk-signature=%x\r\n%s\r\n", offset, sig, payload[i:i+offset])) + if err != nil { + return nil, err + } + break + } + + sig, err := signer.GetSignature(ctx, nil, payload[i:i+chunkSize], signingTime) + if err != nil { + return nil, err + } + + _, err = buf.WriteString(fmt.Sprintf("%x;chunk-signature=%x\r\n%s\r\n", chunkSize, sig, payload[i:i+chunkSize])) + if err != nil { + return nil, err + } + } + + sig, err := signer.GetSignature(ctx, nil, nil, signingTime) + if err != nil { + return nil, err + } + + if trailer != nil { + _, err = buf.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n", sig)) + if err != nil { + return nil, err + } + + sigKey := getSigningKey(secret, signingTime.Format("20060102"), region) + trailerSig, err := getAWS4StreamingTrailer(sigKey, sig, signingTime, region, *trailer) + if err != nil { + return nil, err + } + + _, err = buf.WriteString(fmt.Sprintf("%s\r\nx-amz-trailer-signature:%s\r\n\r\n", *trailer, trailerSig)) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil + } + + _, err = buf.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n\r\n", sig)) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// extractSignature extracts the signature from Authorization header +func extractSignature(req *http.Request) ([]byte, error) { + const key = "Signature=" + + authHdr := req.Header.Get("Authorization") + + _, after, ok := strings.Cut(authHdr, key) + if !ok { + return nil, errors.New("signature not found") + } + + sig := after + + return hex.DecodeString(sig) +} + +// replaceRange replaces dst[start:end] with src and returns the modified slice. +// Used for custom overwrite of request payload bytes. +func replaceRange(dst, src []byte, start, end int) ([]byte, error) { + if start < 0 || end < start || end > len(dst) { + return nil, fmt.Errorf("invalid start/end indexes") + } + + newLen := len(dst) - (end - start) + len(src) + + // Fast path: reuse dst capacity if possible + if cap(dst) >= newLen { + // Extend or shrink dst + dst = dst[:newLen] + + // Move the tail if sizes differ + copy(dst[start+len(src):], dst[end:]) + + // Copy replacement + copy(dst[start:], src) + return dst, nil + } + + // Fallback: allocate new slice + out := make([]byte, newLen) + copy(out, dst[:start]) + copy(out[start:], src) + copy(out[start+len(src):], dst[end:]) + return out, nil +} + +func getAWS4StreamingTrailer( + signingKey, + lastSignature []byte, + signingTime time.Time, + awsRegion, + trailer string, +) (string, error) { + + // yyyyMMdd + yearMonthDay := signingTime.Format("20060102") + + // ISO8601 basic format: yyyyMMdd'T'HHmmss'Z' + currentDateTime := signingTime.Format("20060102T150405Z") + + // ///aws4_request + serviceString := fmt.Sprintf( + "%s/%s/s3/aws4_request", + yearMonthDay, + awsRegion, + ) + + // Trailer must be newline-terminated for hashing/signing + trailerWithNL := trailer + "\n" + + // Hash of trailer + trailerHash := sha256.Sum256([]byte(trailerWithNL)) + trailerHashHex := hex.EncodeToString(trailerHash[:]) + + // String-to-sign prefix + stringToSignPrefix := fmt.Sprintf( + "%s\n%s\n%s", + "AWS4-HMAC-SHA256-TRAILER", + currentDateTime, + serviceString, + ) + + // Full string-to-sign + stringToSign := fmt.Sprintf( + "%s\n%x\n%s", + stringToSignPrefix, + lastSignature, + trailerHashHex, + ) + + // Final trailer signature + finalSignature := hex.EncodeToString( + hmacSHA256(signingKey, stringToSign), + ) + + return finalSignature, nil +} + +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + h.Write([]byte(data)) + return h.Sum(nil) +} + +func getSigningKey(secret, yearMonthDay, region string) []byte { + dateKey := hmacSHA256([]byte("AWS4"+secret), yearMonthDay) + dateRegionKey := hmacSHA256(dateKey, region) + dateRegionServiceKey := hmacSHA256(dateRegionKey, "s3") + return hmacSHA256(dateRegionServiceKey, "aws4_request") +} + +// buildPostObjectBody builds a multipart/form-data body for a POST object request. +// Fields are written first (in the order provided) +// Returns (body bytes, boundary string, error). +func buildPostObjectBody(fields, extraFields map[string]string, fileContent []byte) ([]byte, string, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + for key, value := range fields { + v, ok := extraFields[key] + if ok { + delete(extraFields, key) + if v == "" { + continue + } + value = v + } + if err := w.WriteField(key, value); err != nil { + return nil, "", err + } + } + + for key, value := range extraFields { + if err := w.WriteField(key, value); err != nil { + return nil, "", err + } + } + + fw, err := w.CreateFormFile("file", "upload.bin") + if err != nil { + return nil, "", err + } + if _, err = fw.Write(fileContent); err != nil { + return nil, "", err + } + + if err := w.Close(); err != nil { + return nil, "", err + } + + return buf.Bytes(), w.Boundary(), nil +} + +// encodePostPolicy base64-encodes a POST policy JSON document. +// optionally it omits some fields based on the input configuration +func encodePostPolicy(conditions []any, expiration time.Time, fields map[string]string, omitConditions map[string]struct{}) (string, error) { + for key, value := range fields { + if _, ok := omitConditions[key]; ok { + continue + } + + conditions = append(conditions, map[string]string{ + key: value, + }) + } + + if expiration.IsZero() { + expiration = time.Now().UTC().Add(15 * time.Minute) + } + + policy := map[string]any{ + "expiration": expiration.UTC().Format(time.RFC3339), + "conditions": conditions, + } + + b, err := json.Marshal(policy) + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(b), nil +} + +// signPostPolicy computes the AWS SigV4 HMAC-SHA256 signature over a base64-encoded +// POST policy document. +func signPostPolicy(policyB64, dateShort, region, secret string) string { + signingKey := getSigningKey(secret, dateShort, region) + sig := hmacSHA256(signingKey, policyB64) + return hex.EncodeToString(sig) +} + +// buildSignedPostFields returns the complete set of AWS authentication form fields +// for a signed POST object request. +// - key +// - x-amz-algorithm +// - x-amz-credential +// - x-amz-date +func buildSignedPostFields(bucket, key, access, region string, date time.Time) map[string]string { + dateShort := date.Format("20060102") + dateLong := date.Format(iso8601Format) + credential := fmt.Sprintf("%s/%s/%s/s3/aws4_request", access, dateShort, region) + + return map[string]string{ + "bucket": bucket, + "key": key, + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "x-amz-credential": credential, + "x-amz-date": dateLong, + } +} + +type PostRequestConfig struct { + bucket string + key string + access string + secret string + region string + extraFields map[string]string + fileContent []byte + rawPolicy *string + date time.Time + policyExpiration time.Time + policyConditions []any + omitPolicyConditions map[string]struct{} + s3Conf *S3Conf +} + +// sendPostObject sends a POST multipart/form-data request to /{bucket}. +// Returns the raw *http.Response for flexible per-test assertions. +func sendPostObject(input PostRequestConfig) (*http.Response, error) { + req, _, err := newPostObjectRequest(input) + if err != nil { + return nil, err + } + + return input.s3Conf.httpClient.Do(req) +} + +// sendAnonymousPostObject sends an unauthenticated POST object request to +// /{bucket}: the form carries key and the file, but none of the five +// form-based auth fields. +func sendAnonymousPostObject(s *S3Conf, bucket, key string, fileContent []byte) (*http.Response, error) { + return sendPostObject(PostRequestConfig{ + bucket: bucket, + key: key, + s3Conf: s, + fileContent: fileContent, + extraFields: map[string]string{ + "x-amz-algorithm": "", + "x-amz-credential": "", + "x-amz-date": "", + "policy": "", + "x-amz-signature": "", + }, + }) +} + +// checkPostObjectSuccess checks that resp is the 204 No Content a POST +// object upload returns by default, reporting the response body otherwise. +func checkPostObjectSuccess(resp *http.Response) error { + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("expected status 204, instead got %d: %s", resp.StatusCode, body) + } + return nil +} + +func newPostObjectRequest(input PostRequestConfig) (*http.Request, map[string]string, error) { + if input.date.IsZero() { + input.date = time.Now().UTC() + } + if input.policyExpiration.IsZero() { + input.policyExpiration = time.Now().UTC().AddDate(0, 0, 1) + } + if input.access == "" { + input.access = input.s3Conf.awsID + } + if input.secret == "" { + input.secret = input.s3Conf.awsSecret + } + if input.region == "" { + input.region = input.s3Conf.awsRegion + } + + fields := buildSignedPostFields(input.bucket, input.key, input.access, input.region, input.date) + + var policy string + if input.rawPolicy == nil { + var err error + policy, err = encodePostPolicy(input.policyConditions, input.policyExpiration, fields, input.omitPolicyConditions) + if err != nil { + return nil, nil, err + } + } else { + policy = *input.rawPolicy + } + + fields["policy"] = policy + fields["x-amz-signature"] = signPostPolicy(policy, input.date.Format("20060102"), input.region, input.secret) + + body, boundary, err := buildPostObjectBody(fields, input.extraFields, input.fileContent) + if err != nil { + return nil, nil, err + } + + endpoint := fmt.Sprintf("%s/%s", input.s3Conf.endpoint, input.bucket) + if input.s3Conf.hostStyle { + u, err := url.Parse(input.s3Conf.endpoint) + if err != nil { + return nil, nil, err + } + + u.Host = input.bucket + "." + u.Host + endpoint = u.String() + } + + req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + + req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%s", boundary)) + req.ContentLength = int64(len(body)) + + return req, fields, nil +} + +func getEtagBytes(etag string) ([]byte, error) { + return hex.DecodeString(strings.ReplaceAll(etag, string('"'), "")) +} + +func md5String(data []byte) string { + sum := md5.Sum(data) + return hex.EncodeToString(sum[:]) +} + +func base64ToHexString(s string) string { + data, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return "" + } + return hex.EncodeToString(data) +} + +func hexBytes(s string) string { + data := []byte(s) + parts := make([]string, len(data)) + for i, b := range data { + parts[i] = fmt.Sprintf("%02x", b) + } + return strings.Join(parts, " ") +} + +// keyDenial pairs a DeleteObjects key with the error its response entry is +// expected to carry. +type keyDenial struct { + key string + err s3err.S3Error +} + +// checkDeleteObjectsErrsInOrder checks that got names exactly the (key, +// error) pairs in want, in that order — DeleteObjects preserves the order +// objects were requested in across both the Deleted and Error lists. +func checkDeleteObjectsErrsInOrder(got []types.Error, want []keyDenial) error { + if len(got) != len(want) { + return fmt.Errorf("expected %d per-object errors, got %d: %+v", len(want), len(got), got) + } + for i, w := range want { + if err := checkDeleteObjectsErr(got[i], w.key, w.err); err != nil { + return fmt.Errorf("error %d: %w", i, err) + } + } + return nil +} + +// checkObjectTaggingErr checks that getting, putting and deleting the +// tagging of the version versionId of key fail with expected. An empty +// versionId selects the current version. +func checkObjectTaggingErr(client *s3.Client, bucket, key, versionId string, expected s3err.S3Error) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object tagging: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + Tagging: &types.Tagging{ + TagSet: []types.Tag{{Key: getPtr("other-key"), Value: getPtr("other-value")}}, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object tagging: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("delete object tagging: %w", err) + } + + return nil +} + +// checkObjectLockErr checks that getting and putting the legal hold and the +// retention of the version versionId of key fail with expected. An empty +// versionId selects the current version. +func checkObjectLockErr(client *s3.Client, bucket, key, versionId string, expected s3err.S3Error) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object legal hold: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + LegalHold: &types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOff, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object legal hold: %w", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("get object retention: %w", err) + } + + rDate := time.Now().Add(time.Hour * 2) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: &rDate, + }, + }) + cancel() + if err := checkApiErr(err, expected); err != nil { + return fmt.Errorf("put object retention: %w", err) + } + + return nil +} + +// checkObjectLock checks that the version versionId of key is under legal +// hold and has a governance retention until rDate. An empty versionId +// selects the current version. +func checkObjectLock(client *s3.Client, bucket, key, versionId string, rDate time.Time) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + lHold, err := client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err != nil { + return err + } + if lHold.LegalHold == nil || lHold.LegalHold.Status != types.ObjectLockLegalHoldStatusOn { + return fmt.Errorf("expected the legal hold status to be %q, instead got %v", + types.ObjectLockLegalHoldStatusOn, lHold.LegalHold) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ret, err := client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{ + Bucket: &bucket, + Key: &key, + VersionId: getNonEmptyPtr(versionId), + }) + cancel() + if err != nil { + return err + } + if ret.Retention == nil || ret.Retention.Mode != types.ObjectLockRetentionModeGovernance || + ret.Retention.RetainUntilDate == nil || ret.Retention.RetainUntilDate.Unix() != rDate.Unix() { + return fmt.Errorf("expected a %q retention until %v, instead got %+v", + types.ObjectLockRetentionModeGovernance, rDate.Format(time.RFC3339), ret.Retention) + } + + return nil +} + +// checkAndAbortUpload checks that the upload uploadId of key is the only +// multipart upload in the bucket and aborts it +func checkAndAbortUpload(client *s3.Client, bucket, key, uploadId string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + if len(res.Uploads) != 1 || getString(res.Uploads[0].Key) != key || + getString(res.Uploads[0].UploadId) != uploadId { + return fmt.Errorf("expected the upload %v of %v to be listed, instead got %+v", + uploadId, key, res.Uploads) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + }) + cancel() + return err +} + +// getStoredContentEncoding returns the Content-Encoding stored for an object, +// reporting an absent header as the empty string. +func getStoredContentEncoding(s3client *s3.Client, bucket, object string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &object, + }) + cancel() + if err != nil { + return "", err + } + if out.ContentEncoding == nil { + return "", nil + } + + return *out.ContentEncoding, nil +} diff --git a/tests/test_rest_get_object.sh b/tests/test_rest_get_object.sh new file mode 100644 index 0000000..c6133dd --- /dev/null +++ b/tests/test_rest_get_object.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bats + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/commands/get_object.sh +source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh +source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh +source ./tests/setup.sh + +# tags: curl,GetObject,HeadObject,range +@test "REST - range download and compare" { + run get_file_name + assert_success + test_file="$output" + + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_and_large_file_v2 "$bucket_name" "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" + assert_success + + download_chunk_size="2000000" + run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$download_chunk_size" + assert_success +} + +# tags: curl,PutObject,GetObject,special-chars,encoding +@test "REST - put, get object, encoded name" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + file_name=" \"<>\\^\`{}|+&?%" + run setup_bucket_and_file_v2 "$bucket_name" "$file_name" + assert_success + + run put_object_rest_special_chars "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name/$file_name" + assert_success + + run list_check_single_object "$bucket_name" "$file_name/$file_name" + assert_success + + run get_object_rest_special_chars "$bucket_name" "$file_name/$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" + assert_success + + run compare_files "$TEST_FILE_FOLDER/$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" + assert_success + + run delete_object_rest "$bucket_name" "$file_name/$file_name" + assert_success +} + +# tags: curl,GetObject,invalid-header,x-amz-content-sha256 +@test "REST - GetObject w/invalid payload type" { + run get_file_name + assert_success + test_file="$output" + + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket_and_file_v2 "$bucket_name" "$test_file" + assert_success + + run get_object_rest_with_invalid_streaming_type "$bucket_name" "$test_file" + assert_success +} + +# tags: curl,GetObject,partNumber,invalid-header +@test "REST - GetObject - part number 2 w/o multipart upload" { + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + run send_rest_go_command_expect_error "416" "InvalidPartNumber" "not satisfiable" "-bucketName" "$bucket_name" "-objectKey" "$file_name" "-query" "partNumber=2" + assert_success +} + +# tags: curl,GetObject,partNumber,range,Content-Range +@test "REST - GetObject - part number 1 returns 206, Content-Range header" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2074" + fi + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + run send_rest_go_command_callback "206" "check_for_header_key_and_value" "-bucketName" "$bucket_name" "-objectKey" "$file_name" "-query" "partNumber=1" \ + "--" "Content-Range" "bytes 0-9/10" + assert_success +} + +# tags: curl,GetObject,invalid-header,response-headers +@test "REST - GetObject - response query - invalid response type" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2075" + fi + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + local bad_response_query="response-gibberish" + run send_rest_go_command_expect_error "400" "InvalidArgument" "$bad_response_query is not in the set of overridable response headers" \ + "-bucketName" "$bucket_name" "-objectKey" "$file_name" "-query" "$bad_response_query=dummy" + assert_success +} + +# tags: curl,GetObject,response-cache-control,response-headers +@test "REST - GetObject - response-cache-control" { + run test_get_object_with_custom_content_header "cache-control" "dummy" + assert_success +} + +# tags: curl,GetObject,response-content-disposition,response-headers +@test "REST - GetObject - response-content-disposition" { + run test_get_object_with_custom_content_header "content-disposition" "dummy" + assert_success +} + +# tags: curl,GetObject,response-content-encoding,response-headers +@test "REST - GetObject - response-content-encoding" { + run test_get_object_with_custom_content_header "content-encoding" "dummy" + assert_success +} + +# tags: curl,GetObject,response-content-language,response-headers +@test "REST - GetObject - response-content-language" { + run test_get_object_with_custom_content_header "content-language" "dummy" + assert_success +} + +# tags: curl,GetObject,response-content-type,response-headers +@test "REST - GetObject - response-content-type" { + run test_get_object_with_custom_content_header "content-type" "dummy" + assert_success +} + +# tags: curl,GetObject,response-expires,response-headers +@test "REST - GetObject - response-expires" { + run test_get_object_with_custom_content_header "expires" "dummy+ one" + assert_success +} + +# tags: curl,GetObject,multipart,partNumber +@test "REST - GetObject - partNumber w/multipart" { + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 8 + assert_success + read -r bucket_name test_file <<< "$output" + + run split_file_irregular "$TEST_FILE_FOLDER/$test_file" 5242880 + assert_success + read -r part_one part_two <<< "$output" + log 5 "part one: $part_one, part two: $part_two" + + run perform_multipart_upload_rest_variable_parts "$bucket_name" "$test_file" "$part_one" "$part_two" + assert_success + + run get_file_size "$part_one" + assert_success + part_size="$output" + + run get_file_size "$TEST_FILE_FOLDER/$test_file" + assert_success + file_size="$output" + + run send_get_object_with_part_number_validate_response "$bucket_name" "$test_file" "1" "$part_one" "$file_size" "$part_size" + assert_success + + run send_get_object_with_part_number_validate_response "$bucket_name" "$test_file" "2" "$part_two" "$file_size" "$part_size" + assert_success +} diff --git a/tests/test_rest_head_object.sh b/tests/test_rest_head_object.sh new file mode 100644 index 0000000..2758b9d --- /dev/null +++ b/tests/test_rest_head_object.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bats + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/setup.sh +source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh +source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh +source ./tests/drivers/get_object_attributes/get_object_attributes_rest.sh +source ./tests/drivers/string.sh + +# tags: curl,HeadObject,GetObjectAttributes,ETag,x-amz-object-attributes +@test "REST - head object" { + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run get_etag_rest "$bucket_name" "$file_name" + assert_success + expected_etag=$output + + run check_etag_attribute_rest "$bucket_name" "$file_name" "$expected_etag" + assert_success +} + +# tags: curl,HeadObject,Content-Type +@test "REST - HeadObject - default Content-Type is binary/octet-stream" { + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run head_object_check_header_key_and_value "$bucket_name" "$file_name" "Content-Type" "binary/octet-stream" + assert_success +} + +# tags: curl,HeadObject,partNumber,invalid-query +@test "REST - HeadObject - letter partNumber" { + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run send_rest_go_command "400" "-method" "HEAD" "-query" "partNumber=abc" "-objectKey" "$file_name" "-bucketName" "$bucket_name" + assert_success +} + +# tags: curl,HeadObject,partNumber,invalid-query +@test "REST - HeadObject - invalid partNumber" { + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run send_rest_go_command "416" "-method" "HEAD" "-query" "partNumber=2" "-objectKey" "$file_name" "-bucketName" "$bucket_name" + assert_success +} + +# tags: curl,HeadObject,partNumber,multipart +@test "REST - HeadObject - valid partNumbers" { + file_mb=8 + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" "$file_mb" + assert_success + read -r bucket_name test_file <<< "$output" + file_bytes=$((file_mb*1024*1024)) + + run split_file_irregular "$TEST_FILE_FOLDER/$test_file" 5242880 + assert_success + read -r part_one part_two <<< "$output" + log 5 "part one: $part_one, part two: $part_two" + + run perform_multipart_upload_rest_variable_parts "$bucket_name" "$test_file" "$part_one" "$part_two" + assert_success + + run get_file_size "$part_one" + assert_success + part_size_one="$output" + + run get_file_size "$part_one" + assert_success + part_size_two="$output" + + run send_rest_go_command_callback "206" "check_header_partial_content_response" "-bucketName" "$bucket_name" "-objectKey" "$test_file" \ + "-method" "HEAD" "-query" "partNumber=1" "--" "1" "$file_bytes" "$part_size_one" + assert_success + + run send_rest_go_command_callback "206" "check_header_partial_content_response" "-bucketName" "$bucket_name" "-objectKey" "$test_file" \ + "-method" "HEAD" "-query" "partNumber=2" "--" "2" "$file_bytes" "$part_size_two" + assert_success +} + +# tags: curl,HeadObject,invalid-query +@test "REST - HeadObject - invalid request header type" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2075" + fi + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run send_rest_go_command "400" "-bucketName" "$bucket_name" "-objectKey" "$file_name" \ + "-method" "HEAD" "-query" "response-invalid=invalid" + assert_success +} + +# tags: curl,HeadObject,response-cache-control,response-content-disposition,response-content-encoding,response-content-language,response-content-type,response-expires +@test "REST - HeadObject - response queries" { + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + query_suffixes=("cache-control" "content-disposition" "content-encoding" "content-language" "content-type" "expires") + + for suffix in "${query_suffixes[@]}"; do + run generate_random_string 20 40 + assert_success + value=$output + log 5 "value: $value" + + run send_rest_go_command_callback "200" "check_for_header_key_and_value" "-bucketName" "$bucket_name" "-objectKey" "$file_name" \ + "-method" "HEAD" "-query" "response-$suffix=$value" "--" "$suffix" "$value" + assert_success + done +} + +@test "REST - quick compare" { + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 5 + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + export QUICK_COMPARE_SIZE=1000000 + run check_quick_compare "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + export QUICK_COMPARE_SIZE=10000000 + run check_quick_compare "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_failure 2 +}