From f81c6a5acbf241ccd292f23a2a88240765f86923 Mon Sep 17 00:00:00 2001 From: upstream Date: Fri, 2 Oct 2026 16:40:16 +0200 Subject: [PATCH] Upstream versity/versitygw state of the touched files before daa25ed6bc --- backend/azure/azure.go | 2727 +++++++ backend/mp-lister.go | 14 +- backend/posix/posix.go | 8034 +++++++++++++++++++++ tests/integration/ListMultipartUploads.go | 6 - tests/integration/group-tests.go | 2 - tests/test_rest_list_multipart_uploads.sh | 3 + 6 files changed, 10766 insertions(+), 20 deletions(-) create mode 100644 backend/azure/azure.go create mode 100644 backend/posix/posix.go diff --git a/backend/azure/azure.go b/backend/azure/azure.go new file mode 100644 index 0000000..df9516e --- /dev/null +++ b/backend/azure/azure.go @@ -0,0 +1,2727 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package azure + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/url" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/google/uuid" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +// When getting container metadata with GetProperties method the sdk returns +// the first letter capital, when accessing the metadata after listing the containers +// it returns the first letter lower +type key string + +const ( + keyAclCapital key = "Acl" + keyAclLower key = "acl" + keyOwnership key = "Ownership" + keyTags key = "Tags" + keyPolicy key = "Policy" + keyCors key = "Cors" + keyWebsite key = "Website" + keyBucketLock key = "Bucketlock" + keyObjRetention key = "Objectretention" + keyObjLegalHold key = "Objectlegalhold" + keyExpires key = "Vgwexpires" + keyWebsiteRedirect key = "Vgwwebsiteredirect" + onameAttr key = "Objname" + onameAttrLower key = "objname" + metaTmpMultipartPrefix key = ".sgwtmp" + "/multipart" + // keyMpZeroBytesParts tracks zero-byte upload parts in the sgwtmp metadata. + // Azure StageBlock rejects Content-Length: 0, so zero-byte parts are stored here. + keyMpZeroBytesParts key = "Zerobytesparts" + // keyMpObjectLock marks, in the sgwtmp metadata, an upload with Object + // Lock parameters, each of whose parts must carry an integrity check. + keyMpObjectLock key = "Mpobjectlock" + // keyMpMetadata stores multipart upload part-offset metadata on the final + // committed blob so that GetObject/HeadObject can serve individual parts + // by part-number. + keyMpMetadata key = "Mpmetadata" + + defaultListingMaxKeys = 1000 +) + +func (key) Table() map[string]struct{} { + return map[string]struct{}{ + "acl": {}, + "ownership": {}, + "tags": {}, + "policy": {}, + "bucketlock": {}, + "website": {}, + "objectretention": {}, + "vgwexpires": {}, + "vgwwebsiteredirect": {}, + "objectlegalhold": {}, + "objname": {}, + ".sgwtmp/multipart": {}, + "mpmetadata": {}, + } +} + +type Azure struct { + backend.BackendUnsupported + + client *azblob.Client + sharedkeyCreds *azblob.SharedKeyCredential + defaultCreds *azidentity.DefaultAzureCredential + serviceURL string + sasToken string + copyObjectThreshold int64 + // copySASVersion overrides the service version used to sign the copy-source + // SAS for server-side copy. Empty means use the SDK default. This exists so + // endpoints that lag the SDK's SAS version (e.g. Azurite) can verify the + // signature; production leaves it unset. + copySASVersion string +} + +var _ backend.Backend = &Azure{} + +func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThreshold int64, copySASVersion string) (*Azure, error) { + url := serviceURL + if serviceURL == "" && accountName != "" { + // if not otherwise specified, use the typical form: + // http(s)://.blob.core.windows.net/ + url = fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + } + + if sasToken != "" { + client, err := azblob.NewClientWithNoCredential(url+"?"+sasToken, nil) + if err != nil { + return nil, fmt.Errorf("init client: %w", err) + } + return &Azure{ + client: client, + serviceURL: serviceURL, + sasToken: sasToken, + copyObjectThreshold: copyObjectThreshold, + copySASVersion: copySASVersion, + }, nil + } + + if accountName == "" { + // if account name not provided, try to get from env var + accountName = os.Getenv("AZURE_CLIENT_ID") + } + + if accountName == "" || accountKey == "" { + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, fmt.Errorf("init default credentials: %w", err) + } + client, err := azblob.NewClient(url, cred, nil) + if err != nil { + return nil, fmt.Errorf("init client: %w", err) + } + return &Azure{ + client: client, + serviceURL: url, + defaultCreds: cred, + copyObjectThreshold: copyObjectThreshold, + copySASVersion: copySASVersion, + }, nil + } + + cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + return nil, fmt.Errorf("init credentials: %w", err) + } + + client, err := azblob.NewClientWithSharedKeyCredential(url, cred, nil) + if err != nil { + return nil, fmt.Errorf("init client: %w", err) + } + + return &Azure{ + client: client, + serviceURL: url, + sharedkeyCreds: cred, + copyObjectThreshold: copyObjectThreshold, + copySASVersion: copySASVersion, + }, nil +} + +func (az *Azure) Shutdown() {} + +func (az *Azure) String() string { + return "Azure Blob Gateway" +} + +func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error { + meta := map[string]*string{ + string(keyAclCapital): backend.GetPtrFromString(encodeBytes(acl)), + string(keyOwnership): backend.GetPtrFromString(encodeBytes([]byte(input.ObjectOwnership))), + } + + acct, ok := ctx.Value("bucket-owner").(auth.Account) + if !ok { + acct = auth.Account{} + } + + if input.ObjectLockEnabledForBucket != nil && *input.ObjectLockEnabledForBucket { + now := time.Now() + defaultLock := auth.BucketLockConfig{ + Enabled: true, + CreatedAt: &now, + } + + defaultLockParsed, err := json.Marshal(defaultLock) + if err != nil { + return fmt.Errorf("parse default bucket lock state: %w", err) + } + + meta[string(keyBucketLock)] = backend.GetPtrFromString(encodeBytes(defaultLockParsed)) + } + + tagging, err := backend.ParseCreateBucketTags(input.CreateBucketConfiguration.Tags) + if err != nil { + return err + } + + if tagging != nil { + tags, err := json.Marshal(tagging) + if err != nil { + return fmt.Errorf("marshal tags: %w", err) + } + + meta[string(keyTags)] = backend.GetPtrFromString(encodeBytes(tags)) + } + + _, err = az.client.CreateContainer(ctx, *input.Bucket, &container.CreateOptions{Metadata: meta}) + if errors.Is(s3err.GetAPIError(s3err.ErrBucketAlreadyExists), azureErrToS3Err(err)) { + aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital)) + if err != nil { + return err + } + + acl, err := auth.ParseACL(aclBytes) + if err != nil { + return err + } + + if acl.Owner == acct.Access { + return s3err.GetBucketErr(s3err.ErrBucketAlreadyOwnedByYou, *input.Bucket) + } + return s3err.GetBucketErr(s3err.ErrBucketAlreadyExists, *input.Bucket) + } + return azureErrToS3Err(err) +} + +func (az *Azure) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { + opts := &service.ListContainersOptions{ + Include: service.ListContainersInclude{ + Metadata: true, + }, + Prefix: &input.Prefix, + } + pager := az.client.NewListContainersPager(opts) + + var buckets []s3response.ListAllMyBucketsEntry + var cToken string + result := s3response.ListAllMyBucketsResult{ + Prefix: input.Prefix, + } + +outer: + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return result, azureErrToS3Err(err) + } + for _, v := range resp.ContainerItems { + // If we've already filled MaxBuckets, there is a next page — set token and stop + if input.MaxBuckets > 0 && int32(len(buckets)) == input.MaxBuckets { + cToken = buckets[len(buckets)-1].Name + break outer + } + // Skip items at or before the continuation token (client-side "start after") + if input.ContinuationToken != "" && *v.Name <= input.ContinuationToken { + continue + } + if input.IsAdmin { + buckets = append(buckets, s3response.ListAllMyBucketsEntry{ + Name: *v.Name, + // TODO: using modification date here instead of creation, is that ok? + CreationDate: *v.Properties.LastModified, + }) + } else { + acl, err := getAclFromMetadata(v.Metadata, keyAclLower) + if err != nil { + return result, err + } + if acl.Owner == input.Owner { + buckets = append(buckets, s3response.ListAllMyBucketsEntry{ + Name: *v.Name, + // TODO: using modification date here instead of creation, is that ok? + CreationDate: *v.Properties.LastModified, + }) + } + } + } + } + + result.Buckets.Bucket = buckets + result.Owner.ID = input.Owner + result.ContinuationToken = cToken + + return result, nil +} + +func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { + _, err := az.getContainerMetaData(ctx, *input.Bucket, "any") + if err != nil { + return nil, azureErrToS3Err(err) + } + + return &s3.HeadBucketOutput{}, nil +} + +func (az *Azure) DeleteBucket(ctx context.Context, bucket string) error { + pager := az.client.NewListBlobsFlatPager(bucket, nil) + + for pager.More() { + pg, err := pager.NextPage(ctx) + if err != nil { + return azureErrToS3Err(err) + } + + for _, item := range pg.Segment.BlobItems { + // ignore temp multipart objects when determining if bucket non-empty + if !strings.HasPrefix(backend.GetStringFromPtr(item.Name), string(metaTmpMultipartPrefix)) { + return s3err.GetAPIError(s3err.ErrBucketNotEmpty) + } + } + } + _, err := az.client.DeleteContainer(ctx, bucket, nil) + return azureErrToS3Err(err) +} + +func (az *Azure) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error { + return az.setContainerMetaData(ctx, bucket, string(keyOwnership), []byte(ownership)) +} + +func (az *Azure) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) { + var ownship types.ObjectOwnership + ownership, err := az.getContainerMetaData(ctx, bucket, string(keyOwnership)) + if err != nil { + return ownship, err + } + if len(ownership) == 0 { + return ownship, s3err.GetBucketErr(s3err.ErrOwnershipControlsNotFound, bucket) + } + + return types.ObjectOwnership(ownership), nil +} + +func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error { + return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership)) +} + +func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) { + tags, err := backend.ParseObjectTags(getString(po.Tagging)) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + err = az.evaluateWritePreconditions(ctx, po.Bucket, po.Key, po.IfMatch, po.IfNoneMatch) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + metadata := parseMetadata(po.Metadata) + + // Store the "Expires" property in the object metadata + if getString(po.Expires) != "" { + if metadata == nil { + metadata = map[string]*string{ + string(keyExpires): po.Expires, + } + } else { + metadata[string(keyExpires)] = po.Expires + } + } + if getString(po.WebsiteRedirectLocation) != "" { + if metadata == nil { + metadata = map[string]*string{ + string(keyWebsiteRedirect): po.WebsiteRedirectLocation, + } + } else { + metadata[string(keyWebsiteRedirect)] = po.WebsiteRedirectLocation + } + } + + opts := &blockblob.UploadStreamOptions{ + Metadata: metadata, + Tags: tags, + } + + opts.HTTPHeaders = &blob.HTTPHeaders{} + opts.HTTPHeaders.BlobContentEncoding = po.ContentEncoding + opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage + opts.HTTPHeaders.BlobContentDisposition = po.ContentDisposition + opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage + opts.HTTPHeaders.BlobCacheControl = po.CacheControl + if strings.HasSuffix(*po.Key, "/") { + // Hardcode "application/x-directory" for direcoty objects + opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DirContentType) + } else { + opts.HTTPHeaders.BlobContentType = po.ContentType + } + + uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, opts) + if err != nil { + return s3response.PutObjectOutput{}, azureErrToS3Err(err) + } + + // Set object legal hold + if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { + err := az.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return s3response.PutObjectOutput{}, err + } + } + + // Set object retention + if po.ObjectLockMode != "" { + retention := types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionMode(po.ObjectLockMode), + RetainUntilDate: po.ObjectLockRetainUntilDate, + } + retParsed, err := json.Marshal(retention) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err) + } + err = az.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", retParsed) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return s3response.PutObjectOutput{}, err + } + } + + return s3response.PutObjectOutput{ + ETag: convertAzureEtag(uploadResp.ETag), + Size: po.ContentLength, + }, nil +} + +func (az *Azure) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error { + if tags == nil { + return az.deleteContainerMetaData(ctx, bucket, string(keyTags)) + } + + tagsJson, err := json.Marshal(tags) + if err != nil { + return err + } + + return az.setContainerMetaData(ctx, bucket, string(keyTags), tagsJson) +} + +func (az *Azure) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) { + tagsJson, err := az.getContainerMetaData(ctx, bucket, string(keyTags)) + if err != nil { + return nil, err + } + + if len(tagsJson) == 0 { + return nil, s3err.GetBucketErr(s3err.ErrBucketTaggingNotFound, bucket) + } + + var tags map[string]string + err = json.Unmarshal(tagsJson, &tags) + if err != nil { + return nil, err + } + + return tags, nil +} + +func (az *Azure) DeleteBucketTagging(ctx context.Context, bucket string) error { + return az.PutBucketTagging(ctx, bucket, nil) +} + +func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + client, err := az.getBlobClient(*input.Bucket, *input.Key) + if err != nil { + return nil, err + } + + resp, err := client.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + if resp.ETag != nil && resp.LastModified != nil { + err = backend.EvaluatePreconditions(convertAzureEtag(resp.ETag), *resp.LastModified, + backend.PreConditions{ + IfMatch: input.IfMatch, + IfNoneMatch: input.IfNoneMatch, + IfModSince: input.IfModifiedSince, + IfUnmodeSince: input.IfUnmodifiedSince, + }) + if err != nil { + return nil, err + } + } + + var objSize int64 + if resp.ContentLength != nil { + objSize = *resp.ContentLength + } + + var opts *azblob.DownloadStreamOptions + var partsCount *int32 + var contentRange *string + + if input.PartNumber != nil { + // Serve a specific part if the object has multipart upload metadata. + // For non-multipart objects (no mp-metadata), partNumber=1 returns the + // full object with no Content-Range; any other partNumber is out of range. + if mpMetaStr, ok := resp.Metadata[string(keyMpMetadata)]; ok && mpMetaStr != nil { + mpMeta, err := backend.UnmarshalMpUploadMetadata([]byte(*mpMetaStr), true) + if err != nil { + return nil, fmt.Errorf("parse object multipart metadata: %w", err) + } + + partNum := *input.PartNumber + totalParts := int32(len(mpMeta.Parts)) + partsCount = &totalParts + if partNum > totalParts { + return nil, s3err.GetInvalidPartNumberRangeErr(totalParts, partNum) + } + + var startOffset int64 + if partNum > 1 { + startOffset = mpMeta.Parts[partNum-2] + } + length := mpMeta.Parts[partNum-1] - startOffset + var objSize int64 + if resp.ContentLength != nil { + objSize = *resp.ContentLength + } + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %d-%d/%d", startOffset, startOffset+length-1, objSize)) + opts = &azblob.DownloadStreamOptions{ + Range: blob.HTTPRange{ + Offset: startOffset, + Count: length, + }, + } + } else if *input.PartNumber > 1 { + return nil, s3err.GetInvalidPartNumberRangeErr(1, *input.PartNumber) + } else { + // partNumber=1 on a non-multipart object: fall through and serve the + // full object without a range (opts remains nil) + if objSize != 0 { + // if object size is 0, the whole object is served, no content range should be set + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes 0-%d/%d", objSize-1, objSize)) + } + } + } else if *input.Range != "" { + offset, count, isValid, err := backend.ParseObjectRange(objSize, *input.Range) + if err != nil { + return nil, err + } + if isValid { + opts = &azblob.DownloadStreamOptions{ + Range: blob.HTTPRange{ + Count: count, + Offset: offset, + }, + } + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %v-%v/%v", offset, offset+count-1, objSize)) + } + } + + blobDownloadResponse, err := az.client.DownloadStream(ctx, *input.Bucket, *input.Key, opts) + if err != nil { + return nil, azureErrToS3Err(err) + } + + var tagcount int32 + if blobDownloadResponse.TagCount != nil { + tagcount = int32(*blobDownloadResponse.TagCount) + } + + return &s3.GetObjectOutput{ + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: blobDownloadResponse.ContentLength, + ContentEncoding: blobDownloadResponse.ContentEncoding, + ContentType: blobDownloadResponse.ContentType, + ContentDisposition: blobDownloadResponse.ContentDisposition, + ContentLanguage: blobDownloadResponse.ContentLanguage, + CacheControl: blobDownloadResponse.CacheControl, + ExpiresString: blobDownloadResponse.Metadata[string(keyExpires)], + WebsiteRedirectLocation: blobDownloadResponse.Metadata[string(keyWebsiteRedirect)], + ETag: backend.GetPtrFromString(convertAzureEtag(blobDownloadResponse.ETag)), + LastModified: blobDownloadResponse.LastModified, + Metadata: parseAndFilterAzMetadata(blobDownloadResponse.Metadata), + TagCount: &tagcount, + ContentRange: contentRange, + Body: blobDownloadResponse.Body, + StorageClass: types.StorageClassStandard, + PartsCount: partsCount, + }, nil +} + +func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + client, err := az.getBlobClient(*input.Bucket, *input.Key) + if err != nil { + return nil, err + } + + resp, err := client.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + if resp.ETag != nil && resp.LastModified != nil { + err = backend.EvaluatePreconditions(convertAzureEtag(resp.ETag), *resp.LastModified, + backend.PreConditions{ + IfMatch: input.IfMatch, + IfNoneMatch: input.IfNoneMatch, + IfModSince: input.IfModifiedSince, + IfUnmodeSince: input.IfUnmodifiedSince, + }) + if err != nil { + return nil, err + } + } + + var size int64 + if resp.ContentLength != nil { + size = *resp.ContentLength + } + + var contentRange *string + var length int64 + var partsCount *int32 + + if input.PartNumber != nil { + // Serve a specific part if the object has multipart upload metadata. + // For non-multipart objects (no mp-metadata), partNumber=1 returns the + // full object with no Content-Range; any other partNumber is out of range. + if mpMetaStr, ok := resp.Metadata[string(keyMpMetadata)]; ok && mpMetaStr != nil { + mpMeta, err := backend.UnmarshalMpUploadMetadata([]byte(*mpMetaStr), true) + if err != nil { + return nil, fmt.Errorf("parse object multipart metadata: %w", err) + } + + partNum := *input.PartNumber + totalParts := int32(len(mpMeta.Parts)) + partsCount = &totalParts + if partNum > totalParts { + return nil, s3err.GetInvalidPartNumberRangeErr(totalParts, partNum) + } + + var startOffset int64 + if partNum > 1 { + startOffset = mpMeta.Parts[partNum-2] + } + length = mpMeta.Parts[partNum-1] - startOffset + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %d-%d/%d", startOffset, startOffset+length-1, size)) + } else if *input.PartNumber > 1 { + return nil, s3err.GetInvalidPartNumberRangeErr(1, *input.PartNumber) + } else { + // partNumber=1 on a non-multipart object: return full object size, + // no Content-Range, no PartsCount. + length = size + if length != 0 { + // if object size is 0, the whole object is served, no content range should be set + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes 0-%d/%d", size-1, size)) + } + } + } else { + startOffset, lgth, isValid, err := backend.ParseObjectRange(size, getString(input.Range)) + if err != nil { + return nil, err + } + length = lgth + if isValid { + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %v-%v/%v", + startOffset, startOffset+length-1, size)) + } + } + + result := &s3.HeadObjectOutput{ + ContentRange: contentRange, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + PartsCount: partsCount, + ContentType: resp.ContentType, + ContentEncoding: resp.ContentEncoding, + ContentLanguage: resp.ContentLanguage, + ContentDisposition: resp.ContentDisposition, + CacheControl: resp.CacheControl, + ExpiresString: resp.Metadata[string(keyExpires)], + WebsiteRedirectLocation: resp.Metadata[string(keyWebsiteRedirect)], + ETag: backend.GetPtrFromString(convertAzureEtag(resp.ETag)), + LastModified: resp.LastModified, + Metadata: parseAndFilterAzMetadata(resp.Metadata), + StorageClass: types.StorageClassStandard, + } + + status, ok := resp.Metadata[string(keyObjLegalHold)] + if ok { + if *status == "1" { + result.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn + } else { + result.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff + } + } + + retention, ok := resp.Metadata[string(keyObjRetention)] + if ok { + var config types.ObjectLockRetention + err := json.Unmarshal([]byte(*retention), &config) + if err == nil { + result.ObjectLockMode = types.ObjectLockMode(config.Mode) + result.ObjectLockRetainUntilDate = config.RetainUntilDate + } + } + + if resp.TagCount != nil { + tagcount := int32(*resp.TagCount) + result.TagCount = &tagcount + } + + return result, nil +} + +func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) { + data, err := az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: input.Bucket, + Key: input.Key, + }) + if err != nil { + return s3response.GetObjectAttributesResponse{}, err + } + + return s3response.GetObjectAttributesResponse{ + ETag: data.ETag, + ObjectSize: data.ContentLength, + StorageClass: data.StorageClass, + LastModified: data.LastModified, + VersionId: data.VersionId, + DeleteMarker: data.DeleteMarker, + }, nil +} + +// azMarkerToken is the decoded form of an S3 continuation token. Azure blob +// markers are opaque values that only Azure may mint, so an S3 key can never be +// used as one. The token therefore carries the Azure marker alongside the last +// key of the previous page: the Azure marker resumes the blob listing close to +// where it stopped, and the last key filters out the entries of that Azure page +// which were already returned. +type azMarkerToken struct { + Prefix string `json:"p,omitempty"` + Delimiter string `json:"d,omitempty"` + Marker string `json:"m,omitempty"` + LastKey string `json:"k,omitempty"` +} + +// azTokenPrefix marks continuation tokens minted by this backend. Anything +// without it is treated as a plain key, which keeps tokens issued by older +// versions and hand-crafted markers working. +const azTokenPrefix = "vgw1." + +func encodeAzMarkerToken(t azMarkerToken) string { + data, err := json.Marshal(t) + if err != nil { + // the struct only holds strings, so this can't fail + return t.LastKey + } + return azTokenPrefix + base64.RawURLEncoding.EncodeToString(data) +} + +// decodeAzMarkerToken returns the Azure marker and the last key encoded in the +// continuation token. The Azure marker is only returned for a token minted for +// the same prefix and delimiter, as the marker resumes the underlying blob +// listing and the last key was computed under that request's delimiter; reusing +// either across a different prefix or delimiter would skip the wrong entries. +func decodeAzMarkerToken(token, prefix, delimiter string) (azureMarker, lastKey string) { + if !strings.HasPrefix(token, azTokenPrefix) { + return "", token + } + + data, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(token, azTokenPrefix)) + if err != nil { + return "", token + } + + var t azMarkerToken + if err := json.Unmarshal(data, &t); err != nil { + return "", token + } + if t.Prefix != prefix || t.Delimiter != delimiter { + return "", t.LastKey + } + + return t.Marker, t.LastKey +} + +// azListingOpts describes a single page of an S3 object listing. +type azListingOpts struct { + prefix string + delimiter string + // marker is the S3 key to start after, exclusive. + marker string + // azureMarker optionally resumes the underlying Azure listing instead of + // walking the prefix from the beginning. + azureMarker string + maxKeys int32 + owner string +} + +// azListingPage is a page of objects and common prefixes gathered from Azure. +type azListingPage struct { + objects []s3response.Object + commonPrefixes []types.CommonPrefix + isTruncated bool + // lastKey is the last object key or common prefix of the page. + lastKey string + // resumeMarker is the Azure marker of the blob page the listing stopped in. + resumeMarker string +} + +// listBlobs collects one S3 listing page from Azure. Azure is always paged with +// its own markers and the delimiter is applied client side, both to match S3 +// semantics and because Azure has no equivalent of "start after". +func (az *Azure) listBlobs(ctx context.Context, client *container.Client, opts azListingOpts) (azListingPage, error) { + var page azListingPage + + pager := client.NewListBlobsHierarchyPager("", &container.ListBlobsHierarchyOptions{ + Prefix: backend.GetPtrFromString(opts.prefix), + Marker: backend.GetPtrFromString(opts.azureMarker), + }) + + // the Azure marker that started the blob page currently being processed + blobPageMarker := opts.azureMarker + cpSet := make(map[string]struct{}) + var pastMax bool + var totalFound int32 + +loop: + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return azListingPage{}, azureErrToS3Err(err) + } + + for _, v := range resp.Segment.BlobItems { + name := backend.GetStringFromPtr(v.Name) + + // Filter out multipart upload blobs + if strings.HasPrefix(name, string(metaTmpMultipartPrefix)) { + continue + } + + // Apply delimiter logic to determine if this blob contributes to + // a common prefix or is a regular object + key := name + isCP := false + if opts.delimiter != "" { + suffix := strings.TrimPrefix(name, opts.prefix) + before, _, found := strings.Cut(suffix, opts.delimiter) + if found { + isCP = true + key = opts.prefix + before + opts.delimiter + } + } + + // Skip everything at or before the marker + if key <= opts.marker { + continue + } + // Deduplicate: multiple blobs can map to the same common prefix + if isCP { + if _, exists := cpSet[key]; exists { + continue + } + } + // A further entry beyond maxKeys means the listing is truncated + if pastMax { + page.isTruncated = true + page.resumeMarker = blobPageMarker + break loop + } + + if isCP { + cpSet[key] = struct{}{} + page.commonPrefixes = append(page.commonPrefixes, types.CommonPrefix{ + Prefix: backend.GetPtrFromString(key), + }) + } else { + page.objects = append(page.objects, s3response.Object{ + ETag: backend.GetPtrFromString(convertAzureEtag(v.Properties.ETag)), + Key: v.Name, + LastModified: v.Properties.LastModified, + Size: v.Properties.ContentLength, + StorageClass: types.ObjectStorageClassStandard, + Owner: &types.Owner{ + ID: &opts.owner, + }, + }) + } + + page.lastKey = key + totalFound++ + if totalFound == opts.maxKeys { + pastMax = true + } + } + + blobPageMarker = backend.GetStringFromPtr(resp.NextMarker) + } + + return page, nil +} + +func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { + // Retrieve the bucket acl to get the bucket owner + // All the objects in the bucket are owner by the bucket owner + aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital)) + if err != nil { + return s3response.ListObjectsResult{}, azureErrToS3Err(err) + } + + acl, err := auth.ParseACL(aclBytes) + if err != nil { + return s3response.ListObjectsResult{}, err + } + + client, err := az.getContainerClient(*input.Bucket) + if err != nil { + return s3response.ListObjectsResult{}, nil + } + + var maxKeys int32 = defaultListingMaxKeys + if input.MaxKeys != nil { + maxKeys = *input.MaxKeys + } + + delimiter := backend.GetStringFromPtr(input.Delimiter) + prefix := backend.GetStringFromPtr(input.Prefix) + effectiveMarker := backend.GetStringFromPtr(input.Marker) + + if maxKeys == 0 { + isFalse := false + return s3response.ListObjectsResult{ + IsTruncated: &isFalse, + MaxKeys: &maxKeys, + Name: input.Bucket, + Prefix: backend.GetPtrFromString(prefix), + Marker: backend.GetPtrFromString(effectiveMarker), + Delimiter: backend.GetPtrFromString(delimiter), + CommonPrefixes: []types.CommonPrefix{}, + }, nil + } + + // The S3 marker is an object key, which Azure rejects as a blob listing + // marker, so it is applied client side. ListObjects has no opaque token to + // carry an Azure marker in, hence every page walks the prefix from the + // start; use ListObjectsV2 to page large listings efficiently. + page, err := az.listBlobs(ctx, client, azListingOpts{ + prefix: prefix, + delimiter: delimiter, + marker: effectiveMarker, + maxKeys: maxKeys, + owner: acl.Owner, + }) + if err != nil { + return s3response.ListObjectsResult{}, err + } + + var nextMarker string + if page.isTruncated { + nextMarker = page.lastKey + } + + return s3response.ListObjectsResult{ + Contents: page.objects, + Marker: backend.GetPtrFromString(effectiveMarker), + MaxKeys: &maxKeys, + Name: input.Bucket, + NextMarker: backend.GetPtrFromString(nextMarker), + Prefix: backend.GetPtrFromString(prefix), + IsTruncated: &page.isTruncated, + Delimiter: backend.GetPtrFromString(delimiter), + CommonPrefixes: page.commonPrefixes, + }, nil +} + +func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { + // Retrieve the bucket acl to get the bucket owner + // All the objects in the bucket are owner by the bucket owner + aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital)) + if err != nil { + return s3response.ListObjectsV2Result{}, azureErrToS3Err(err) + } + + acl, err := auth.ParseACL(aclBytes) + if err != nil { + return s3response.ListObjectsV2Result{}, err + } + + client, err := az.getContainerClient(*input.Bucket) + if err != nil { + return s3response.ListObjectsV2Result{}, nil + } + + var maxKeys int32 = defaultListingMaxKeys + if input.MaxKeys != nil { + maxKeys = *input.MaxKeys + } + + delimiter := backend.GetStringFromPtr(input.Delimiter) + prefix := backend.GetStringFromPtr(input.Prefix) + startAfterVal := backend.GetStringFromPtr(input.StartAfter) + continuationTokenVal := backend.GetStringFromPtr(input.ContinuationToken) + + azureMarker, tokenKey := decodeAzMarkerToken(continuationTokenVal, prefix, delimiter) + + // Take the lexicographically larger of startAfter and the continuation + // token key so listing starts strictly after both constraints. + effectiveMarker := startAfterVal + if tokenKey > effectiveMarker { + effectiveMarker = tokenKey + } + + if maxKeys == 0 { + isFalse := false + return s3response.ListObjectsV2Result{ + IsTruncated: &isFalse, + MaxKeys: &maxKeys, + Name: input.Bucket, + Prefix: backend.GetPtrFromString(prefix), + ContinuationToken: backend.GetPtrFromString(continuationTokenVal), + Delimiter: backend.GetPtrFromString(delimiter), + StartAfter: backend.GetPtrFromString(startAfterVal), + CommonPrefixes: []types.CommonPrefix{}, + }, nil + } + + // The S3 marker is an object key, which Azure rejects as a blob listing + // marker, so it is applied client side while Azure is paged with the marker + // carried in the continuation token. + page, err := az.listBlobs(ctx, client, azListingOpts{ + prefix: prefix, + delimiter: delimiter, + marker: effectiveMarker, + azureMarker: azureMarker, + maxKeys: maxKeys, + owner: acl.Owner, + }) + if err != nil { + return s3response.ListObjectsV2Result{}, err + } + + var nextToken string + if page.isTruncated { + nextToken = encodeAzMarkerToken(azMarkerToken{ + Prefix: prefix, + Delimiter: delimiter, + Marker: page.resumeMarker, + LastKey: page.lastKey, + }) + } + + keyCount := int32(len(page.objects) + len(page.commonPrefixes)) + + return s3response.ListObjectsV2Result{ + Contents: page.objects, + ContinuationToken: backend.GetPtrFromString(continuationTokenVal), + KeyCount: &keyCount, + MaxKeys: &maxKeys, + Name: input.Bucket, + NextContinuationToken: backend.GetPtrFromString(nextToken), + Prefix: backend.GetPtrFromString(prefix), + IsTruncated: &page.isTruncated, + Delimiter: backend.GetPtrFromString(delimiter), + CommonPrefixes: page.commonPrefixes, + StartAfter: backend.GetPtrFromString(startAfterVal), + }, nil +} + +func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + if input.IfMatch != nil || input.IfMatchLastModifiedTime != nil || input.IfMatchSize != nil { + // evaluate the preconditions before deleting the object + props, err := az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: input.Bucket, + Key: input.Key, + }) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + // if object doesn't exist, skip preconditions + // if unexpected error shows up, return the error + return nil, err + } + if err == nil { + var etag string + if props.ETag != nil { + etag = *props.ETag + } + var lastMod time.Time + if props.LastModified != nil { + lastMod = *props.LastModified + } + var size int64 + if props.ContentLength != nil { + size = *props.ContentLength + } + err := backend.EvaluateObjectDeletePreconditions(etag, lastMod, size, + backend.ObjectDeletePreconditions{ + IfMatch: input.IfMatch, + IfMatchLastModTime: input.IfMatchLastModifiedTime, + IfMatchSize: input.IfMatchSize, + }) + if err != nil { + return nil, err + } + } + } + + _, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) + if err != nil { + azerr, ok := err.(*azcore.ResponseError) + if ok && azerr.StatusCode == 404 { + // if the object does not exist, S3 returns success + return &s3.DeleteObjectOutput{}, nil + } + } + return &s3.DeleteObjectOutput{}, azureErrToS3Err(err) +} + +func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + delResult, errs := []types.DeletedObject{}, []types.Error{} + for _, obj := range input.Delete.Objects { + _, err := az.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: input.Bucket, + Key: obj.Key, + }) + if err == nil { + delResult = append(delResult, types.DeletedObject{Key: obj.Key}) + } else { + errs = append(errs, s3err.ObjectDeleteError(obj.Key, obj.VersionId, err)) + } + } + + return s3response.DeleteResult{ + Deleted: delResult, + Error: errs, + }, nil +} + +func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) { + dstClient, err := az.getBlobClient(*input.Bucket, *input.Key) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + if input.ExpectedSourceBucketOwner != nil && *input.ExpectedSourceBucketOwner != "" { + aclData, err := az.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket}) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + srcAcl, err := auth.ParseACL(aclData) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + if srcAcl.Owner != *input.ExpectedSourceBucketOwner { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrAccessDenied) + } + } + + if !areNils(input.CopySourceIfMatch, input.CopySourceIfNoneMatch) || !areNils(input.CopySourceIfModifiedSince, input.CopySourceIfUnmodifiedSince) { + _, err = az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + IfMatch: input.CopySourceIfMatch, + IfNoneMatch: input.CopySourceIfNoneMatch, + IfModifiedSince: input.CopySourceIfModifiedSince, + IfUnmodifiedSince: input.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + } + + if srcBucket == *input.Bucket && srcObj == *input.Key { + if input.MetadataDirective != types.MetadataDirectiveReplace { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) + } + + resp, err := dstClient.GetProperties(ctx, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + + if resp.ContentLength != nil && *resp.ContentLength > az.copyObjectThreshold { + return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold) + } + + // Set object meta http headers + res, err := dstClient.SetHTTPHeaders(ctx, blob.HTTPHeaders{ + BlobCacheControl: input.CacheControl, + BlobContentDisposition: input.ContentDisposition, + BlobContentEncoding: input.ContentEncoding, + BlobContentLanguage: input.ContentLanguage, + BlobContentType: input.ContentType, + }, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + + meta := input.Metadata + if meta == nil { + meta = make(map[string]string) + } + + // Embed "Expires" in object metadata + if getString(input.Expires) != "" { + meta[string(keyExpires)] = *input.Expires + } + if getString(input.WebsiteRedirectLocation) != "" { + meta[string(keyWebsiteRedirect)] = *input.WebsiteRedirectLocation + } + // Set object metadata + _, err = dstClient.SetMetadata(ctx, parseMetadata(meta), nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + + // Set object legal hold + if input.ObjectLockLegalHoldStatus != "" { + err = az.PutObjectLegalHold(ctx, *input.Bucket, *input.Key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + } + // Set object retention + if input.ObjectLockMode != "" && input.ObjectLockRetainUntilDate != nil { + retention := s3response.PutObjectRetentionInput{ + Mode: types.ObjectLockRetentionMode(input.ObjectLockMode), + RetainUntilDate: s3response.AmzDate{ + Time: *input.ObjectLockRetainUntilDate, + }, + } + + retParsed, err := json.Marshal(retention) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("parse object retention: %w", err) + } + err = az.PutObjectRetention(ctx, *input.Bucket, *input.Key, "", retParsed) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + } + + // Set object Tagging, if tagging directive is "REPLACE" + if input.TaggingDirective == types.TaggingDirectiveReplace { + tags, err := backend.ParseObjectTags(getString(input.Tagging)) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + _, err = dstClient.SetTags(ctx, tags, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + } + + return s3response.CopyObjectOutput{ + CopyObjectResult: &s3response.CopyObjectResult{ + LastModified: res.LastModified, + ETag: backend.GetPtrFromString(convertAzureEtag(res.ETag)), + }, + }, nil + } + + // Cross-object copy: try server-side copy first, fall back to download+reupload. + srcClient, err := az.getBlobClient(srcBucket, srcObj) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + srcProps, err := srcClient.GetProperties(ctx, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + + if srcProps.ContentLength != nil && *srcProps.ContentLength > az.copyObjectThreshold { + return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold) + } + + out, err := az.serverSideCopyObject(ctx, input, srcBucket, srcObj, dstClient, srcClient, &srcProps) + if err == nil { + return out, nil + } + if !errors.Is(err, errServerSideCopyFallback) { + return s3response.CopyObjectOutput{}, err + } + + debuglogger.Logf("falling back to download+reupload (%q/%q -> %q/%q): %v", + srcBucket, srcObj, *input.Bucket, *input.Key, err) + + // Fallback: download and re-upload through the gateway. + downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + defer downloadResp.Body.Close() + + if downloadResp.ContentLength != nil && *downloadResp.ContentLength > az.copyObjectThreshold { + return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold) + } + + pInput := s3response.PutObjectInput{ + Body: downloadResp.Body, + Bucket: input.Bucket, + Key: input.Key, + ContentLength: downloadResp.ContentLength, + ContentType: input.ContentType, + ContentEncoding: input.ContentEncoding, + ContentDisposition: input.ContentDisposition, + ContentLanguage: input.ContentLanguage, + CacheControl: input.CacheControl, + Expires: input.Expires, + WebsiteRedirectLocation: input.WebsiteRedirectLocation, + Metadata: input.Metadata, + ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate, + ObjectLockMode: input.ObjectLockMode, + ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus, + } + + if input.MetadataDirective == types.MetadataDirectiveCopy { + // Expires is in downloadResp.Metadata + pInput.Expires = nil + pInput.CacheControl = downloadResp.CacheControl + pInput.ContentDisposition = downloadResp.ContentDisposition + pInput.ContentEncoding = downloadResp.ContentEncoding + pInput.ContentLanguage = downloadResp.ContentLanguage + pInput.ContentType = downloadResp.ContentType + pInput.Metadata = parseAzMetadata(downloadResp.Metadata) + delete(pInput.Metadata, string(keyWebsiteRedirect)) + } + + if input.TaggingDirective == types.TaggingDirectiveReplace { + pInput.Tagging = input.Tagging + } + + // Create the destination object + resp, err := az.PutObject(ctx, pInput) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + // Copy the object tagging, if tagging directive is "COPY" + if input.TaggingDirective == types.TaggingDirectiveCopy { + srcClient, err := az.getBlobClient(srcBucket, srcObj) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + res, err := srcClient.GetTags(ctx, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + + _, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + } + + return s3response.CopyObjectOutput{ + CopyObjectResult: &s3response.CopyObjectResult{ + ETag: &resp.ETag, + }, + }, nil +} + +func (az *Azure) PutObjectTagging(ctx context.Context, bucket, object, _ string, tags map[string]string) error { + client, err := az.getBlobClient(bucket, object) + if err != nil { + return err + } + + _, err = client.SetTags(ctx, tags, nil) + if err != nil { + return azureErrToS3Err(err) + } + + return nil +} + +func (az *Azure) GetObjectTagging(ctx context.Context, bucket, object, _ string) (map[string]string, error) { + client, err := az.getBlobClient(bucket, object) + if err != nil { + return nil, err + } + + tags, err := client.GetTags(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + return parseAzTags(tags.BlobTagSet), nil +} + +func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object, _ string) error { + client, err := az.getBlobClient(bucket, object) + if err != nil { + return err + } + + _, err = client.SetTags(ctx, map[string]string{}, nil) + if err != nil { + return azureErrToS3Err(err) + } + + return nil +} + +func (az *Azure) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { + bucketLock, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyBucketLock)) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err) + } + + var bucketLockConfig auth.BucketLockConfig + if len(bucketLock) != 0 { + err = json.Unmarshal(bucketLock, &bucketLockConfig) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, + fmt.Errorf("parse bucket lock config: %w", err) + } + } + + explicitLock := input.ObjectLockLegalHoldStatus != "" || input.ObjectLockMode != "" + if explicitLock && !bucketLockConfig.Enabled { + return s3response.InitiateMultipartUploadResult{}, + s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + + // The upload carries Object Lock parameters when it sets its own, or the + // bucket has a default retention rule as of now: a rule set or removed + // later doesn't change what the upload's parts need. + lockParams := explicitLock || (bucketLockConfig.Enabled && bucketLockConfig.DefaultRetention != nil) + + meta := parseMetadata(input.Metadata) + meta[string(onameAttr)] = input.Key + + if getString(input.Expires) != "" { + meta[string(keyExpires)] = input.Expires + } + if getString(input.WebsiteRedirectLocation) != "" { + meta[string(keyWebsiteRedirect)] = input.WebsiteRedirectLocation + } + + // parse object tags + tags, err := backend.ParseObjectTags(getString(input.Tagging)) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, err + } + + // set blob legal hold status in metadata + if input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { + meta[string(keyObjLegalHold)] = backend.GetPtrFromString("1") + } + + // set blob retention date + if input.ObjectLockMode != "" { + retention := types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionMode(input.ObjectLockMode), + RetainUntilDate: input.ObjectLockRetainUntilDate, + } + retParsed, err := json.Marshal(retention) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err) + } + meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retParsed)) + } + + // mark an upload with Object Lock parameters, so that each of its parts + // is required to carry an integrity check + if lockParams { + meta[string(keyMpObjectLock)] = backend.GetPtrFromString("1") + } + + uploadId := uuid.New().String() + + tmpPath := createMetaTmpPath(*input.Key, uploadId) + + opts := &blockblob.UploadBufferOptions{ + Metadata: meta, + Tags: tags, + HTTPHeaders: &blob.HTTPHeaders{ + BlobContentType: input.ContentType, + BlobContentEncoding: input.ContentEncoding, + BlobCacheControl: input.CacheControl, + BlobContentDisposition: input.ContentDisposition, + BlobContentLanguage: input.ContentLanguage, + }, + } + + // Create and empty blob in .sgwtmp/multipart// + // The blob indicates multipart upload initialization and holds the mp metadata + // e.g tagging, content-type, metadata, object lock status ... + _, err = az.client.UploadBuffer(ctx, *input.Bucket, tmpPath, []byte{}, opts) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err) + } + + return s3response.InitiateMultipartUploadResult{ + Bucket: *input.Bucket, + Key: *input.Key, + UploadId: uploadId, + }, nil +} + +// Each part is translated into an uncommitted block in a newly created blob in staging area +func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + mpMeta, err := az.getMpMetadata(ctx, *input.Bucket, *input.Key, *input.UploadId) + if err != nil { + return nil, err + } + + // A part of an upload with Object Lock parameters needs an integrity + // check of its body + if mpMeta[string(keyMpObjectLock)] != nil && !backend.UploadPartHasIntegrityCheck(input) { + return nil, s3err.GetAPIError(s3err.ErrObjectLockPartChecksumRequired) + } + + // TODO: request streamable version of StageBlock() + // (*blockblob.Client).StageBlock does not have a streamable + // version of this function at this time, so we need to cache + // the body in memory to create an io.ReadSeekCloser + rdr, err := getReadSeekCloser(input.Body) + if err != nil { + return nil, err + } + + // block id serves as etag here + etag := blockIDInt32ToBase64(*input.PartNumber) + quotedETag := quoteETag(etag) + + // Azure StageBlock rejects Content-Length: 0 as an invalid header value. + // Track zero-byte parts in the sgwtmp metadata instead of staging them. + size, err := rdr.Seek(0, io.SeekEnd) + if err != nil { + return nil, err + } + if _, err = rdr.Seek(0, io.SeekStart); err != nil { + return nil, err + } + if size == 0 { + err := az.trackZeroBytePart(ctx, *input.Bucket, *input.Key, *input.UploadId, *input.PartNumber) + if err != nil { + return nil, err + } + return &s3.UploadPartOutput{ETag: "edETag}, nil + } + + client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) + if err != nil { + return nil, err + } + + _, err = client.StageBlock(ctx, etag, rdr, nil) + if err != nil { + return nil, parseMpError(err) + } + + return &s3.UploadPartOutput{ + ETag: "edETag, + }, nil +} + +func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { + client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) + if err != nil { + return s3response.CopyPartResult{}, err + } + + err = az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId) + if err != nil { + return s3response.CopyPartResult{}, err + } + + eTag := blockIDInt32ToBase64(*input.PartNumber) + //TODO: handle block copy by range + //TODO: the action returns not implemented on azurite, maybe in production this will work? + _, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil) + if err != nil { + return s3response.CopyPartResult{}, parseMpError(err) + } + + // The staged block id serves as the part ETag, returned quoted to match + // the S3 contract and the form UploadPart/ListParts emit. + quotedETag := quoteETag(eTag) + return s3response.CopyPartResult{ + ETag: "edETag, + LastModified: time.Now(), + }, nil +} + +// Lists all uncommitted parts from the blob +func (az *Azure) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) { + err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId) + if err != nil { + return s3response.ListPartsResult{}, err + } + client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) + if err != nil { + return s3response.ListPartsResult{}, nil + } + + var partNumberMarker int + var nextPartNumberMarker int + var maxParts int32 = math.MaxInt32 + var isTruncated bool + + if *input.PartNumberMarker != "" { + partNumberMarker, err = strconv.Atoi(*input.PartNumberMarker) + if err != nil { + return s3response.ListPartsResult{}, + s3err.GetInvalidArgMaxLimiter("part-number-marker", *input.PartNumberMarker) + } + } + if input.MaxParts != nil { + maxParts = *input.MaxParts + } + + resp, blockListErr := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil) + if blockListErr != nil { + if !errors.Is(azureErrToS3Err(blockListErr), s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return s3response.ListPartsResult{}, blockListErr + } + // NoSuchKey means no blocks have been staged yet (possible if only zero-byte + // parts exist). Continue so we can still return those from metadata. + } + + parts := []s3response.Part{} + if blockListErr == nil { + for _, el := range resp.UncommittedBlocks { + partNumber, err := decodeBlockId(*el.Name) + if err != nil { + return s3response.ListPartsResult{}, err + } + if partNumberMarker >= partNumber { + continue + } + parts = append(parts, s3response.Part{ + Size: *el.Size, + ETag: quoteETag(*el.Name), + PartNumber: partNumber, + LastModified: time.Now(), + }) + } + } + + // Merge in zero-byte parts tracked in the sgwtmp metadata. + zbParts, _ := az.getZeroByteParts(ctx, *input.Bucket, *input.Key, *input.UploadId) + for _, zbPartNum := range zbParts { + if partNumberMarker >= int(zbPartNum) { + continue + } + parts = append(parts, s3response.Part{ + Size: 0, + ETag: quoteETag(blockIDInt32ToBase64(zbPartNum)), + PartNumber: int(zbPartNum), + LastModified: time.Now(), + }) + } + + // Sort by part number and apply maxParts limit. + sort.Slice(parts, func(i, j int) bool { + return parts[i].PartNumber < parts[j].PartNumber + }) + if int32(len(parts)) > maxParts { + parts = parts[:maxParts] + nextPartNumberMarker = parts[len(parts)-1].PartNumber + isTruncated = true + } + + return s3response.ListPartsResult{ + Bucket: *input.Bucket, + Key: *input.Key, + Parts: parts, + NextPartNumberMarker: nextPartNumberMarker, + PartNumberMarker: partNumberMarker, + IsTruncated: isTruncated, + MaxParts: int(maxParts), + StorageClass: types.StorageClassStandard, + }, nil +} + +// Lists all the multipart uploads initiated with .sgwtmp/multipart prefix +func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + var bucket string + if input.Bucket != nil { + bucket = *input.Bucket + } + + client, err := az.getContainerClient(bucket) + if err != nil { + return s3response.ListMultipartUploadsResult{}, err + } + + var delimiter string + if input.Delimiter != nil { + delimiter = *input.Delimiter + } + var prefix string + if input.Prefix != nil { + prefix = *input.Prefix + } + var keyMarker string + if input.KeyMarker != nil { + keyMarker = *input.KeyMarker + } + var uploadIDMarker string + if input.UploadIdMarker != nil { + uploadIDMarker = *input.UploadIdMarker + } + maxUploads := int(*input.MaxUploads) + + mpPrefix := string(metaTmpMultipartPrefix) + pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Include: container.ListBlobsInclude{Metadata: true}, + Prefix: &mpPrefix, + }) + + uploads := []s3response.Upload{} + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return s3response.ListMultipartUploadsResult{}, azureErrToS3Err(err) + } + for _, el := range resp.Segment.BlobItems { + key, ok := el.Metadata[string(onameAttrLower)] + if !ok { + continue + } + if keyMarker != "" && *key <= keyMarker { + continue + } + if prefix != "" && !strings.HasPrefix(*key, prefix) { + continue + } + + path := filepath.Clean(*el.Name) + parts := strings.Split(path, "/") + uploadId := parts[2] + + uploads = append(uploads, s3response.Upload{ + Key: *key, + Initiated: *el.Properties.CreationTime, + UploadID: uploadId, + StorageClass: types.StorageClassStandard, + }) + } + } + + // Sort once: Key asc, Initiated asc, UploadID asc + // UploadID is a tie-breaker for uploads that share both key and + // initiated time, since azure reports the same identical creation + // time for uploads created back to back. + sort.Slice(uploads, func(i, j int) bool { + if uploads[i].Key != uploads[j].Key { + return uploads[i].Key < uploads[j].Key + } + if !uploads[i].Initiated.Equal(uploads[j].Initiated) { + return uploads[i].Initiated.Before(uploads[j].Initiated) + } + return uploads[i].UploadID < uploads[j].UploadID + }) + + result, err := backend.ListMultipartUploads(uploads, prefix, delimiter, keyMarker, uploadIDMarker, maxUploads) + if err != nil { + return s3response.ListMultipartUploadsResult{}, err + } + + return s3response.ListMultipartUploadsResult{ + Bucket: bucket, + Delimiter: delimiter, + KeyMarker: keyMarker, + MaxUploads: maxUploads, + Prefix: prefix, + NextKeyMarker: result.NextKeyMarker, + NextUploadIDMarker: result.NextUploadIDMarker, + UploadIDMarker: uploadIDMarker, + IsTruncated: result.IsTruncated, + Uploads: result.Uploads, + CommonPrefixes: result.CommonPrefixes, + }, nil +} + +// Deletes the block blob with committed/uncommitted blocks +// Cleans up the initiated multipart upload in .sgwtmp namespace +func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { + tmpPath := createMetaTmpPath(*input.Key, *input.UploadId) + + if input.IfMatchInitiatedTime != nil { + client, err := az.getBlobClient(*input.Bucket, tmpPath) + if err != nil { + return err + } + + resp, err := client.GetProperties(ctx, nil) + if err != nil { + return azureErrToS3Err(err) + } + + if resp.LastModified != nil && resp.LastModified.Unix() != input.IfMatchInitiatedTime.Unix() { + return s3err.GetPreconditionFailedErr(s3err.ConditionIfMatchInitiatedTime) + } + } + _, err := az.client.DeleteBlob(ctx, *input.Bucket, tmpPath, nil) + if err != nil { + return parseMpError(err) + } + + // Cleanup the uploaded parts + _, err = az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) + if err != nil { + err = azureErrToS3Err(err) + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return nil + } + + return err + } + + return nil +} + +// Commits all the uncommitted blocks inside the block blob +// And moves the block blob from staging area into the blobs list. +// Copeies the multipart metadata from .sgwtmp namespace into the newly created blob +// Deletes the multipart upload 'blob' from .sgwtmp namespace +// It indicates the end of the multipart upload +func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { + var res s3response.CompleteMultipartUploadResult + + err := az.evaluateWritePreconditions(ctx, input.Bucket, input.Key, input.IfMatch, input.IfNoneMatch) + if err != nil { + return s3response.CompleteMultipartUploadResult{}, "", err + } + + tmpPath := createMetaTmpPath(*input.Key, *input.UploadId) + blobClient, err := az.getBlobClient(*input.Bucket, tmpPath) + if err != nil { + return res, "", err + } + + props, err := blobClient.GetProperties(ctx, nil) + if err != nil { + mpErr := parseMpError(err) + // If the tmp blob is already gone, the upload may have already been + // completed. Check the final object's mp-metadata and return success + // if the upload IDs match. + if errors.Is(mpErr, s3err.GetAPIError(s3err.ErrNoSuchUpload)) { + finalClient, clientErr := az.getBlobClient(*input.Bucket, *input.Key) + if clientErr == nil { + finalProps, propErr := finalClient.GetProperties(ctx, nil) + if propErr == nil { + if mpMetaStr, ok := finalProps.Metadata[string(keyMpMetadata)]; ok && mpMetaStr != nil { + mpMeta, metaErr := backend.UnmarshalMpUploadMetadata([]byte(*mpMetaStr), true) + if metaErr == nil && mpMeta.UploadID == *input.UploadId { + return s3response.CompleteMultipartUploadResult{ + Bucket: input.Bucket, + Key: input.Key, + ETag: backend.GetPtrFromString(convertAzureEtag(finalProps.ETag)), + }, "", nil + } + } + } + } + } + return res, "", mpErr + } + tags, err := blobClient.GetTags(ctx, nil) + if err != nil { + return res, "", parseMpError(err) + } + + client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) + if err != nil { + return res, "", err + } + blockIds := []string{} + + blockList, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil) + if err != nil { + if !errors.Is(azureErrToS3Err(err), s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return res, "", azureErrToS3Err(err) + } + // NoSuchKey: no blocks staged; only zero-byte parts may exist. + } + + // Collect zero-byte parts tracked in the sgwtmp metadata. + zbParts, err := az.getZeroByteParts(ctx, *input.Bucket, *input.Key, *input.UploadId) + if err != nil { + return res, "", err + } + zbPartsMap := make(map[int32]bool, len(zbParts)) + for _, p := range zbParts { + zbPartsMap[p] = true + } + + if len(blockList.UncommittedBlocks)+len(zbParts) != len(input.MultipartUpload.Parts) { + return res, "", s3err.GetInvalidPartErr(*input.UploadId, 0, "") + } + + uncommittedBlocks := map[int32]*blockblob.Block{} + for _, el := range blockList.UncommittedBlocks { + ptNumber, err := decodeBlockId(backend.GetStringFromPtr(el.Name)) + if err != nil { + return res, "", fmt.Errorf("invalid block name: %w", err) + } + + uncommittedBlocks[int32(ptNumber)] = el + } + + // The initial value is the lower limit of partNumber: 0 + var totalSize int64 + var partNumber int32 + // partSizes[i] = cumulative byte offset after part i+1 (see backend.MpUploadMetadata) + var partSizes []int64 + last := len(input.MultipartUpload.Parts) - 1 + for i, part := range input.MultipartUpload.Parts { + if part.PartNumber == nil { + return res, "", s3err.GetAPIError(s3err.ErrMalformedXML) + } + if part.ETag == nil { + return res, "", s3err.GetAPIError(s3err.ErrMalformedXML) + } + // Clients may submit the part ETag in quoted form (as returned by + // UploadPart/ListParts) or raw; normalize before comparing to the + // raw Azure block id. + clientETag := getString(backend.TrimEtag(part.ETag)) + if *part.PartNumber < 1 { + return res, "", s3err.GetInvalidArgumentErr(s3err.InvalidArgCompleteMpPartNumber, fmt.Sprint(*part.PartNumber)) + } + if *part.PartNumber <= partNumber { + return res, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder) + } + partNumber = *part.PartNumber + + block, ok := uncommittedBlocks[*part.PartNumber] + if !ok { + // Check if this is a tracked zero-byte part. + if zbPartsMap[*part.PartNumber] { + expectedETag := blockIDInt32ToBase64(*part.PartNumber) + if clientETag != expectedETag { + return res, "", s3err.GetInvalidPartErr(*input.UploadId, *part.PartNumber, expectedETag) + } + // Non-last zero-byte parts violate the minimum part size. + if i < last { + return res, "", s3err.GetEntityTooSmallErr(0, backend.MinPartSize) + } + // Zero-byte parts contribute no data; skip adding to blockIds. + partSizes = append(partSizes, totalSize) + continue + } + return res, "", s3err.GetInvalidPartErr(*input.UploadId, *part.PartNumber, "") + } + + if clientETag != *block.Name { + return res, "", s3err.GetInvalidPartErr(*input.UploadId, *part.PartNumber, clientETag) + } + // all parts except the last need to be greater, than + // the minimum allowed size (5 Mib) + if i < last && *block.Size < backend.MinPartSize { + return res, "", s3err.GetEntityTooSmallErr(*block.Size, backend.MinPartSize) + } + totalSize += *block.Size + partSizes = append(partSizes, totalSize) + blockIds = append(blockIds, *block.Name) + } + + if input.MpuObjectSize != nil && totalSize != *input.MpuObjectSize { + return res, "", + s3err.GetIncorrectMpObjectSizeErr(totalSize, *input.MpuObjectSize) + } + + // Remove internal tracking keys from metadata before storing on the final blob. + delete(props.Metadata, string(keyMpZeroBytesParts)) + delete(props.Metadata, string(keyMpObjectLock)) + + // Serialize multipart metadata so GetObject/HeadObject can serve by part-number. + mpMeta := backend.MpUploadMetadata{UploadID: *input.UploadId, Parts: partSizes} + mpMetaBytes, err := backend.MarshalMpUploadMetadata(mpMeta, true) + if err != nil { + return res, "", fmt.Errorf("marshal mp metadata: %w", err) + } + mpMetaStr := string(mpMetaBytes) + if props.Metadata == nil { + props.Metadata = map[string]*string{} + } + props.Metadata[string(keyMpMetadata)] = &mpMetaStr + + opts := &blockblob.CommitBlockListOptions{ + Metadata: props.Metadata, + Tags: parseAzTags(tags.BlobTagSet), + } + opts.HTTPHeaders = &blob.HTTPHeaders{ + BlobContentType: props.ContentType, + BlobContentEncoding: props.ContentEncoding, + BlobContentDisposition: props.ContentDisposition, + BlobContentLanguage: props.ContentLanguage, + BlobCacheControl: props.CacheControl, + } + + resp, err := client.CommitBlockList(ctx, blockIds, opts) + if err != nil { + return res, "", parseMpError(err) + } + + // cleanup the multipart upload + _, err = blobClient.Delete(ctx, nil) + if err != nil { + return res, "", parseMpError(err) + } + + return s3response.CompleteMultipartUploadResult{ + Bucket: input.Bucket, + Key: input.Key, + ETag: backend.GetPtrFromString(convertAzureEtag(resp.ETag)), + }, "", nil +} + +func (az *Azure) PutBucketAcl(ctx context.Context, bucket string, data []byte) error { + return az.setContainerMetaData(ctx, bucket, string(keyAclCapital), data) +} + +func (az *Azure) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) { + return az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital)) +} + +func (az *Azure) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error { + if policy == nil { + return az.deleteContainerMetaData(ctx, bucket, string(keyPolicy)) + } + + return az.setContainerMetaData(ctx, bucket, string(keyPolicy), policy) +} + +func (az *Azure) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) { + p, err := az.getContainerMetaData(ctx, bucket, string(keyPolicy)) + if err != nil { + return nil, err + } + if len(p) == 0 { + return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucketPolicy, bucket) + } + return p, nil +} + +func (az *Azure) DeleteBucketPolicy(ctx context.Context, bucket string) error { + return az.PutBucketPolicy(ctx, bucket, nil) +} + +func (az *Azure) PutBucketCors(ctx context.Context, bucket string, cors []byte) error { + if cors == nil { + return az.deleteContainerMetaData(ctx, bucket, string(keyCors)) + } + + return az.setContainerMetaData(ctx, bucket, string(keyCors), cors) +} + +func (az *Azure) GetBucketCors(ctx context.Context, bucket string) ([]byte, error) { + p, err := az.getContainerMetaData(ctx, bucket, string(keyCors)) + if err != nil { + return nil, err + } + if len(p) == 0 { + return nil, s3err.GetBucketErr(s3err.ErrNoSuchCORSConfiguration, bucket) + } + return p, nil +} + +func (az *Azure) DeleteBucketCors(ctx context.Context, bucket string) error { + return az.PutBucketCors(ctx, bucket, nil) +} + +func (az *Azure) PutBucketWebsite(ctx context.Context, bucket string, website []byte) error { + if website == nil { + return az.deleteContainerMetaData(ctx, bucket, string(keyWebsite)) + } + + encoded, err := backend.MarshalWebsiteConfig(website, true) + if err != nil { + return err + } + + return az.setContainerMetaData(ctx, bucket, string(keyWebsite), encoded) +} + +func (az *Azure) GetBucketWebsite(ctx context.Context, bucket string) ([]byte, error) { + website, err := az.getContainerMetaData(ctx, bucket, string(keyWebsite)) + if err != nil { + return nil, err + } + if len(website) == 0 { + return nil, s3err.GetBucketErr(s3err.ErrNoSuchWebsiteConfiguration, bucket) + } + + decoded, err := backend.UnmarshalWebsiteConfig(website, true) + if err != nil { + return nil, err + } + + return decoded, nil +} + +func (az *Azure) DeleteBucketWebsite(ctx context.Context, bucket string) error { + return az.PutBucketWebsite(ctx, bucket, nil) +} + +func (az *Azure) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error { + return az.setContainerMetaData(ctx, bucket, string(keyBucketLock), config) +} + +func (az *Azure) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) { + cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock)) + if err != nil { + return nil, err + } + + if len(cfg) == 0 { + return nil, s3err.GetBucketErr(s3err.ErrObjectLockConfigurationNotFound, bucket) + } + + return cfg, nil +} + +func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, versionId string, retention []byte) error { + err := az.isBucketObjectLockEnabled(ctx, bucket) + if err != nil { + return err + } + + blobClient, err := az.getBlobClient(bucket, object) + if err != nil { + return err + } + + blobProps, err := blobClient.GetProperties(ctx, nil) + if err != nil { + return azureErrToS3Err(err) + } + + meta := blobProps.Metadata + if meta == nil { + meta = map[string]*string{ + string(keyObjRetention): backend.GetPtrFromString(string(retention)), + } + } else { + meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retention)) + } + + _, err = blobClient.SetMetadata(ctx, meta, nil) + if err != nil { + return azureErrToS3Err(err) + } + + return nil +} + +func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) { + client, err := az.getBlobClient(bucket, object) + if err != nil { + return nil, err + } + props, err := client.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + err = az.isBucketObjectLockEnabled(ctx, bucket) + if err != nil { + return nil, err + } + + retentionPtr, ok := props.Metadata[string(keyObjRetention)] + if !ok { + return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration) + } + + return []byte(*retentionPtr), nil +} + +func (az *Azure) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error { + err := az.isBucketObjectLockEnabled(ctx, bucket) + if err != nil { + return err + } + + blobClient, err := az.getBlobClient(bucket, object) + if err != nil { + return err + } + + blobProps, err := blobClient.GetProperties(ctx, nil) + if err != nil { + return azureErrToS3Err(err) + } + + var statusData string + if status { + statusData = "1" + } else { + statusData = "0" + } + + meta := blobProps.Metadata + if meta == nil { + meta = map[string]*string{ + string(keyObjLegalHold): &statusData, + } + } else { + meta[string(keyObjLegalHold)] = &statusData + } + + _, err = blobClient.SetMetadata(ctx, meta, nil) + if err != nil { + return azureErrToS3Err(err) + } + + return nil +} + +func (az *Azure) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) { + client, err := az.getBlobClient(bucket, object) + if err != nil { + return nil, err + } + props, err := client.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + err = az.isBucketObjectLockEnabled(ctx, bucket) + if err != nil { + return nil, err + } + + retentionPtr, ok := props.Metadata[string(keyObjLegalHold)] + if !ok { + return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration) + } + + status := *retentionPtr == "1" + + return &status, nil +} + +func (az *Azure) ChangeBucketOwner(ctx context.Context, bucket, owner string) error { + return auth.UpdateBucketACLOwner(ctx, az, bucket, owner) +} + +// The action actually returns the containers owned by the user, who initialized the gateway +// TODO: Not sure if there's a way to list all the containers and owners? +func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) { + opts := &service.ListContainersOptions{ + Include: service.ListContainersInclude{Metadata: true}, + } + pager := az.client.NewListContainersPager(opts) + + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return buckets, azureErrToS3Err(err) + } + for _, v := range resp.ContainerItems { + acl, err := getAclFromMetadata(v.Metadata, keyAclLower) + if err != nil { + return buckets, err + } + + buckets = append(buckets, s3response.Bucket{ + Name: *v.Name, + Owner: acl.Owner, + }) + } + } + return buckets, nil +} + +func (az *Azure) isBucketObjectLockEnabled(ctx context.Context, bucket string) error { + cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock)) + if err != nil { + return azureErrToS3Err(err) + } + + if len(cfg) == 0 { + return s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration) + } + + var bucketLockConfig auth.BucketLockConfig + err = json.Unmarshal(cfg, &bucketLockConfig) + if err != nil { + return fmt.Errorf("parse bucket lock config: %w", err) + } + + if !bucketLockConfig.Enabled { + return s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration) + } + + return nil +} + +func (az *Azure) getContainerURL(cntr string) string { + return fmt.Sprintf("%v/%v", strings.TrimRight(az.serviceURL, "/"), cntr) +} + +func (az *Azure) getBlobURL(cntr, blb string) string { + // URL-encode the blob name to handle special characters like {, }, #, spaces, etc. + // Use PathEscape to encode the blob name while preserving forward slashes + encodedBlob := url.PathEscape(blb) + return fmt.Sprintf("%v/%v", az.getContainerURL(cntr), encodedBlob) +} + +func (az *Azure) getBlobClient(cntr, blb string) (*blob.Client, error) { + blobURL := az.getBlobURL(cntr, blb) + if az.defaultCreds != nil { + return blob.NewClient(blobURL, az.defaultCreds, nil) + } + if az.sasToken != "" { + return blob.NewClientWithNoCredential(blobURL+"?"+az.sasToken, nil) + } + return blob.NewClientWithSharedKeyCredential(blobURL, az.sharedkeyCreds, nil) +} + +func (az *Azure) getContainerClient(cntr string) (*container.Client, error) { + containerURL := az.getContainerURL(cntr) + if az.defaultCreds != nil { + return container.NewClient(containerURL, az.defaultCreds, nil) + } + if az.sasToken != "" { + return container.NewClientWithNoCredential(containerURL+"?"+az.sasToken, nil) + } + return container.NewClientWithSharedKeyCredential(containerURL, az.sharedkeyCreds, nil) +} + +func (az *Azure) getBlockBlobClient(cntr, blb string) (*blockblob.Client, error) { + blobURL := az.getBlobURL(cntr, blb) + if az.defaultCreds != nil { + return blockblob.NewClient(blobURL, az.defaultCreds, nil) + } + if az.sasToken != "" { + return blockblob.NewClientWithNoCredential(blobURL+"?"+az.sasToken, nil) + } + return blockblob.NewClientWithSharedKeyCredential(blobURL, az.sharedkeyCreds, nil) +} + +func parseMetadata(m map[string]string) map[string]*string { + if m == nil { + return nil + } + + meta := make(map[string]*string) + + for k, v := range m { + val := v + meta[k] = &val + } + return meta +} + +func parseAndFilterAzMetadata(m map[string]*string) map[string]string { + if m == nil { + return nil + } + + keywords := keyTags.Table() + + meta := make(map[string]string) + + for k, v := range m { + _, ok := keywords[strings.ToLower(k)] + if ok { + continue + } + meta[k] = *v + } + return meta +} + +func parseAzMetadata(m map[string]*string) map[string]string { + if m == nil { + return nil + } + + meta := make(map[string]string) + + for k, v := range m { + meta[k] = *v + } + return meta +} + +func parseAzTags(tagSet []*blob.Tags) map[string]string { + tags := map[string]string{} + for _, tag := range tagSet { + tags[*tag.Key] = *tag.Value + } + + return tags +} + +func getString(str *string) string { + if str == nil { + return "" + } + return *str +} + +// Converts io.Reader into io.ReadSeekCloser +func getReadSeekCloser(input io.Reader) (io.ReadSeekCloser, error) { + var buffer bytes.Buffer + _, err := io.Copy(&buffer, input) + if err != nil { + return nil, err + } + + return streaming.NopCloser(bytes.NewReader(buffer.Bytes())), nil +} + +// quoteETag wraps a raw Azure block id (used as a multipart part ETag) in +// double quotes. S3 ETags are quoted strings, and the posix backend follows +// the same convention (see backend.GenerateEtag). Keeping all Azure part-ETag +// surfaces (UploadPart, ListParts, UploadPartCopy) quoted ensures the values +// returned to clients are consistent, while the raw (unquoted) block id is +// still used for the Azure StageBlock/GetBlockList APIs. +func quoteETag(etag string) string { + return fmt.Sprintf("%q", etag) +} + +// Creates a new Base64 encoded block id from a 32 bit integer +func blockIDInt32ToBase64(blockID int32) string { + binaryBlockID := &[4]byte{} // All block IDs are 4 bytes long + binary.LittleEndian.PutUint32(binaryBlockID[:], uint32(blockID)) + return base64.StdEncoding.EncodeToString(binaryBlockID[:]) +} + +// Decodes Base64 encoded string to integer +func decodeBlockId(blockID string) (int, error) { + slice, err := base64.StdEncoding.DecodeString(blockID) + if err != nil { + return 0, nil + } + + return int(binary.LittleEndian.Uint32(slice)), nil +} + +func encodeBytes(b []byte) string { + return base64.StdEncoding.EncodeToString(b) +} + +func decodeString(str string) ([]byte, error) { + if str == "" { + return []byte{}, nil + } + + decoded, err := base64.StdEncoding.DecodeString(str) + if err != nil { + return nil, err + } + return decoded, nil +} + +func (az *Azure) getContainerMetaData(ctx context.Context, bucket, key string) ([]byte, error) { + client, err := az.getContainerClient(bucket) + if err != nil { + return nil, err + } + + props, err := client.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + if props.Metadata == nil { + return []byte{}, nil + } + + data, ok := props.Metadata[key] + if !ok { + return []byte{}, nil + } + + value, err := decodeString(*data) + if err != nil { + return nil, err + } + + return value, nil +} + +func (az *Azure) setContainerMetaData(ctx context.Context, bucket, key string, value []byte) error { + client, err := az.getContainerClient(bucket) + if err != nil { + return err + } + + props, err := client.GetProperties(ctx, nil) + if err != nil { + return azureErrToS3Err(err) + } + + mdmap := props.Metadata + if mdmap == nil { + mdmap = make(map[string]*string) + } + + str := encodeBytes(value) + mdmap[key] = backend.GetPtrFromString(str) + + _, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: mdmap}) + if err != nil { + return err + } + return nil +} + +func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string) error { + client, err := az.getContainerClient(bucket) + if err != nil { + return err + } + + props, err := client.GetProperties(ctx, nil) + if err != nil { + return azureErrToS3Err(err) + } + + mdmap := props.Metadata + if mdmap == nil { + mdmap = make(map[string]*string) + } + + delete(mdmap, key) + + _, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: mdmap}) + if err != nil { + return err + } + return nil +} + +func (az *Azure) evaluateWritePreconditions(ctx context.Context, bucket, object, ifMatch, ifNoneMatch *string) error { + if areNils(ifMatch, ifNoneMatch) { + return nil + } + // call HeadObject to evaluate preconditions + res, err := az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: bucket, + Key: object, + }) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return err + } + + var etag string + if res != nil { + etag = backend.GetStringFromPtr(res.ETag) + } + + return backend.EvaluateObjectPutPreconditions(etag, ifMatch, ifNoneMatch, !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey))) +} + +func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) { + data, ok := meta[string(key)] + if !ok { + return &auth.ACL{}, nil + } + + value, err := decodeString(*data) + if err != nil { + return nil, err + } + + acl, err := auth.ParseACL(value) + if err != nil { + return nil, err + } + + return &acl, nil +} + +func createMetaTmpPath(obj, uploadId string) string { + objNameSum := sha256.Sum256([]byte(obj)) + return filepath.Join(string(metaTmpMultipartPrefix), uploadId, fmt.Sprintf("%x", objNameSum)) +} + +// trackZeroBytePart records a zero-byte upload part in the sgwtmp metadata. +// Azure StageBlock rejects Content-Length: 0, so zero-byte parts are stored here. +func (az *Azure) trackZeroBytePart(ctx context.Context, bucket, key, uploadId string, partNumber int32) error { + tmpPath := createMetaTmpPath(key, uploadId) + blobClient, err := az.getBlobClient(bucket, tmpPath) + if err != nil { + return err + } + + props, err := blobClient.GetProperties(ctx, nil) + if err != nil { + return azureErrToS3Err(err) + } + + meta := props.Metadata + if meta == nil { + meta = map[string]*string{} + } + + // Deduplicate: replace an existing entry for the same partNumber. + parts := parseZeroByteParts(meta) + found := slices.Contains(parts, partNumber) + if !found { + parts = append(parts, partNumber) + } + + serialized := serializeZeroByteParts(parts) + meta[string(keyMpZeroBytesParts)] = &serialized + _, err = blobClient.SetMetadata(ctx, meta, nil) + return azureErrToS3Err(err) +} + +// getZeroByteParts returns the list of zero-byte parts tracked in the sgwtmp metadata. +func (az *Azure) getZeroByteParts(ctx context.Context, bucket, key, uploadId string) ([]int32, error) { + tmpPath := createMetaTmpPath(key, uploadId) + blobClient, err := az.getBlobClient(bucket, tmpPath) + if err != nil { + return nil, err + } + + props, err := blobClient.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + return parseZeroByteParts(props.Metadata), nil +} + +func parseZeroByteParts(meta map[string]*string) []int32 { + val, ok := meta[string(keyMpZeroBytesParts)] + if !ok || val == nil || *val == "" { + return nil + } + var parts []int32 + for s := range strings.SplitSeq(*val, ",") { + n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 32) + if err == nil { + parts = append(parts, int32(n)) + } + } + return parts +} + +func serializeZeroByteParts(parts []int32) string { + strs := make([]string, len(parts)) + for i, p := range parts { + strs[i] = strconv.Itoa(int(p)) + } + return strings.Join(strs, ",") +} + +// Checks if the multipart upload existis with the given bucket, key and uploadId +func (az *Azure) checkIfMpExists(ctx context.Context, bucket, obj, uploadId string) error { + _, err := az.getMpMetadata(ctx, bucket, obj, uploadId) + return err +} + +// getMpMetadata returns the sgwtmp metadata of a multipart upload, or +// NoSuchUpload when there is no such upload. +func (az *Azure) getMpMetadata(ctx context.Context, bucket, obj, uploadId string) (map[string]*string, error) { + tmpPath := createMetaTmpPath(obj, uploadId) + blobClient, err := az.getBlobClient(bucket, tmpPath) + if err != nil { + return nil, err + } + + props, err := blobClient.GetProperties(ctx, nil) + if err != nil { + return nil, s3err.GetNoSuchUploadErr(uploadId) + } + + return props.Metadata, nil +} + +func convertAzureEtag(etag *azcore.ETag) string { + // Azure ETag values are not S3 compatible, + // so append "-1" to avoid client SDK ETag validation issues. + str := (*string)(etag) + + return *backend.TrimEtag(str) + "-1" +} + +func areNils[T any](args ...*T) bool { + for _, arg := range args { + if arg != nil { + return false + } + } + + return true +} diff --git a/backend/mp-lister.go b/backend/mp-lister.go index 3d49051..75e8f7c 100644 --- a/backend/mp-lister.go +++ b/backend/mp-lister.go @@ -126,9 +126,8 @@ func (l *MultipartUploadLister) Run() (*ListMultipartUploadsPage, error) { if out.IsTruncated { out.NextKeyMarker = lastKey out.NextUploadIDMarker = up.UploadID - return out, nil } - break + return out, nil } } continue @@ -140,20 +139,11 @@ func (l *MultipartUploadLister) Run() (*ListMultipartUploadsPage, error) { if out.IsTruncated { out.NextKeyMarker = lastKey out.NextUploadIDMarker = up.UploadID - return out, nil } - break + return out, nil } } - // a listing that is not truncated still reports its last upload - // in the next markers - if len(out.Uploads) != 0 { - last := out.Uploads[len(out.Uploads)-1] - out.NextKeyMarker = last.Key - out.NextUploadIDMarker = last.UploadID - } - return out, nil } diff --git a/backend/posix/posix.go b/backend/posix/posix.go new file mode 100644 index 0000000..b30a63a --- /dev/null +++ b/backend/posix/posix.go @@ -0,0 +1,8034 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package posix + +import ( + "context" + "crypto/md5" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/google/uuid" + "github.com/oklog/ulid/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/backend/meta" + "github.com/versity/versitygw/debuglogger" + "github.com/versity/versitygw/s3api/middlewares" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" + "golang.org/x/sync/semaphore" +) + +type Posix struct { + backend.BackendUnsupported + + // bucket/object metadata storage facility + meta meta.MetadataStorer + + rootfd *os.File + rootdir string + // pathRoot is the directory bucket paths are built from: rootdir when + // PosixOpts.AbsolutePaths is set, otherwise "" because the process + // working directory is the root directory. See BucketPath. + pathRoot string + + // chownuid/gid enable chowning of files to the account uid/gid + // when objects are uploaded + chownuid bool + chowngid bool + + // euid/egid are the effective uid/gid of the running versitygw process + // used to determine if chowning is needed + euid int + egid int + + // bucketlinks is a flag to enable symlinks to directories at the top + // level gateway directory to be treated as buckets the same as directories + bucketlinks bool + + // bucket versioning directory path + versioningDir string + + // newDirPerm is the permission to set on newly created directories + newDirPerm fs.FileMode + + // newFilePerm is the permission to set on newly created object files + newFilePerm fs.FileMode + + // forceNoTmpFile is a flag to disable the use of O_TMPFILE even + // if the filesystem supports it. This is needed for cases where + // there are different filesystems mounted below the bucket level. + forceNoTmpFile bool + + // forceNoCopyFileRange is a flag to disable the use of io.Copy to + // reassemble multipart upload parts, which uses copy_file_range on + // linux. This is needed for cases where a filesystem that does not + // support copy_file_range is mounted over NFSv4.2. + forceNoCopyFileRange bool + + // objectLockMode selects the advisory lock used for conditional object + // publishes. The local mode retains process-local serialization only, and + // the none mode rejects conditional writes. + objectLockMode ObjectLockMode + + // enableODirect is a flag to open object data files with O_DIRECT. + // This is best-effort and falls back to buffered I/O when unsupported. + enableODirect bool + + // enable posix level bucket name validations, not needed if the + // frontend handlers are already validating bucket names + validateBucketName bool + + // defaultEtag is returned for objects that do not have an etag attribute + // stored (e.g. files placed on the filesystem outside of versitygw). + defaultEtag string + + // actionLimiter limits the number of concurrently running POSIX actions. + // The primary goal is to bound OS thread growth: when goroutines block in + // filesystem syscalls, the Go scheduler may spawn additional OS threads to + // keep other goroutines runnable. Since nearly all POSIX actions eventually + // execute blocking syscalls (stat, readdir, xattr, open, etc.), this limiter + // constrains parallelism to prevent excessive thread creation under load. + actionLimiter *semaphore.Weighted + + // copyObjectThreshold is the maximum allowed size (in bytes) for a copy + // source object. Requests to copy objects larger than this value are + // rejected with an 'InvalidRequest' to comply with the S3 limit + // of 5 GiB. + copyObjectThreshold int64 + + // ioBufferSize is the buffer size used by buffered copy/read paths. + ioBufferSize int + ioBufferPool sync.Pool + + // dataIntegrityEtag, when true, replaces the standard MD5-based ETag with + // a checksum-derived ETag that embeds the algorithm name and value. + // For multipart uploads this applies to both part ETags and the final + // completed object ETag: + // single PUT: "ALGO-" + // multipart part: "CRC64NVME-" + // multipart CRC64NVME: "CRC64NVME-" + // multipart composite: "ALGO--" + dataIntegrityEtag bool + + // objLockSlots are process-local striped slots taken alongside the shared + // per-bucket lock files by lockObjectPublish. Channels make local lock + // acquisition cancelable while avoiding filesystem lock thrash. + objLockSlots [objLockShards]chan struct{} +} + +func (o *PosixOpts) SetNewDirPerm(perm fs.FileMode) { + o.NewDirPerm = perm + o.newDirPermSet = true +} + +func (o *PosixOpts) SetNewFilePerm(perm fs.FileMode) { + o.NewFilePerm = perm + o.newFilePermSet = true +} + +var _ backend.Backend = &Posix{} + +const ( + defaultNewDirPerm fs.FileMode = 0755 + defaultNewFilePerm fs.FileMode = 0644 + + MetaTmpDir = ".sgwtmp" + MetaTmpMultipartDir = MetaTmpDir + "/multipart" + onameAttr = "objname" + inProgressSuffix = ".inprogress" + tagHdr = "X-Amz-Tagging" + oldMetaHdr = "X-Amz-Meta" + metadataHdr = "metadata" + contentTypeHdr = "content-type" + contentEncHdr = "content-encoding" + contentLangHdr = "content-language" + contentDispHdr = "content-disposition" + cacheCtrlHdr = "cache-control" + expiresHdr = "expires" + websiteRedirectHdr = "website-redirect-location" + emptyMD5 = "\"d41d8cd98f00b204e9800998ecf8427e\"" + aclkey = "acl" + ownershipkey = "ownership" + etagkey = "etag" + checksumsKey = "checksums" + policykey = "policy" + bucketLockKey = "bucket-lock" + objectRetentionKey = "object-retention" + objectLegalHoldKey = "object-legal-hold" + corskey = "cors" + websitekey = "website" + versioningKey = "versioning" + deleteMarkerKey = "delete-marker" + versionIdKey = "version-id" + nullVersionPrevKey = "null-version-prev" + partCrc64nvme = "part-crc64nvme" + mpMetaKey = "mp-metadata" + mpObjectLockKey = "mp-object-lock" + + nullVersionId = "null" + + doFalloc = true + skipFalloc = false + + odirectAllowed odirectPolicy = true + odirectNotAllowed odirectPolicy = false + + // defaultConcurrency is the default limit for concurrent POSIX actions. + defaultConcurrency = 5000 + // defaultIOBufferSize is the default buffer size used by io.CopyBuffer paths. + defaultIOBufferSize = 1024 * 1024 +) + +type odirectPolicy bool + +// PosixOpts are the options for the Posix backend +type PosixOpts struct { + // ChownUID sets the UID of the object to the UID of the user on PUT + ChownUID bool + // ChownGID sets the GID of the object to the GID of the user on PUT + ChownGID bool + // BucketLinks enables symlinks to directories to be treated as buckets + BucketLinks bool + // AbsolutePaths makes the backend address every bucket and object by an + // absolute path under the root directory instead of changing the + // process working directory to the root and using relative paths. Use it + // when embedding the gateway in a process that must keep its working + // directory, such as a Go test that reads test data by relative path. + // Absolute paths cost slightly more per filesystem operation. A relative + // VersioningDir or SideCarDir is then resolved against the working + // directory rather than the root directory. + AbsolutePaths bool + //VersioningDir sets the version directory to enable object versioning + VersioningDir string + // NewDirPerm specifies the permission to set on newly created directories. + // When unset, the default is 0755. An explicit zero value (0000) is valid + // and is preserved when SetNewDirPerm(0000) is used. + NewDirPerm fs.FileMode + newDirPermSet bool + // NewFilePerm specifies the permission to set on newly created object + // files. Defaults to 0644 when unset. Unlike NewDirPerm, this mode is + // applied explicitly and is therefore not reduced by the process umask. + // An explicit zero value (0000) is valid and is preserved when + // SetNewFilePerm(0000) is used. + NewFilePerm fs.FileMode + newFilePermSet bool + // SideCarDir sets the directory to store sidecar metadata + SideCarDir string + // ForceNoTmpFile disables the use of O_TMPFILE even if the filesystem + // supports it + ForceNoTmpFile bool + // ForceNoCopyFileRange disables the use of io.Copy for multipart uploads parts + ForceNoCopyFileRange bool + // ForceNoObjLockFile is a deprecated compatibility alias for + // ObjectLockModeLocal. + ForceNoObjLockFile bool + // ObjectLockMode selects flock, fcntl, local, or none for conditional object + // publishes. An empty value defaults to flock. Local locking is only atomic + // within a single gateway process; none rejects conditional writes. + ObjectLockMode ObjectLockMode + // EnableODirect enables best-effort O_DIRECT for object data reads/writes. + // Disabled by default. + EnableODirect bool + // ValidateBucketNames enables minimal bucket name validation to prevent + // incorrect access to the filesystem. This is only needed if the + // frontend is not already validating bucket names. + ValidateBucketNames bool + // Concurrency sets the maximum number of concurrently running POSIX actions. + // It is intended to bound OS thread growth caused by goroutines blocking in + // filesystem-heavy syscalls (e.g., stat, readdir, xattr), which can otherwise + // trigger additional thread creation under load. + // This acts as a backpressure mechanism: once the limit is reached, new + // actions are queued until capacity becomes available. As a result, if the + // queue depth grows under sustained load, request latency increases and + // upstream timeouts may occur. + Concurrency int + // CopyObjectThreshold sets the maximum allowed source object size (in bytes) + // for CopyObject and UploadPartCopy operations. Requests exceeding this + // threshold are rejected with an 'InvalidRequest' error. Defaults to the + // S3 specification limit of 5 GiB. + CopyObjectThreshold int64 + // DefaultEtag is returned for objects that do not have a stored etag + // attribute (e.g. files placed on the filesystem outside of versitygw). + // When empty, such objects are served with an empty ETag. + DefaultEtag string + // IOBufferSize sets the buffer size (in bytes) for copy/read paths that use + // io.CopyBuffer or buffered readers. Defaults to 1MiB when unset or invalid. + IOBufferSize int + // DataIntegrityEtag, when enabled, replaces the standard MD5-based ETag + // with a checksum-derived value that embeds the algorithm name and checksum + // (e.g. "CRC64NVME-"). For multipart uploads, part ETags become + // CRC64NVME-based values and the completed object ETag is checksum-derived. + // For CRC64NVME full-object checksums, the composable whole-file checksum is + // used; other algorithms embed the part count (e.g. "SHA256--"). + DataIntegrityEtag bool +} + +// New returns a Posix backend serving buckets from the directories under rootdir. +func New(rootdir string, ms meta.MetadataStorer, opts PosixOpts) (*Posix, error) { + objectLockMode, err := resolveObjectLockMode(opts.ObjectLockMode, opts.ForceNoObjLockFile) + if err != nil { + return nil, err + } + ioBufferSize := ioBufferSizeOrDefault(opts.IOBufferSize) + rootdirAbs, err := filepath.Abs(rootdir) + if err != nil { + return nil, fmt.Errorf("get absolute path of %v: %w", rootdir, err) + } + + if opts.SideCarDir != "" && strings.HasPrefix(opts.SideCarDir, rootdir) { + return nil, fmt.Errorf("sidecar directory cannot be inside the gateway root directory") + } + + // A storer that keeps metadata on the object files (xattr) resolves + // bucket names the same way this backend does: relative to the working + // directory by default, under the absolute root with AbsolutePaths. + var pathRoot string + if opts.AbsolutePaths { + pathRoot = rootdirAbs + if rs, ok := ms.(meta.RootDirSetter); ok { + ms = rs.WithRootDir(rootdirAbs) + } + } + + if !opts.AbsolutePaths { + // Change directory before resolving the versioning and sidecar + // directories: a relative one is then relative to the root, which is + // also how a SideCar storer constructed with the same relative path + // will resolve it once the process runs from the root. + err = os.Chdir(rootdirAbs) + if err != nil { + return nil, fmt.Errorf("chdir %v: %w", rootdir, err) + } + } + + var versioningdirAbs string + // Ensure the versioning directory isn't within the root directory + if opts.VersioningDir != "" { + versioningdirAbs, err = validateSubDir(rootdirAbs, opts.VersioningDir) + if err != nil { + return nil, err + } + } + + var sidecardirAbs string + // Ensure the sidecar directory isn't within the root directory + if opts.SideCarDir != "" { + sidecardirAbs, err = validateSubDir(rootdirAbs, opts.SideCarDir) + if err != nil { + return nil, err + } + } + + // Opened last so that no error path below has to close it. + f, err := os.Open(rootdirAbs) + if err != nil { + return nil, fmt.Errorf("open %v: %w", rootdir, err) + } + if opts.AbsolutePaths { + // Without the chdir above nothing else rejects a root that is not + // a directory. + if fi, err := f.Stat(); err != nil || !fi.IsDir() { + f.Close() + if err != nil { + return nil, fmt.Errorf("stat %v: %w", rootdir, err) + } + return nil, fmt.Errorf("%v is not a directory", rootdir) + } + } + + if versioningdirAbs != "" { + fmt.Println("Bucket versioning enabled with directory:", versioningdirAbs) + } + + if sidecardirAbs != "" { + fmt.Println("Using sidecar directory for metadata:", sidecardirAbs) + } + + // A gateway that is not root can only chown a file to an account whose + // uid/gid it already has, so --chuid/--chgid either do nothing or fail + // every write for every other account. That is a configuration mistake + // worth naming at startup rather than one InternalError per request, but + // it is not fatal: the process may hold CAP_CHOWN without being root, and + // a setup where every account shares the gateway's own uid/gid is a + // legitimate no-op. + euid, egid := os.Geteuid(), os.Getegid() + if (opts.ChownUID || opts.ChownGID) && euid != 0 { + fmt.Printf("Warning: --chuid/--chgid requested, but the gateway runs as euid %v/egid %v: "+ + "writes for any account with a different uid/gid will fail unless this process can chown\n", + euid, egid) + } + + newDirPerm := defaultNewDirPerm + if opts.newDirPermSet { + newDirPerm = opts.NewDirPerm.Perm() + } + newFilePerm := defaultNewFilePerm + if opts.newFilePermSet { + newFilePerm = opts.NewFilePerm.Perm() + } + + p := &Posix{ + meta: ms, + rootfd: f, + rootdir: rootdirAbs, + pathRoot: pathRoot, + euid: euid, + egid: egid, + chownuid: opts.ChownUID, + chowngid: opts.ChownGID, + bucketlinks: opts.BucketLinks, + versioningDir: versioningdirAbs, + newDirPerm: newDirPerm, + newFilePerm: newFilePerm, + forceNoTmpFile: opts.ForceNoTmpFile, + forceNoCopyFileRange: opts.ForceNoCopyFileRange, + objectLockMode: objectLockMode, + enableODirect: opts.EnableODirect, + validateBucketName: opts.ValidateBucketNames, + actionLimiter: semaphore.NewWeighted(int64(concurrencyOrDefault(opts.Concurrency))), + copyObjectThreshold: opts.CopyObjectThreshold, + defaultEtag: opts.DefaultEtag, + ioBufferSize: ioBufferSize, + ioBufferPool: sync.Pool{New: func() any { + b := alignedBuffer(ioBufferSize) + return &b + }}, + dataIntegrityEtag: opts.DataIntegrityEtag, + objLockSlots: newObjLockSlots(), + } + if err := p.verifyObjectLockMode(); err != nil { + p.Shutdown() + return nil, err + } + return p, nil +} + +// BucketPath returns the filesystem path of the bucket directory: the bucket +// name itself by default (the process working directory is the root +// directory), or the absolute path under the root with +// PosixOpts.AbsolutePaths. +// +// The versioning code passes the absolute path of a bucket's versioning +// directory where a bucket name is expected, and the metadata storer accepts +// the same substitution; an absolute bucket is therefore returned as given. +func (p *Posix) BucketPath(bucket string) string { + if p.pathRoot == "" || filepath.IsAbs(bucket) { + return bucket + } + return filepath.Join(p.pathRoot, bucket) +} + +// rootPath returns the filesystem path of the root directory in the form +// BucketPath builds bucket paths from. +func (p *Posix) rootPath() string { + if p.pathRoot == "" { + return "." + } + return p.pathRoot +} + +// ObjectPath returns the filesystem path of object within bucket. object is +// any path relative to the bucket directory: an object key, a multipart +// upload's temporary directory, or a part file within one. +func (p *Posix) ObjectPath(bucket, object string) string { + return filepath.Join(p.BucketPath(bucket), object) +} + +// concurrencyOrDefault returns n if it is positive, otherwise defaultConcurrency. +func concurrencyOrDefault(n int) int { + if n > 0 { + return n + } + return defaultConcurrency +} + +func ioBufferSizeOrDefault(n int) int { + if n > 0 { + return n + } + return defaultIOBufferSize +} + +func (p *Posix) getIOBuffer() []byte { + bp, ok := p.ioBufferPool.Get().(*[]byte) + if !ok || bp == nil || cap(*bp) < p.ioBufferSize { + return alignedBuffer(p.ioBufferSize) + } + return (*bp)[:p.ioBufferSize] +} + +func (p *Posix) putIOBuffer(b []byte) { + if b == nil { + return + } + if cap(b) < p.ioBufferSize { + return + } + b = b[:p.ioBufferSize] + p.ioBufferPool.Put(&b) +} + +func validateSubDir(root, dir string) (string, error) { + absDir, err := filepath.Abs(dir) + if err != nil { + return "", fmt.Errorf("get absolute path of %v: %w", + dir, err) + } + + if isDirBelowRoot(root, absDir) { + return "", fmt.Errorf("the root directory %v contains the directory %v", + root, dir) + } + + vDir, err := os.Stat(absDir) + if err != nil { + return "", fmt.Errorf("stat %q: %w", absDir, err) + } + + if !vDir.IsDir() { + return "", fmt.Errorf("path %q is not a directory", absDir) + } + + return absDir, nil +} + +func isDirBelowRoot(root, dir string) bool { + // Ensure the paths ends with a separator + if !strings.HasSuffix(root, string(filepath.Separator)) { + root += string(filepath.Separator) + } + + if !strings.HasSuffix(dir, string(filepath.Separator)) { + dir += string(filepath.Separator) + } + + // Ensure the root directory doesn't contain the directory + if strings.HasPrefix(dir, root) { + return true + } + + return false +} + +func (p *Posix) Shutdown() { + p.rootfd.Close() +} + +func (p *Posix) String() string { + return "Posix Gateway" +} + +// define a private key type +type ctxKey int + +const ( + ctxKeyNoAcquireSlot ctxKey = iota +) + +// withCtxNoSlot is a context wrapper with ctxKeyNoAcquireSlot key +func withCtxNoSlot(ctx context.Context) context.Context { + return context.WithValue(ctx, ctxKeyNoAcquireSlot, struct{}{}) +} + +// acquireActionSlot reserves a concurrency slot for an action +// it blocks until a slot is available or the context is canceled. +// On success, it returns a releaser that must be called to free the slot. +// If ctxKeyNoAcquireSlot is set in the context, the limiter is bypassed +// and a no-op releaser is returned. +func (p *Posix) acquireActionSlot(ctx context.Context) (func(), error) { + // if no acquire lock is set, do not reserve a slot + // and return a blank releaser + val := ctx.Value(ctxKeyNoAcquireSlot) + if val != nil { + return func() {}, nil + } + + if err := p.actionLimiter.Acquire(ctx, 1); err != nil { + return nil, err + } + + return func() { + p.actionLimiter.Release(1) + }, nil +} + +// returns the versioning state +func (p *Posix) versioningEnabled() bool { + return p.versioningDir != "" +} + +// validateVersionId checks if the input versionId is 'ulid' compatible +func (p *Posix) validateVersionId(versionId string) error { + if versionId == "" || versionId == "null" { + return nil + } + _, err := ulid.Parse(versionId) + if err != nil { + return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + return nil +} + +// DoesBucketExist returns NoSuchBucket unless bucket names a bucket. Backends +// built on Posix that add their own entry points must check buckets with it +// rather than stat the bucket path themselves. +func (p *Posix) DoesBucketExist(bucket string) error { + return p.doesBucketExist(bucket) +} + +// doesBucketExist returns NoSuchBucket unless bucket names a directory under +// the root, or with bucketlinks a symlink to one: the entries +// listBucketFileInfos returns. The root of a dataset the gateway did not +// create may also hold regular files, symlinks and other entries; a request +// naming one of them fails like one naming a missing bucket instead of +// storing bucket metadata on it or building object paths through it. +func (p *Posix) doesBucketExist(bucket string) error { + path := p.BucketPath(bucket) + fi, err := os.Lstat(path) + if err == nil && p.bucketlinks && fi.Mode()&fs.ModeSymlink != 0 { + fi, err = os.Stat(path) + } + // a symlink loop never resolves to a directory + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ELOOP) { + return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket) + } + if err != nil { + return fmt.Errorf("stat bucket: %w", err) + } + if !fi.IsDir() { + return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket) + } + + return nil +} + +// doesBucketAndObjectExist checks that bucket exists and, when versionId is +// empty, that object has a current version. A specific version is looked up +// once versionId is validated. +func (p *Posix) doesBucketAndObjectExist(bucket, object, versionId string) error { + err := p.doesBucketExist(bucket) + if err != nil { + return err + } + if versionId != "" { + return nil + } + + _, _, err = p.objVersionAttrPath(bucket, object, "") + return err +} + +func (p *Posix) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.ListAllMyBucketsResult{}, err + } + defer release() + + fis, err := listBucketFileInfos(p.rootPath(), p.bucketlinks) + if err != nil { + return s3response.ListAllMyBucketsResult{}, fmt.Errorf("listBucketFileInfos : %w", err) + } + + var cToken string + + var buckets []s3response.ListAllMyBucketsEntry + for _, fi := range fis { + if !strings.HasPrefix(fi.Name(), input.Prefix) { + continue + } + + if !utils.IsValidBucketName(fi.Name()) { + continue + } + + if len(buckets) == int(input.MaxBuckets) { + cToken = buckets[len(buckets)-1].Name + break + } + + if fi.Name() <= input.ContinuationToken { + continue + } + + // return all the buckets for admin users + if input.IsAdmin { + buckets = append(buckets, s3response.ListAllMyBucketsEntry{ + Name: fi.Name(), + CreationDate: fi.ModTime(), + }) + continue + } + + aclJSON, err := p.meta.RetrieveAttribute(nil, fi.Name(), "", aclkey) + if errors.Is(err, meta.ErrNoSuchKey) { + // skip buckets without acl tag + continue + } + if err != nil { + return s3response.ListAllMyBucketsResult{}, fmt.Errorf("get acl tag: %w", err) + } + + acl, err := auth.ParseACL(aclJSON) + if err != nil { + return s3response.ListAllMyBucketsResult{}, err + } + + if acl.Owner == input.Owner { + buckets = append(buckets, s3response.ListAllMyBucketsEntry{ + Name: fi.Name(), + CreationDate: fi.ModTime(), + }) + } + } + + return s3response.ListAllMyBucketsResult{ + Buckets: s3response.ListAllMyBucketsList{ + Bucket: buckets, + }, + Owner: s3response.CanonicalUser{ + ID: input.Owner, + }, + Prefix: input.Prefix, + ContinuationToken: cToken, + }, nil +} + +// IsBucketValid reports whether bucket may be used as a bucket name by this +// backend. Backends built on Posix that add their own entry points must check +// bucket names with it before building paths from them. +func (p *Posix) IsBucketValid(bucket string) bool { + return p.isBucketValid(bucket) +} + +func (p *Posix) isBucketValid(bucket string) bool { + // Regardless of validation mode, a bucket name must name a single + // directory entry under the root: "", "." and ".." would resolve to the + // root or its parent (see BucketPath), a separator or an absolute path + // would escape it. Callers that need the versioning directory pass it in + // place of the bucket only after this check. + if bucket == "" || bucket == "." || bucket == ".." || bucket == objLockDir || + filepath.IsAbs(bucket) || strings.ContainsRune(bucket, '/') || + strings.ContainsRune(bucket, os.PathSeparator) { + return false + } + + if !p.validateBucketName { + return true + } + + return backend.IsValidDirectoryName(bucket) +} + +func (p *Posix) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + if !p.isBucketValid(*input.Bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, *input.Bucket) + } + err = p.doesBucketExist(*input.Bucket) + if err != nil { + return nil, err + } + + return &s3.HeadBucketOutput{}, nil +} + +func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + acct, ok := ctx.Value("bucket-owner").(auth.Account) + if !ok { + acct = auth.Account{} + } + + uid, gid, doChown := p.getChownIDs(acct) + + bucket := *input.Bucket + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + tagging, err := backend.ParseCreateBucketTags(input.CreateBucketConfiguration.Tags) + if err != nil { + return err + } + + err = os.Mkdir(p.BucketPath(bucket), p.newDirPerm) + if err != nil && os.IsExist(err) { + // An existing entry that is not a bucket, such as a regular file at + // the root of a preexisting dataset, still takes the name but has + // no bucket metadata to consult. + err := p.doesBucketExist(bucket) + if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)) { + return s3err.GetBucketErr(s3err.ErrBucketAlreadyExists, bucket) + } + if err != nil { + return err + } + + aclJSON, err := p.meta.RetrieveAttribute(nil, bucket, "", aclkey) + if errors.Is(err, meta.ErrNoSuchKey) { + // The directory already exists but has no gateway-managed acl + // attribute, e.g. a preexisting directory on a dataset the + // gateway was pointed at rather than one created via CreateBucket + return s3err.GetBucketErr(s3err.ErrBucketAlreadyExists, bucket) + } + if err != nil { + return fmt.Errorf("get bucket acl: %w", err) + } + + acl, err := auth.ParseACL(aclJSON) + if err != nil { + return err + } + + if acl.Owner == acct.Access { + return s3err.GetBucketErr(s3err.ErrBucketAlreadyOwnedByYou, bucket) + } + return s3err.GetBucketErr(s3err.ErrBucketAlreadyExists, bucket) + } + if err != nil { + if errors.Is(err, syscall.EROFS) { + return s3err.GetAPIError(s3err.ErrMethodNotAllowed) + } + return fmt.Errorf("mkdir bucket: %w", err) + } + + // The directory now exists but is not yet a usable bucket: until the acl + // xattr below is stored, every request for this name — a retry of this + // same CreateBucket included — fails with "get bucket acl: no such key", + // and the name stays poisoned until someone removes the directory by + // hand. Undo the mkdir on any failure from here on so the name stays + // retryable. + defer func() { + if err == nil { + return + } + p.removePartialBucket(bucket) + }() + + if doChown { + err = os.Chown(p.BucketPath(bucket), uid, gid) + if err != nil { + return p.chownErr(bucket, uid, gid, err) + } + } + + err = p.meta.StoreAttribute(nil, bucket, "", aclkey, acl) + if err != nil { + return fmt.Errorf("set acl: %w", err) + } + err = p.meta.StoreAttribute(nil, bucket, "", ownershipkey, []byte(input.ObjectOwnership)) + if err != nil { + return fmt.Errorf("set ownership: %w", err) + } + if tagging != nil { + tags, err := json.Marshal(tagging) + if err != nil { + return fmt.Errorf("marshal tags: %w", err) + } + err = p.meta.StoreAttribute(nil, bucket, "", tagHdr, tags) + if err != nil { + return fmt.Errorf("set tags: %w", err) + } + } + + if input.ObjectLockEnabledForBucket != nil && *input.ObjectLockEnabledForBucket { + // First enable bucket versioning + // Bucket versioning is enabled automatically with object lock + if p.versioningEnabled() { + err = p.PutBucketVersioning(withCtxNoSlot(ctx), bucket, types.BucketVersioningStatusEnabled) + if err != nil { + return err + } + } + + now := time.Now() + defaultLock := auth.BucketLockConfig{ + Enabled: true, + CreatedAt: &now, + } + + defaultLockParsed, err := json.Marshal(defaultLock) + if err != nil { + return fmt.Errorf("parse default bucket lock state: %w", err) + } + + err = p.meta.StoreAttribute(nil, bucket, "", bucketLockKey, defaultLockParsed) + if err != nil { + return fmt.Errorf("set default bucket lock: %w", err) + } + } + + return nil +} + +// removePartialBucket undoes a partially completed CreateBucket: the bucket +// directory, any metadata stored for it (which may live in a sidecar +// directory outside the bucket) and its versioning directory. Failures here +// are logged rather than returned — the caller is already failing with the +// error that matters, and reporting a cleanup failure instead would hide it. +func (p *Posix) removePartialBucket(bucket string) { + if err := os.RemoveAll(p.BucketPath(bucket)); err != nil { + debuglogger.Logf("failed to remove partially created bucket (%q): %v", bucket, err) + } + if err := p.meta.DeleteAttributes(bucket, ""); err != nil { + debuglogger.Logf("failed to delete partially created bucket sidecar attributes (%q): %v", bucket, err) + } + if p.versioningEnabled() { + err := os.RemoveAll(filepath.Join(p.versioningDir, bucket)) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + debuglogger.Logf("failed to remove partially created bucket version directory (%q): %v", bucket, err) + } + } +} + +// mkdirAll is backend.MkdirAll with this backend's directory permissions and +// the same chown annotation CreateBucket uses: an EPERM here is the +// --chuid/--chgid misconfiguration, not a filesystem problem. Only EPERM is +// rewritten, so callers matching on EROFS or ErrObjectParentIsFile are +// unaffected. +func (p *Posix) mkdirAll(path string, uid, gid int, doChown bool) error { + err := backend.MkdirAll(path, uid, gid, doChown, p.newDirPerm) + if doChown && errors.Is(err, syscall.EPERM) { + return p.chownErr(path, uid, gid, err) + } + return err +} + +// chownErr annotates a chown failure with the configuration that asked for +// it. EPERM is not transient here: an unprivileged gateway can never give a +// file away to another uid or to a group it is not in, so every write for +// that account fails the same way until the configuration changes. Name the +// flags and the process identity that make the request impossible rather +// than leaving a bare "operation not permitted" in the log. +func (p *Posix) chownErr(name string, uid, gid int, err error) error { + // The errno is wrapped rather than the *fs.PathError it arrived in: the + // PathError repeats a path this message already names, and errors.Is + // still matches EPERM either way. + var errno syscall.Errno + if errors.As(err, &errno) && errno == syscall.EPERM { + return fmt.Errorf("chown %v to %v:%v: %w: --chuid/--chgid need a privileged gateway, but this one runs as euid %v/egid %v", + name, uid, gid, errno, p.euid, p.egid) + } + return fmt.Errorf("chown %v: %w", name, err) +} + +func (p *Posix) isBucketEmpty(bucket string) error { + if p.versioningEnabled() { + ents, err := os.ReadDir(filepath.Join(p.versioningDir, bucket)) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("readdir bucket: %w", err) + } + if err == nil { + if len(ents) == 1 && ents[0].Name() != MetaTmpDir { + return s3err.GetBucketErr(s3err.ErrVersionedBucketNotEmpty, bucket) + } else if len(ents) > 1 { + return s3err.GetBucketErr(s3err.ErrVersionedBucketNotEmpty, bucket) + } + } + } + + ents, err := os.ReadDir(p.BucketPath(bucket)) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("readdir bucket: %w", err) + } + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket) + } + if len(ents) == 1 && ents[0].Name() != MetaTmpDir { + return s3err.GetBucketErr(s3err.ErrBucketNotEmpty, bucket) + } else if len(ents) > 1 { + return s3err.GetBucketErr(s3err.ErrBucketNotEmpty, bucket) + } + + return nil +} + +func (p *Posix) DeleteBucket(ctx context.Context, bucket string) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + // Check if the bucket is empty + err = p.isBucketEmpty(bucket) + if err != nil { + return err + } + + // Remove the bucket + err = os.RemoveAll(p.BucketPath(bucket)) + if err != nil { + return fmt.Errorf("remove bucket: %w", err) + } + // Bucket data is already removed; a metadata cleanup failure orphans only + // the sidecar directory, which is not user-visible. Log and continue rather + // than returning an error that would mislead callers into thinking the + // bucket still exists. + if err = p.meta.DeleteAttributes(bucket, ""); err != nil { + debuglogger.Logf("failed to delete bucket sidecar attributes (%q): %v", bucket, err) + } + // Remove the bucket from versioning directory + if p.versioningEnabled() { + err = os.RemoveAll(filepath.Join(p.versioningDir, bucket)) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove bucket version: %w", err) + } + } + + return nil +} + +func (p *Posix) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + err = p.meta.StoreAttribute(nil, bucket, "", ownershipkey, []byte(ownership)) + if err != nil { + return fmt.Errorf("set ownership: %w", err) + } + + return nil +} +func (p *Posix) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return "", err + } + defer release() + + var ownship types.ObjectOwnership + if !p.isBucketValid(bucket) { + return ownship, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return ownship, err + } + + ownership, err := p.meta.RetrieveAttribute(nil, bucket, "", ownershipkey) + if errors.Is(err, meta.ErrNoSuchKey) { + return ownship, s3err.GetBucketErr(s3err.ErrOwnershipControlsNotFound, bucket) + } + if err != nil { + return ownship, fmt.Errorf("get bucket ownership status: %w", err) + } + + return types.ObjectOwnership(ownership), nil +} +func (p *Posix) DeleteBucketOwnershipControls(ctx context.Context, bucket string) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + err = p.meta.DeleteAttribute(bucket, "", ownershipkey) + if err != nil { + if errors.Is(err, meta.ErrNoSuchKey) { + return nil + } + + return fmt.Errorf("delete ownership: %w", err) + } + + return nil +} + +func (p *Posix) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + if !p.versioningEnabled() { + return s3err.GetAPIError(s3err.ErrVersioningNotConfigured) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + // Store 1 bit for bucket versioning state + var versioning []byte + switch status { + case types.BucketVersioningStatusEnabled: + // '1' maps to 'Enabled' + versioning = []byte{1} + case types.BucketVersioningStatusSuspended: + lockRaw, err := p.GetObjectLockConfiguration(withCtxNoSlot(ctx), bucket) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) { + return err + } + if err == nil { + lockStatus, err := auth.ParseBucketLockConfigurationOutput(lockRaw) + if err != nil { + return err + } + if lockStatus.ObjectLockEnabled == types.ObjectLockEnabledEnabled { + return s3err.GetAPIError(s3err.ErrSuspendedVersioningNotAllowed) + } + } + + // '0' maps to 'Suspended' + versioning = []byte{0} + } + + err = p.meta.StoreAttribute(nil, bucket, "", versioningKey, versioning) + if err != nil { + return fmt.Errorf("set versioning: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.GetBucketVersioningOutput{}, err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3response.GetBucketVersioningOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + err = p.doesBucketExist(bucket) + if err != nil { + return s3response.GetBucketVersioningOutput{}, err + } + + if !p.versioningEnabled() { + // No versioning directory is configured: report the bucket as + // never-versioned (200 with an empty configuration) to match AWS, + // instead of failing with VersioningNotConfigured. GUI clients + // (S3 Browser, CloudBerry, ...) poll this API while browsing + // buckets. PutBucketVersioning still rejects the request. + return s3response.GetBucketVersioningOutput{}, nil + } + + vData, err := p.meta.RetrieveAttribute(nil, bucket, "", versioningKey) + if errors.Is(err, meta.ErrNoSuchKey) { + return s3response.GetBucketVersioningOutput{}, nil + } else if err != nil { + return s3response.GetBucketVersioningOutput{}, fmt.Errorf("get bucket versioning config: %w", err) + } + + enabled, suspended := types.BucketVersioningStatusEnabled, types.BucketVersioningStatusSuspended + switch vData[0] { + case 1: + return s3response.GetBucketVersioningOutput{ + Status: &enabled, + }, nil + case 0: + return s3response.GetBucketVersioningOutput{ + Status: &suspended, + }, nil + } + + return s3response.GetBucketVersioningOutput{}, nil +} + +// Returns the specified bucket versioning status +func (p *Posix) getBucketVersioningStatus(ctx context.Context, bucket string) (types.BucketVersioningStatus, error) { + res, err := p.GetBucketVersioning(withCtxNoSlot(ctx), bucket) + if errors.Is(err, s3err.GetAPIError(s3err.ErrVersioningNotConfigured)) { + return "", nil + } + if err != nil { + return "", err + } + + if res.Status == nil { + return "", nil + } + + return *res.Status, nil +} + +// Checks if the given bucket versioning status is 'Enabled' +func (p *Posix) isBucketVersioningEnabled(s types.BucketVersioningStatus) bool { + return s == types.BucketVersioningStatusEnabled +} + +// Checks if the given bucket versioning status is 'Suspended' +func (p *Posix) isBucketVersioningSuspended(s types.BucketVersioningStatus) bool { + return s == types.BucketVersioningStatusSuspended +} + +// liveObjVersionId returns the version id to report for the current version +// of bucket/object. An object with no versionId attribute is the null +// version, but only a bucket with versioning configured has versions at all: +// in a bucket that was never versioned the object has no version id. +func (p *Posix) liveObjVersionId(ctx context.Context, bucket, object string) (string, error) { + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) + if err == nil { + return string(vId), nil + } + if !errors.Is(err, meta.ErrNoSuchKey) { + return "", fmt.Errorf("get obj versionId: %w", err) + } + + status, err := p.getBucketVersioningStatus(ctx, bucket) + if err != nil { + return "", fmt.Errorf("get bucket versioning status: %w", err) + } + if status == "" { + return "", nil + } + + return nullVersionId, nil +} + +// Generates the object version path in the versioning directory +func (p *Posix) genObjVersionPath(bucket, key string) string { + return filepath.Join(p.versioningDir, bucket, genObjVersionKey(key)) +} + +// Generates the versioning path for the given object key +func genObjVersionKey(key string) string { + sum := fmt.Sprintf("%x", sha256.Sum256([]byte(key))) + + return filepath.Join(sum[:2], sum[2:4], sum[4:6], sum) +} + +// Removes the null versionId object from versioning directory +func (p *Posix) deleteNullVersionIdObject(bucket, key string) error { + versionPath := filepath.Join(p.genObjVersionPath(bucket, key), nullVersionId) + + err := os.Remove(versionPath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return err + } + + _ = p.meta.DeleteAttributes(versionPath, "") + return nil +} + +func isRemovableAttr(attr string) bool { + switch attr { + case objectLegalHoldKey, objectRetentionKey: + return true + default: + return false + } +} + +// dirObjectAttrs are the attributes that make up the state of a directory +// object. The directory is kept across versions and is also the parent of +// other objects, so it may carry attributes that don't belong to the object: +// only these are copied into, restored from and cleared between versions. +// An attribute stored on objects must also be listed here. +var dirObjectAttrs = []string{ + etagkey, + checksumsKey, + contentTypeHdr, + contentEncHdr, + contentLangHdr, + contentDispHdr, + cacheCtrlHdr, + expiresHdr, + websiteRedirectHdr, + metadataHdr, + tagHdr, + objectLegalHoldKey, + objectRetentionKey, + versionIdKey, + deleteMarkerKey, +} + +// isDirObject reports whether the directory at bucket/key is a directory +// object: only directories created with a put carry an etag +func (p *Posix) isDirObject(bucket, key string) (bool, error) { + _, err := p.meta.RetrieveAttribute(nil, bucket, key, etagkey) + if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get dir etag: %w", err) + } + return true, nil +} + +// isLiveObject reports whether fi, the entry at the path of key, is the +// current version of key. A key and the same key with a trailing slash +// share one path and one set of attributes: a file there is the object of +// the key without the slash, a directory object the object of the key with +// it. +func (p *Posix) isLiveObject(fi os.FileInfo, bucket, key string) (bool, error) { + if fi.IsDir() != strings.HasSuffix(key, "/") { + return false, nil + } + if !fi.IsDir() { + return true, nil + } + return p.isDirObject(bucket, key) +} + +// statLiveObject returns the file info of the current version of key. An +// error matching fs.ErrNotExist is returned when key has no current +// version, including when the entry at its path is another key's object. +func (p *Posix) statLiveObject(bucket, key string) (os.FileInfo, error) { + fi, err := os.Stat(p.ObjectPath(bucket, key)) + if isErrNotDir(err) { + return nil, fs.ErrNotExist + } + if err != nil { + return nil, err + } + isObj, err := p.isLiveObject(fi, bucket, key) + if err != nil { + return nil, err + } + if !isObj { + return nil, fs.ErrNotExist + } + return fi, nil +} + +// objVersionAttrPath returns the bucket and object that the attributes of +// the version versionId of key are stored at: key itself for its current +// version, an entry of the versioning directory for other versions. An +// empty versionId selects the current version, and NoSuchKey is returned if +// key has none. +func (p *Posix) objVersionAttrPath(bucket, key, versionId string) (string, string, error) { + _, err := p.statLiveObject(bucket, key) + if isErrNameTooLong(err) { + return "", "", s3err.GetKeyTooLongErr(int64(len(key)), 1024) + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", "", fmt.Errorf("stat object: %w", err) + } + isLive := err == nil + + if versionId == "" { + if !isLive { + return "", "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + return bucket, key, nil + } + + if isLive { + vId, err := p.meta.RetrieveAttribute(nil, bucket, key, versionIdKey) + if errors.Is(err, meta.ErrNoSuchKey) { + vId = []byte(nullVersionId) + } else if err != nil { + return "", "", fmt.Errorf("get obj versionId: %w", err) + } + if string(vId) == versionId { + return bucket, key, nil + } + } + + return filepath.Join(p.versioningDir, bucket), filepath.Join(genObjVersionKey(key), versionId), nil +} + +// clearDirObjectAttrs removes the directory object attributes, including +// legacy metadata attributes, from the directory at bucket/key. The etag is +// kept: it marks the directory as an object, so a failure before the new +// attributes are stored leaves the object and its versions visible. +func (p *Posix) clearDirObjectAttrs(bucket, key string) error { + attrs, err := p.meta.ListAttributes(bucket, key) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("list object attributes: %w", err) + } + for _, attr := range attrs { + if isValidMeta(attr) { + err := p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + } + } + + for _, attr := range dirObjectAttrs { + if attr == etagkey { + continue + } + err := p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + } + return nil +} + +// Creates a new copy(version) of an object in the versioning directory +func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Account, removeAttributes bool) (versionPath string, err error) { + sf, err := os.Open(p.ObjectPath(bucket, key)) + if err != nil { + return "", err + } + defer sf.Close() + + var versionId string + data, err := p.meta.RetrieveAttribute(sf, bucket, key, versionIdKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return versionPath, fmt.Errorf("get object versionId: %w", err) + } + if err == nil { + versionId = string(data) + } else { + versionId = nullVersionId + } + + // a directory object version is an empty file carrying the + // directory object attributes + isDir := strings.HasSuffix(key, "/") + attrs := dirObjectAttrs + if isDir { + size = 0 + // store legacy metadata attributes as the metadata attribute + p.loadObjectMetadata(sf, bucket, key) + } else { + attrs, err = p.meta.ListAttributes(bucket, key) + if err != nil { + return versionPath, fmt.Errorf("load object attributes: %w", err) + } + } + + versionBucketPath := filepath.Join(p.versioningDir, bucket) + versioningKey := filepath.Join(genObjVersionKey(key), versionId) + versionTmpPath := filepath.Join(versionBucketPath, MetaTmpDir) + f, err := p.openTmpFile(versionTmpPath, versionBucketPath, versioningKey, + size, acc, doFalloc, p.forceNoTmpFile, odirectNotAllowed) + if err != nil { + return versionPath, err + } + defer f.cleanup() + + originalMTime := time.Now() + if srcInfo, err := sf.Stat(); err == nil { + originalMTime = srcInfo.ModTime() + } + + if !isDir { + // Prioritize copy_file_range for internal file-to-file version copies. + _, err = io.Copy(f.File(), sf) + if err != nil { + return versionPath, err + } + } + + // the version keeps the last modified time of the object + f.setModTime(originalMTime) + + versionPath = filepath.Join(versionBucketPath, versioningKey) + + err = os.MkdirAll(filepath.Join(versionBucketPath, genObjVersionKey(key)), p.newDirPerm) + if err != nil { + return versionPath, err + } + + // Copy the object attributes(metadata) + for _, attr := range attrs { + data, err := p.meta.RetrieveAttribute(sf, bucket, key, attr) + if isDir && errors.Is(err, meta.ErrNoSuchKey) { + continue + } + if err != nil { + return versionPath, fmt.Errorf("list %v attribute: %w", attr, err) + } + + err = p.meta.StoreAttribute(f.File(), versionPath, "", attr, data) + if err != nil { + return versionPath, fmt.Errorf("store %v attribute: %w", attr, err) + } + + // remove object lock attributes in delete marker + if removeAttributes && isRemovableAttr(attr) { + err := p.meta.DeleteAttribute(bucket, key, attr) + if err != nil { + return versionPath, fmt.Errorf("remove %s attribute: %w", attr, err) + } + } + } + + if versionId == nullVersionId { + // the null version id doesn't sort with the ulid version ids, so + // record its place in the version history: it's the current + // version, which makes it newer than the versions already in the + // versioning directory and older than any version created later + prev, err := newestObjVersionId(filepath.Join(versionBucketPath, genObjVersionKey(key))) + if err != nil { + return versionPath, fmt.Errorf("get newest object version: %w", err) + } + err = p.meta.StoreAttribute(f.File(), versionPath, "", nullVersionPrevKey, []byte(prev)) + if err != nil { + return versionPath, fmt.Errorf("store %v attribute: %w", nullVersionPrevKey, err) + } + } + + if err := f.link(); err != nil { + return versionPath, err + } + + return versionPath, nil +} + +func (p *Posix) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.ListVersionsResult{}, err + } + defer release() + + bucket := *input.Bucket + var prefix, delim, keyMarker, versionIdMarker string + var max int + + if !p.isBucketValid(bucket) { + return s3response.ListVersionsResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + if input.Prefix != nil { + prefix = *input.Prefix + } + if input.Delimiter != nil { + delim = *input.Delimiter + } + if input.KeyMarker != nil { + keyMarker = *input.KeyMarker + } + if input.VersionIdMarker != nil { + versionIdMarker = *input.VersionIdMarker + } + if input.MaxKeys != nil { + max = int(*input.MaxKeys) + } + + err = p.doesBucketExist(bucket) + if err != nil { + return s3response.ListVersionsResult{}, err + } + + fileSystem := os.DirFS(p.BucketPath(bucket)) + results, err := backend.WalkVersions(ctx, fileSystem, prefix, delim, keyMarker, versionIdMarker, max, + p.fileToObjVersions(bucket), []string{MetaTmpDir}) + if err != nil { + return s3response.ListVersionsResult{}, fmt.Errorf("walk %v: %w", bucket, err) + } + + return s3response.ListVersionsResult{ + CommonPrefixes: results.CommonPrefixes, + DeleteMarkers: results.DelMarkers, + Delimiter: &delim, + IsTruncated: &results.Truncated, + KeyMarker: &keyMarker, + MaxKeys: input.MaxKeys, + Name: input.Bucket, + NextKeyMarker: &results.NextMarker, + NextVersionIdMarker: &results.NextVersionIdMarker, + Prefix: &prefix, + VersionIdMarker: &versionIdMarker, + Versions: results.ObjectVersions, + }, nil +} + +func getBoolPtr(b bool) *bool { + return &b +} + +// ensureNotDeleteMarker return a `MethodNotAllowd` error +// if the provided object(version) is a delete marker +func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error { + if !p.versioningEnabled() { + return nil + } + + // With path-based metadata backends (e.g. sidecar), RetrieveAttribute + // returns ErrNoSuchKey whether the sidecar attribute is absent OR the + // data file simply doesn't exist — the two cases are indistinguishable + // from metadata alone. Verify the data file directly so callers + // receive the correct NoSuchVersion / NoSuchKey error. + if _, statErr := os.Stat(p.ObjectPath(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) { + if versionId != "" { + return s3err.GetAPIError(s3err.ErrNoSuchVersion) + } + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + _, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return s3err.GetNoSuchVersionErr(object, versionId) + } + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return nil + } + if err != nil { + return fmt.Errorf("get delete marker attr: %w", err) + } + + return s3err.GetAPIError(s3err.ErrMethodNotAllowed) +} + +// Check if the given object is a delete marker +func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) { + _, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + return false, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get object delete-marker: %w", err) + } + + return true, nil +} + +// checkCopySourceDeleteMarker rejects a copy whose source resolves to a +// delete marker: the key has no current version when the marker is the +// latest, and a marker named by version id holds no data to copy. +func (p *Posix) checkCopySourceDeleteMarker(bucket, object, versionId string) error { + isDel, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return err + } + if !isDel { + return nil + } + if versionId != "" { + return s3err.GetAPIError(s3err.ErrCopySourceDeleteMarker) + } + + return s3err.GetAPIError(s3err.ErrNoSuchKey) +} + +// Converts the file to object version. Finds all the object versions, +// delete markers from the versioning directory and returns +func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { + return func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*backend.ObjVersionFuncResult, error) { + var objects []s3response.ObjectVersion + var delMarkers []s3response.DeleteMarkerEntry + // if the number of available objects is 0, return truncated response + if availableObjCount <= 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + }, nil + } + // a directory is listed under the key with the trailing slash + key := path + if d.IsDir() { + key = path + "/" + } + + // get object info and fill out object data + etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if d.IsDir() && errors.Is(err, meta.ErrNoSuchKey) { + // a directory is listed only if it's a directory object + return nil, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + etag := string(etagBytes) + + // If the object doesn't have versionId, it's 'null' + versionId := "null" + versionIdBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, versionIdKey) + if err == nil { + versionId = string(versionIdBytes) + } + // the version id marker is the version listed last, so the + // listing continues with the version following it + if !*pastVersionIdMarker { + *pastVersionIdMarker = versionId == versionIdMarker + } else { + fi, err := d.Info() + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil { + return nil, fmt.Errorf("get fileinfo: %w", err) + } + + size := fi.Size() + if d.IsDir() { + // directory objects don't contain data + size = 0 + } + + isDel, err := p.isObjDeleteMarker(bucket, path) + if err != nil { + return nil, err + } + + if isDel { + delMarkers = append(delMarkers, s3response.DeleteMarkerEntry{ + IsLatest: getBoolPtr(true), + VersionId: &versionId, + LastModified: backend.GetTimePtr(fi.ModTime()), + Key: &key, + }) + } else { + // Retrieve checksum + checksum, err := p.retrieveChecksums(nil, bucket, path) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get checksum: %w", err) + } + + objects = append(objects, s3response.ObjectVersion{ + ETag: &etag, + Key: &key, + LastModified: backend.GetTimePtr(fi.ModTime()), + Size: &size, + VersionId: &versionId, + IsLatest: getBoolPtr(true), + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{checksum.Algorithm}, + ChecksumType: checksum.Type, + }) + } + + availableObjCount-- + if availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: versionId, + }, nil + } + } + + if !p.versioningEnabled() { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } + + // List all the versions of the object in the versioning directory + versionPath := p.genObjVersionPath(bucket, key) + dirEnts, err := os.ReadDir(versionPath) + if errors.Is(err, fs.ErrNotExist) { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } + if err != nil { + return nil, fmt.Errorf("read version dir: %w", err) + } + + if len(dirEnts) == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } + + // First find the null versionId object(if exists) + // before starting the object versions listing + var nullVersionIdObj *s3response.ObjectVersion + var nullObjDelMarker *s3response.DeleteMarkerEntry + var nullPos nullVersionPos + nf, err := os.Stat(filepath.Join(versionPath, nullVersionId)) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + if err == nil { + nullPos, err = p.getNullVersionPos(versionPath, nf) + if err != nil { + return nil, err + } + + isDel, err := p.isObjDeleteMarker(versionPath, nullVersionId) + if err != nil { + return nil, err + } + + // Check to see if the null versionId object is delete marker or not + if isDel { + nullObjDelMarker = &s3response.DeleteMarkerEntry{ + VersionId: backend.GetPtrFromString("null"), + LastModified: backend.GetTimePtr(nf.ModTime()), + Key: &key, + IsLatest: getBoolPtr(false), + } + } else { + etagBytes, err := p.meta.RetrieveAttribute(nil, versionPath, nullVersionId, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + etag := string(etagBytes) + size := nf.Size() + // Retrieve checksum + checksum, err := p.retrieveChecksums(nil, versionPath, nullVersionId) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get checksum: %w", err) + } + + nullVersionIdObj = &s3response.ObjectVersion{ + ETag: &etag, + Key: &key, + LastModified: backend.GetTimePtr(nf.ModTime()), + Size: &size, + VersionId: backend.GetPtrFromString("null"), + IsLatest: getBoolPtr(false), + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{ + checksum.Algorithm, + }, + ChecksumType: checksum.Type, + } + } + } + + isNullVersionIdObjFound := nullVersionIdObj != nil || nullObjDelMarker != nil + isNullVersionIdObjAdded := false + + // addNullVersion lists the null version at its place in the + // version history, or continues the listing after it when it's + // the version id marker. It returns the truncated result when + // the null version fills up the listing. + addNullVersion := func() *backend.ObjVersionFuncResult { + isNullVersionIdObjAdded = true + if !*pastVersionIdMarker { + *pastVersionIdMarker = versionIdMarker == nullVersionId + return nil + } + + if nullVersionIdObj != nil { + objects = append(objects, *nullVersionIdObj) + } + if nullObjDelMarker != nil { + delMarkers = append(delMarkers, *nullObjDelMarker) + } + + if availableObjCount--; availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: nullVersionId, + } + } + return nil + } + + for _, dEntry := range slices.Backward(dirEnts) { + + // Skip the null versionId object to not + // break the object versions list + if dEntry.Name() == nullVersionId { + continue + } + + f, err := dEntry.Info() + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return nil, fmt.Errorf("get fileinfo: %w", err) + } + + // If the null versionId object is found, first push it + // by checking its place in the version history, then + // continue the adding + if isNullVersionIdObjFound && !isNullVersionIdObjAdded && nullPos.isNewerThan(f) { + if res := addNullVersion(); res != nil { + return res, nil + } + } + versionId := f.Name() + size := f.Size() + + if !*pastVersionIdMarker { + if versionId == versionIdMarker { + *pastVersionIdMarker = true + } + continue + } + + etagBytes, err := p.meta.RetrieveAttribute(nil, versionPath, versionId, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return nil, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + etag := string(etagBytes) + + isDel, err := p.isObjDeleteMarker(versionPath, versionId) + if err != nil { + return nil, err + } + + if isDel { + delMarkers = append(delMarkers, s3response.DeleteMarkerEntry{ + VersionId: &versionId, + LastModified: backend.GetTimePtr(f.ModTime()), + Key: &key, + IsLatest: getBoolPtr(false), + }) + } else { + // Retrieve checksum + checksum, err := p.retrieveChecksums(nil, versionPath, versionId) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get checksum: %w", err) + } + objects = append(objects, s3response.ObjectVersion{ + ETag: &etag, + Key: &key, + LastModified: backend.GetTimePtr(f.ModTime()), + Size: &size, + VersionId: &versionId, + IsLatest: getBoolPtr(false), + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{checksum.Algorithm}, + ChecksumType: checksum.Type, + }) + } + + // if the available object count reaches to 0, return truncated response with nextVersionIdMarker + availableObjCount-- + if availableObjCount == 0 { + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + Truncated: true, + NextVersionIdMarker: versionId, + }, nil + } + } + + // If null versionId object is found but not yet pushed, + // push it after the listing, as it's the oldest object version + if isNullVersionIdObjFound && !isNullVersionIdObjAdded { + if res := addNullVersion(); res != nil { + return res, nil + } + } + + return &backend.ObjVersionFuncResult{ + ObjectVersions: objects, + DelMarkers: delMarkers, + }, nil + } +} + +func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.CreateMultipartUploadInput) (_ s3response.InitiateMultipartUploadResult, err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, err + } + defer release() + + if mpu.Key == nil { + return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + bucket := *mpu.Bucket + object := *mpu.Key + + if !p.isBucketValid(bucket) { + return s3response.InitiateMultipartUploadResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + err = p.doesBucketExist(bucket) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, err + } + + if strings.HasSuffix(*mpu.Key, "/") { + // directory objects can't be uploaded with multipart uploads + // because posix directories can't contain data + return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData) + } + + // parse object tags + tags, err := backend.ParseObjectTags(getString(mpu.Tagging)) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, err + } + + // generate random uuid for upload id + uploadID := uuid.New().String() + // hash object name for multipart container + objNameSum := sha256.Sum256([]byte(*mpu.Key)) + // multiple uploads for same object name allowed, + // they will all go into the same hashed name directory + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", objNameSum)) + tmppath := p.ObjectPath(bucket, objdir) + // the unique upload id is a directory for all of the parts + // associated with this specific multipart upload + err = os.MkdirAll(filepath.Join(tmppath, uploadID), p.newDirPerm) + if err != nil { + return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("create upload temp dir: %w", err) + } + + // set an attribute with the original object name so that we can + // map the hashed name back to the original object name + err = p.meta.StoreAttribute(nil, bucket, objdir, onameAttr, []byte(object)) + if err != nil { + // if we fail, cleanup the container directories + // but ignore errors because there might still be + // other uploads for the same object name outstanding + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("set name attr for upload: %w", err) + } + + // set object tagging + if tags != nil { + err := p.storeObjectTags(bucket, filepath.Join(objdir, uploadID), tags) + if err != nil { + // cleanup object if returning error + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) + return s3response.InitiateMultipartUploadResult{}, err + } + } + + err = p.storeObjectMetaProperties(nil, bucket, filepath.Join(objdir, uploadID), + metaProperties{ + ContentType: mpu.ContentType, + ContentEncoding: mpu.ContentEncoding, + ContentDisposition: mpu.ContentDisposition, + ContentLanguage: mpu.ContentLanguage, + CacheControl: mpu.CacheControl, + Expires: mpu.Expires, + WebsiteRedirectLocation: mpu.WebsiteRedirectLocation, + Metadata: mpu.Metadata, + }) + if err != nil { + // cleanup object if returning error + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) + return s3response.InitiateMultipartUploadResult{}, err + } + + // set object legal hold + if mpu.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { + err := p.isBucketObjectLockEnabled(bucket) + if err == nil { + err = p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), objectLegalHoldKey, []byte{1}) + } + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + // cleanup object if returning error + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) + return s3response.InitiateMultipartUploadResult{}, err + } + } + + // Set object retention + if mpu.ObjectLockMode != "" { + retention := types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionMode(mpu.ObjectLockMode), + RetainUntilDate: mpu.ObjectLockRetainUntilDate, + } + retParsed, err := json.Marshal(retention) + if err != nil { + // cleanup object if returning error + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) + return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("parse object lock retention: %w", err) + } + err = p.isBucketObjectLockEnabled(bucket) + if err == nil { + err = p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), objectRetentionKey, retParsed) + } + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + // cleanup object if returning error + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) + return s3response.InitiateMultipartUploadResult{}, err + } + } + + // Mark an upload with Object Lock parameters, so that each of its parts + // is required to carry an integrity check. They are its own lock + // headers, or the bucket's default retention rule as of now: a rule set + // or removed later doesn't change what the upload's parts need. + lockParams := mpu.ObjectLockLegalHoldStatus != "" || mpu.ObjectLockMode != "" + if !lockParams { + lockParams, err = p.bucketHasDefaultRetention(bucket) + if err != nil { + // cleanup object if returning error + _ = os.RemoveAll(filepath.Join(tmppath, uploadID)) + _ = os.Remove(tmppath) + _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) + return s3response.InitiateMultipartUploadResult{}, err + } + } + if lockParams { + err := p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), mpObjectLockKey, []byte{1}) + if err != nil { + // cleanup object if returning error + _ = os.RemoveAll(filepath.Join(tmppath, uploadID)) + _ = os.Remove(tmppath) + _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) + return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("store mp object lock: %w", err) + } + } + + // Set object checksum algorithm + if mpu.ChecksumAlgorithm != "" { + err := p.storeChecksums(nil, bucket, filepath.Join(objdir, uploadID), s3response.Checksum{ + Algorithm: mpu.ChecksumAlgorithm, + Type: mpu.ChecksumType, + }) + if err != nil { + // cleanup object if returning error + _ = os.RemoveAll(filepath.Join(tmppath, uploadID)) + _ = os.Remove(tmppath) + _ = p.meta.DeleteAttributes(bucket, filepath.Join(objdir, uploadID)) + return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("store mp checksum algorithm: %w", err) + } + } + + return s3response.InitiateMultipartUploadResult{ + Bucket: bucket, + Key: object, + UploadId: uploadID, + }, nil +} + +// getChownIDs returns the uid and gid that should be used for chowning +// the object to the account uid/gid. It also returns a boolean indicating +// if chowning is needed. +func (p *Posix) getChownIDs(acct auth.Account) (int, int, bool) { + uid := p.euid + gid := p.egid + var needsChown bool + if p.chownuid && acct.UserID != p.euid { + uid = acct.UserID + needsChown = true + } + if p.chowngid && acct.GroupID != p.egid { + gid = acct.GroupID + needsChown = true + } + + return uid, gid, needsChown +} + +func getPartChecksum(algo types.ChecksumAlgorithm, part types.CompletedPart) string { + switch algo { + case types.ChecksumAlgorithmCrc32: + return backend.GetStringFromPtr(part.ChecksumCRC32) + case types.ChecksumAlgorithmCrc32c: + return backend.GetStringFromPtr(part.ChecksumCRC32C) + case types.ChecksumAlgorithmSha1: + return backend.GetStringFromPtr(part.ChecksumSHA1) + case types.ChecksumAlgorithmSha256: + return backend.GetStringFromPtr(part.ChecksumSHA256) + case types.ChecksumAlgorithmCrc64nvme: + return backend.GetStringFromPtr(part.ChecksumCRC64NVME) + case types.ChecksumAlgorithmSha512: + return backend.GetStringFromPtr(part.ChecksumSHA512) + case types.ChecksumAlgorithmMd5: + return backend.GetStringFromPtr(part.ChecksumMD5) + case types.ChecksumAlgorithmXxhash64: + return backend.GetStringFromPtr(part.ChecksumXXHASH64) + case types.ChecksumAlgorithmXxhash3: + return backend.GetStringFromPtr(part.ChecksumXXHASH3) + case types.ChecksumAlgorithmXxhash128: + return backend.GetStringFromPtr(part.ChecksumXXHASH128) + default: + return "" + } +} + +func setUploadPartChecksum(res *s3.UploadPartOutput, algo types.ChecksumAlgorithm, sum *string) { + if sum == nil { + return + } + + switch algo { + case types.ChecksumAlgorithmCrc32: + res.ChecksumCRC32 = sum + case types.ChecksumAlgorithmCrc32c: + res.ChecksumCRC32C = sum + case types.ChecksumAlgorithmSha1: + res.ChecksumSHA1 = sum + case types.ChecksumAlgorithmSha256: + res.ChecksumSHA256 = sum + case types.ChecksumAlgorithmCrc64nvme: + res.ChecksumCRC64NVME = sum + case types.ChecksumAlgorithmSha512: + res.ChecksumSHA512 = sum + case types.ChecksumAlgorithmMd5: + res.ChecksumMD5 = sum + case types.ChecksumAlgorithmXxhash64: + res.ChecksumXXHASH64 = sum + case types.ChecksumAlgorithmXxhash3: + res.ChecksumXXHASH3 = sum + case types.ChecksumAlgorithmXxhash128: + res.ChecksumXXHASH128 = sum + } +} + +func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.CompleteMultipartUploadResult{}, "", err + } + defer release() + + return p.CompleteMultipartUploadWithCopy(ctx, input, nil) +} + +type onlyRead struct { + io.Reader +} + +// CustomCopyFunc implements copying/moving data from one file descriptor +// to another. The bool return signifies if function is idempotent. If true +// the system will allow clients to retry CompleteMultipartUpload. If +// false, then once a copy function is called successfully the upload will +// be completely aborted on any subsequent failure. +type CustomCopyFunc func(from *os.File, to *os.File) (bool, error) + +func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.CompleteMultipartUploadInput, customCopy CustomCopyFunc) (_ s3response.CompleteMultipartUploadResult, _ string, err error) { + defer backend.MapPermissionErr(&err) + + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + res := s3response.CompleteMultipartUploadResult{} + + if input.Key == nil { + return res, "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if input.UploadId == nil { + return res, "", s3err.GetNoSuchUploadErr("") + } + if input.MultipartUpload == nil { + return res, "", s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + bucket := *input.Bucket + object := *input.Key + uploadID := *input.UploadId + parts := input.MultipartUpload.Parts + + if !p.isBucketValid(bucket) { + return res, "", s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + err = p.doesBucketExist(bucket) + if err != nil { + return res, "", err + } + + if err := validateUploadID(uploadID); err != nil { + return res, "", err + } + + // Rename the upload directory to to atomically claim + // the processing slot. A concurrent call with the same uploadId will compute + // the same ETag, so it will either find the directory still present (still + // processing) or gone (already completed) and react accordingly. + sum := sha256.Sum256([]byte(object)) + objdirFull := filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) + uploadIDDir := filepath.Join(objdirFull, uploadID) + // Compute the default multipart ETag token used for claim naming. + // In standard mode this is the S3-compatible multipart MD5 ETag; in + // dataIntegrityEtag mode it may fall back to a deterministic claim token. + multipartClaimToken, err := backend.ComputeMultipartETagFromPartETags(parts) + if err != nil { + if !p.dataIntegrityEtag { + return res, "", err + } + + // dataIntegrityEtag allows non-MD5 multipart part ETags (for example, + // "CRC64NVME-"). In that mode, ComputeMultipartETagFromPartETags + // cannot decode part ETags as hex MD5 bytes. Build a deterministic + // claim token from part-number/etag pairs so concurrent complete calls + // still contend on the same in-progress directory name. + h := sha256.New() + for _, part := range parts { + if part.ETag == nil || part.PartNumber == nil { + return res, "", s3err.GetAPIError(s3err.ErrMalformedXML) + } + _, _ = h.Write([]byte(strconv.FormatInt(int64(*part.PartNumber), 10))) + _, _ = h.Write([]byte{':'}) + _, _ = h.Write([]byte(strings.Trim(*part.ETag, "\""))) + _, _ = h.Write([]byte{';'}) + } + multipartClaimToken = fmt.Sprintf("\"%x-%d\"", h.Sum(nil), len(parts)) + } + activeUploadName := fmt.Sprintf("%s.%s%s", uploadID, strings.Trim(multipartClaimToken, "\""), inProgressSuffix) + uploadIDInProgress := filepath.Join(objdirFull, activeUploadName) + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) + + predictDataIntegrityFinalETag := func(uploadName string) (string, error) { + checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadName)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return "", fmt.Errorf("get mp checksums: %w", err) + } + + mpChecksumType := checksums.Type + if checksums.Type == "" { + checksums.Type = types.ChecksumTypeFullObject + checksums.Algorithm = types.ChecksumAlgorithmCrc64nvme + } + + var compositeChecksumRdr *utils.CompositeChecksumReader + if checksums.Type == types.ChecksumTypeComposite { + compositeChecksumRdr, err = utils.NewCompositeChecksumReader(utils.HashType(strings.ToLower(string(checksums.Algorithm)))) + if err != nil { + return "", fmt.Errorf("initialize composite checksum reader: %w", err) + } + } + + var value string + var composableCsum string + for i, part := range parts { + if part.PartNumber == nil || part.ETag == nil { + return "", s3err.GetAPIError(s3err.ErrMalformedXML) + } + + partObjPath := filepath.Join(objdir, uploadName, fmt.Sprintf("%v", *part.PartNumber)) + fi, err := os.Lstat(p.ObjectPath(bucket, partObjPath)) + if err != nil { + return "", s3err.GetInvalidPartErr(uploadID, *part.PartNumber, backend.GetStringFromPtr(part.ETag)) + } + + switch checksums.Type { + case types.ChecksumTypeFullObject: + var pcs string + if mpChecksumType != "" { + pcs = getPartChecksum(checksums.Algorithm, part) + } else { + crc64nvme, err := p.meta.RetrieveAttribute(nil, bucket, partObjPath, partCrc64nvme) + if err != nil { + return "", fmt.Errorf("retrieve part internal crc64nvme: %w", err) + } + pcs = string(crc64nvme) + } + + if i == 0 { + composableCsum = pcs + } else { + composableCsum, err = utils.AddCRCChecksum(checksums.Algorithm, composableCsum, pcs, fi.Size()) + if err != nil { + return "", fmt.Errorf("add part %v checksum: %w", *part.PartNumber, err) + } + } + case types.ChecksumTypeComposite: + if err := compositeChecksumRdr.Process(getPartChecksum(checksums.Algorithm, part)); err != nil { + return "", fmt.Errorf("process %v part checksum: %w", *part.PartNumber, err) + } + } + } + + switch checksums.Type { + case types.ChecksumTypeComposite: + value = fmt.Sprintf("%s-%v", compositeChecksumRdr.Sum(), len(parts)) + case types.ChecksumTypeFullObject: + value = composableCsum + } + + if checksums.Algorithm == "" || value == "" { + return multipartClaimToken, nil + } + return fmt.Sprintf("\"%s-%s\"", strings.ToUpper(string(checksums.Algorithm)), value), nil + } + + err = os.Rename(uploadIDDir, uploadIDInProgress) + if errors.Is(err, fs.ErrNotExist) { + // Another call already claimed this slot and is still assembling the object. + if _, statErr := os.Stat(uploadIDInProgress); statErr == nil { + etag := multipartClaimToken + if p.dataIntegrityEtag { + etag, err = predictDataIntegrityFinalETag(activeUploadName) + if err != nil { + return res, "", err + } + } + // Still in progress — treat as success for idempotency. + return s3response.CompleteMultipartUploadResult{ + Bucket: &bucket, + ETag: &etag, + Key: &object, + }, "", nil + } + // Directory is gone: the concurrent call already completed and cleaned up. + // A directory at the object path is the object of the key with a + // trailing slash, not the completed upload. + if fi, statErr := os.Stat(p.ObjectPath(bucket, object)); statErr == nil && !fi.IsDir() { + etag := multipartClaimToken + if p.dataIntegrityEtag { + etagBytes, etagErr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + if etagErr != nil { + return res, "", fmt.Errorf("get object etag: %w", etagErr) + } + etag = string(etagBytes) + } + return s3response.CompleteMultipartUploadResult{ + Bucket: &bucket, + ETag: &etag, + Key: &object, + }, "", nil + } + + // Last resort: the object stat above may have lost a race with the + // concurrent call's link step. Check the mp-metadata xattr, as this + // multipart upload may have been finalized and the final object has been created + // before or by the racing request + if mpMetaBytes, statErr := p.meta.RetrieveAttribute(nil, bucket, object, mpMetaKey); statErr == nil { + mpMeta, err := backend.UnmarshalMpUploadMetadata(mpMetaBytes, false) + if err != nil { + return res, "", fmt.Errorf("parse object multipart metadata: %w", err) + } + + // The object may have been overwritten by a newer upload or + // it's the result of a completely different multipart upload; only + // treat it as our completion if the upload IDs match. + if mpMeta.UploadID != uploadID { + return res, "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + + etag := multipartClaimToken + if p.dataIntegrityEtag { + etagBytes, etagErr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + if etagErr != nil { + return res, "", fmt.Errorf("get object etag: %w", etagErr) + } + etag = string(etagBytes) + } + return s3response.CompleteMultipartUploadResult{ + Bucket: &bucket, + ETag: &etag, + Key: &object, + }, "", nil + } + + return res, "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + if err != nil { + return res, "", fmt.Errorf("rename upload to etag dir: %w", err) + } + + // Rename sidecar metadata to match the new data directory path. + // For xattr this is a no-op since attributes follow the inode. + metaObjDir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) + oldMetaObj := filepath.Join(metaObjDir, uploadID) + newMetaObj := filepath.Join(metaObjDir, activeUploadName) + if err := p.meta.RenameObject(bucket, oldMetaObj, newMetaObj); err != nil { + // Roll back the data directory rename so a future retry can succeed. + os.Rename(uploadIDInProgress, uploadIDDir) + return res, "", fmt.Errorf("rename metadata for in-progress: %w", err) + } + + // Best-effort rename back on failure so a future retry can still complete. + // On success, os.RemoveAll below removes uploadIDInProgress so this is a no-op. + defer os.Rename(uploadIDInProgress, uploadIDDir) + defer p.meta.RenameObject(bucket, newMetaObj, oldMetaObj) + + // Fast-fail precondition and directory checks before the parts are + // assembled. These are only advisory: the authoritative checks are + // repeated while holding the object publish lock just before the final + // link. + err = p.checkPutPreconditions(bucket, object, input.IfMatch, input.IfNoneMatch) + if err != nil { + return res, "", err + } + if d, err := os.Stat(p.ObjectPath(bucket, object)); err == nil && d.IsDir() { + return res, "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } + + checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, activeUploadName)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return res, "", fmt.Errorf("get mp checksums: %w", err) + } + + // ChecksumType should be the same as specified on CreateMultipartUpload + if input.ChecksumType != "" && checksums.Type != input.ChecksumType { + checksumType := checksums.Type + if checksumType == "" { + checksumType = types.ChecksumType("null") + } + + return res, "", s3err.GetChecksumTypeMismatchOnMpErr(checksumType) + } + + // mpChecksumType holds the multipart upload checksum type + mpChecksumType := checksums.Type + + // The checksum type/algorithm should default to FULL_OBJECT(crc64nvme) + if checksums.Type == "" { + checksums.Type = types.ChecksumTypeFullObject + checksums.Algorithm = types.ChecksumAlgorithmCrc64nvme + } + + // Initialize composite checksum reader + var compositeChecksumRdr *utils.CompositeChecksumReader + if checksums.Type == types.ChecksumTypeComposite { + compositeChecksumRdr, err = utils.NewCompositeChecksumReader(utils.HashType(strings.ToLower(string(checksums.Algorithm)))) + if err != nil { + return res, "", fmt.Errorf("initialize composite checksum reader: %w", err) + } + } + + // check all parts ok + last := len(parts) - 1 + var totalsize int64 + // cumulative byte offsets: partSizes[i] = sum of sizes of parts 1..i+1 + var partSizes []int64 + var composableCsum string + + // The initial value is the lower limit of partNumber: 0 + var partNumber int32 + for i, part := range parts { + if part.PartNumber == nil { + return res, "", s3err.GetAPIError(s3err.ErrMalformedXML) + } + if *part.PartNumber < 1 { + return res, "", s3err.GetInvalidArgumentErr(s3err.InvalidArgCompleteMpPartNumber, fmt.Sprint(*part.PartNumber)) + } + if *part.PartNumber <= partNumber { + return res, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder) + } + + partNumber = *part.PartNumber + + partObjPath := filepath.Join(objdir, activeUploadName, fmt.Sprintf("%v", *part.PartNumber)) + fullPartPath := p.ObjectPath(bucket, partObjPath) + fi, err := os.Lstat(fullPartPath) + if err != nil { + return res, "", s3err.GetInvalidPartErr(uploadID, *part.PartNumber, backend.GetStringFromPtr(part.ETag)) + } + + totalsize += fi.Size() + partSizes = append(partSizes, totalsize) + // all parts except the last need to be greater, than or equal to + // the minimum allowed size (5 Mib) + if i < last && fi.Size() < backend.MinPartSize { + return res, "", s3err.GetEntityTooSmallErr(fi.Size(), backend.MinPartSize) + } + + b, err := p.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey) + etag := string(b) + if err != nil { + etag = "" + } + if parts[i].ETag == nil { + return res, "", s3err.GetAPIError(s3err.ErrMalformedXML) + } + if !backend.AreEtagsSame(etag, *parts[i].ETag) { + return res, "", s3err.GetInvalidPartErr(uploadID, *part.PartNumber, etag) + } + + partChecksum, err := p.retrieveChecksums(nil, bucket, partObjPath) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return res, "", fmt.Errorf("get part checksum: %w", err) + } + + // If checksum has been provided on mp initialization + err = validatePartChecksum(partChecksum, part, uploadID) + if err != nil { + return res, "", err + } + + // Accumulate checksum state. + switch checksums.Type { + case types.ChecksumTypeFullObject: + var pcs string + if mpChecksumType != "" { + pcs = getPartChecksum(checksums.Algorithm, part) + } else { + crc64nvme, err := p.meta.RetrieveAttribute(nil, bucket, partObjPath, partCrc64nvme) + if err != nil { + return res, "", fmt.Errorf("retrieve part internal crc64nvme: %w", err) + } + pcs = string(crc64nvme) + } + if i == 0 { + composableCsum = pcs + } else { + composableCsum, err = utils.AddCRCChecksum(checksums.Algorithm, composableCsum, pcs, fi.Size()) + if err != nil { + return res, "", fmt.Errorf("add part %v checksum: %w", *part.PartNumber, err) + } + } + case types.ChecksumTypeComposite: + if err := compositeChecksumRdr.Process(getPartChecksum(checksums.Algorithm, part)); err != nil { + return res, "", fmt.Errorf("process %v part checksum: %w", *part.PartNumber, err) + } + } + } + + if input.MpuObjectSize != nil && totalsize != *input.MpuObjectSize { + return res, "", s3err.GetIncorrectMpObjectSizeErr(totalsize, *input.MpuObjectSize) + } + + // Compute the final checksum value. + var value string + switch checksums.Type { + case types.ChecksumTypeComposite: + value = fmt.Sprintf("%s-%v", compositeChecksumRdr.Sum(), len(parts)) + case types.ChecksumTypeFullObject: + value = composableCsum + } + + var crc32 *string + var crc32c *string + var sha1 *string + var sha256 *string + var crc64nvme *string + var sha512 *string + var md5sum *string + var xxhash64 *string + var xxhash3 *string + var xxhash128 *string + var gotSum *string + + switch checksums.Algorithm { + case types.ChecksumAlgorithmCrc32: + gotSum = input.ChecksumCRC32 + checksums.CRC32 = &value + crc32 = &value + case types.ChecksumAlgorithmCrc32c: + gotSum = input.ChecksumCRC32C + checksums.CRC32C = &value + crc32c = &value + case types.ChecksumAlgorithmSha1: + gotSum = input.ChecksumSHA1 + checksums.SHA1 = &value + sha1 = &value + case types.ChecksumAlgorithmSha256: + gotSum = input.ChecksumSHA256 + checksums.SHA256 = &value + sha256 = &value + case types.ChecksumAlgorithmCrc64nvme: + gotSum = input.ChecksumCRC64NVME + checksums.CRC64NVME = &value + crc64nvme = &value + case types.ChecksumAlgorithmSha512: + gotSum = input.ChecksumSHA512 + checksums.SHA512 = &value + sha512 = &value + case types.ChecksumAlgorithmMd5: + gotSum = input.ChecksumMD5 + checksums.MD5 = &value + md5sum = &value + case types.ChecksumAlgorithmXxhash64: + gotSum = input.ChecksumXXHASH64 + checksums.XXHASH64 = &value + xxhash64 = &value + case types.ChecksumAlgorithmXxhash3: + gotSum = input.ChecksumXXHASH3 + checksums.XXHASH3 = &value + xxhash3 = &value + case types.ChecksumAlgorithmXxhash128: + gotSum = input.ChecksumXXHASH128 + checksums.XXHASH128 = &value + xxhash128 = &value + } + + // Check if the provided checksum and the calculated one are the same. + if mpChecksumType != "" && gotSum != nil { + s := *gotSum + if checksums.Type == types.ChecksumTypeComposite && !strings.Contains(s, "-") { + s = fmt.Sprintf("%s-%v", s, len(parts)) + } + if s != value { + return res, "", s3err.GetChecksumBadDigestErr(checksums.Algorithm) + } + } + + // Determine the ETag that will be stored on the final object and returned + // to the client. By default this is the S3-compatible MD5-of-part-ETags + // value. When dataIntegrityEtag is enabled, embed the checksum algorithm + // and computed value instead: + // CRC64NVME full-object: "CRC64NVME-" + // composite other algo: "ALGO--" + // full-object other algo: "ALGO-" + finalEtag := multipartClaimToken + if p.dataIntegrityEtag && checksums.Algorithm != "" && value != "" { + finalEtag = fmt.Sprintf("\"%s-%s\"", strings.ToUpper(string(checksums.Algorithm)), value) + } + + f, err := p.openTmpFile(p.ObjectPath(bucket, MetaTmpDir), bucket, object, + totalsize, acct, skipFalloc, p.forceNoTmpFile, odirectNotAllowed) + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + return res, "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + return res, "", s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + return res, "", fmt.Errorf("open temp file: %w", err) + } + defer f.cleanup() + + var abortOnErrSet bool + for _, part := range parts { + partObjPath := filepath.Join(objdir, activeUploadName, fmt.Sprintf("%v", *part.PartNumber)) + fullPartPath := p.ObjectPath(bucket, partObjPath) + pf, err := os.Open(fullPartPath) + if err != nil { + return res, "", fmt.Errorf("open part %v: %v", *part.PartNumber, err) + } + + if customCopy != nil { + idemp, err := customCopy(pf, f.File()) + if err != nil { + // Fail back to standard copy + debuglogger.Logf("custom data block move failed (%q/%q): %v, failing back to io.Copy()", + bucket, object, err) + _, _ = f.File().Seek(0, io.SeekEnd) + if p.forceNoCopyFileRange { + _, err = io.Copy(f, &onlyRead{pf}) + } else { + // Keep both endpoints as *os.File here so + // io.Copy can use copy_file_range. + _, err = io.Copy(f.File(), pf) + } + } + if !idemp && err == nil { + // a successful non-idempotent call means we can no longer + // retry this upload. any failure needs to abort the complete + // upload. + if !abortOnErrSet { + defer func() { + // cleanup tmp dirs + os.RemoveAll(filepath.Join(p.BucketPath(bucket), objdir, activeUploadName)) + // use Remove for objdir in case there are still other + // uploads for same object name outstanding, this will + // fail if there are any + os.Remove(p.ObjectPath(bucket, objdir)) + }() + } + abortOnErrSet = true + } + } else { + if p.forceNoCopyFileRange { + _, err = io.Copy(f, &onlyRead{pf}) + } else { + // Keep both endpoints as *os.File here so + // io.Copy can use copy_file_range. + _, err = io.Copy(f.File(), pf) + } + } + pf.Close() + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + return res, "", s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + return res, "", s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + return res, "", fmt.Errorf("copy part %v: %v", part.PartNumber, err) + } + } + + // The parts are fully assembled into the staging file; serialize the + // commit phase (condition evaluation, metadata stores, and final link) + // with competing writers to the same key. + unlock, err := p.lockObjectPublish(ctx, bucket, object) + if err != nil { + return res, "", err + } + defer unlock() + + // authoritative precondition check under the publish lock + err = p.checkPutPreconditions(bucket, object, input.IfMatch, input.IfNoneMatch) + if err != nil { + return res, "", err + } + + upiddir := filepath.Join(objdir, activeUploadName) + + objname := p.ObjectPath(bucket, object) + dir := filepath.Dir(objname) + if dir != "" { + uid, gid, doChown := p.getChownIDs(acct) + err = p.mkdirAll(dir, uid, gid, doChown) + if err != nil { + return res, "", err + } + } + + vStatus, err := p.getBucketVersioningStatus(ctx, bucket) + if err != nil { + return res, "", err + } + vEnabled := p.isBucketVersioningEnabled(vStatus) + + d, err := os.Stat(objname) + if err == nil && d.IsDir() { + // the directory is the object of the key with a trailing slash, or + // the parent of other objects: its attributes are not this object's + return res, "", s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } + + // Snapshot the object that is about to be replaced before any metadata + // of the new object is stored: with path-based metadata (sidecar) the + // new object's attributes are written at the object path, so storing + // them first would archive them with the previous version. + err = p.snapshotObjVersion(bucket, object, vStatus, acct) + if err != nil { + return res, "", err + } + + // The new object starts with none of the replaced object's attributes, + // whether or not they were archived above: the null version and the + // unversioned object are replaced without a snapshot. + keep := []string{etagkey, checksumsKey, mpMetaKey} + if p.versioningEnabled() && vEnabled { + keep = append(keep, versionIdKey) + } + err = p.meta.ReplaceObject(bucket, object, keep) + if err != nil { + return res, "", fmt.Errorf("remove replaced object attributes: %w", err) + } + + // With versioning suspended the new object becomes the null version: + // it replaces the null version in the versioning directory, if any. + if p.isBucketVersioningSuspended(vStatus) { + err = p.deleteNullVersionIdObject(bucket, object) + if err != nil { + return res, "", err + } + } + + objMeta := p.loadObjectMetaProperties(nil, bucket, upiddir, nil) + err = p.storeObjectMetaProperties(f.File(), bucket, object, objMeta) + if err != nil { + return res, "", err + } + + // if the versioning is enabled, generate a new versionID for the object + var versionID string + if p.versioningEnabled() && vEnabled { + versionID = ulid.Make().String() + + err := p.meta.StoreAttribute(f.File(), bucket, object, versionIdKey, []byte(versionID)) + if err != nil { + return res, "", fmt.Errorf("set versionId attr: %w", err) + } + } + + // load and set tagging + tagging, err := p.meta.RetrieveAttribute(nil, bucket, upiddir, tagHdr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return res, "", fmt.Errorf("get object tagging: %w", err) + } + if err == nil { + err := p.meta.StoreAttribute(f.File(), bucket, object, tagHdr, tagging) + if err != nil { + return res, "", fmt.Errorf("set object tagging: %w", err) + } + } + + // load and set legal hold + lHold, err := p.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return res, "", fmt.Errorf("get object legal hold: %w", err) + } + if err == nil { + err := p.meta.StoreAttribute(f.File(), bucket, object, objectLegalHoldKey, lHold) + if err != nil { + return res, "", fmt.Errorf("set object legal hold: %w", err) + } + } + + err = p.storeChecksums(f.File(), bucket, object, checksums) + if err != nil { + return res, "", fmt.Errorf("store object checksum: %w", err) + } + + // load and set retention + ret, err := p.meta.RetrieveAttribute(nil, bucket, upiddir, objectRetentionKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return res, "", fmt.Errorf("get object retention: %w", err) + } + if err == nil { + err := p.meta.StoreAttribute(f.File(), bucket, object, objectRetentionKey, ret) + if err != nil { + return res, "", fmt.Errorf("set object retention: %w", err) + } + } + + err = p.meta.StoreAttribute(f.File(), bucket, object, etagkey, []byte(finalEtag)) + if err != nil { + return res, "", fmt.Errorf("set etag attr: %w", err) + } + + // Store multipart upload metadata on the final object so that GetObject / + // HeadObject can serve individual parts by part-number. + mpMeta := backend.MpUploadMetadata{UploadID: uploadID, Parts: partSizes} + mpMetaBytes, err := backend.MarshalMpUploadMetadata(mpMeta, false) + if err != nil { + return res, "", fmt.Errorf("marshal object multipart metadata: %w", err) + } + + err = p.meta.StoreAttribute(f.File(), bucket, object, mpMetaKey, mpMetaBytes) + if err != nil { + return res, "", fmt.Errorf("set object multipart metadata: %w", err) + } + + err = f.link() + if err != nil { + return res, "", fmt.Errorf("link object in namespace: %w", err) + } + + // cleanup tmp dirs + os.RemoveAll(filepath.Join(p.BucketPath(bucket), objdir, activeUploadName)) + // use Remove for objdir in case there are still other uploads + // for same object name outstanding, this will fail if there are any + os.Remove(p.ObjectPath(bucket, objdir)) + + return s3response.CompleteMultipartUploadResult{ + Bucket: &bucket, + ETag: &finalEtag, + Key: &object, + ChecksumCRC32: crc32, + ChecksumCRC32C: crc32c, + ChecksumSHA1: sha1, + ChecksumSHA256: sha256, + ChecksumCRC64NVME: crc64nvme, + ChecksumSHA512: sha512, + ChecksumMD5: md5sum, + ChecksumXXHASH64: xxhash64, + ChecksumXXHASH3: xxhash3, + ChecksumXXHASH128: xxhash128, + ChecksumType: &checksums.Type, + }, versionID, nil +} + +func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart, uploadId string) error { + n, argValue := numberOfChecksums(part) + if n > 1 { + return s3err.GetInvalidArgumentErr(s3err.InvalidArgChecksumPart, argValue) + } + if checksum.Algorithm == "" { + if n != 0 { + return s3err.GetInvalidPartErr(uploadId, *part.PartNumber, *part.ETag) + } + + return nil + } + + algo := checksum.Algorithm + if n == 0 { + return s3err.APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The upload was created using a %v checksum. The complete request must include the checksum for each part. It was missing for part %v in the request.", strings.ToLower(string(algo)), *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + + for _, cs := range []struct { + checksum *string + expectedChecksum string + algo types.ChecksumAlgorithm + }{ + {part.ChecksumCRC32, getString(checksum.CRC32), types.ChecksumAlgorithmCrc32}, + {part.ChecksumCRC32C, getString(checksum.CRC32C), types.ChecksumAlgorithmCrc32c}, + {part.ChecksumSHA1, getString(checksum.SHA1), types.ChecksumAlgorithmSha1}, + {part.ChecksumSHA256, getString(checksum.SHA256), types.ChecksumAlgorithmSha256}, + {part.ChecksumCRC64NVME, getString(checksum.CRC64NVME), types.ChecksumAlgorithmCrc64nvme}, + {part.ChecksumSHA512, getString(checksum.SHA512), types.ChecksumAlgorithmSha512}, + {part.ChecksumMD5, getString(checksum.MD5), types.ChecksumAlgorithmMd5}, + {part.ChecksumXXHASH64, getString(checksum.XXHASH64), types.ChecksumAlgorithmXxhash64}, + {part.ChecksumXXHASH3, getString(checksum.XXHASH3), types.ChecksumAlgorithmXxhash3}, + {part.ChecksumXXHASH128, getString(checksum.XXHASH128), types.ChecksumAlgorithmXxhash128}, + } { + if cs.checksum == nil { + continue + } + + if !utils.IsValidChecksum(*cs.checksum, cs.algo) { + return s3err.GetInvalidArgumentErr(s3err.InvalidArgChecksumPart, *cs.checksum) + } + + if *cs.checksum != cs.expectedChecksum { + if algo == cs.algo { + return s3err.GetInvalidPartErr(uploadId, *part.PartNumber, *part.ETag) + } + + return s3err.APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The %v you specified for part %v did not match what we received.", strings.ToLower(string(cs.algo)), *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + + return nil +} + +func numberOfChecksums(part types.CompletedPart) (int, string) { + counter := 0 + builder := &strings.Builder{} + + for _, ch := range []struct { + algo types.ChecksumAlgorithm + value *string + }{ + {types.ChecksumAlgorithmCrc32, part.ChecksumCRC32}, + {types.ChecksumAlgorithmCrc32c, part.ChecksumCRC32C}, + {types.ChecksumAlgorithmCrc64nvme, part.ChecksumCRC64NVME}, + {types.ChecksumAlgorithmSha1, part.ChecksumSHA1}, + {types.ChecksumAlgorithmSha256, part.ChecksumSHA256}, + } { + if getString(ch.value) != "" { + counter++ + fmt.Fprintf(builder, "%s:%s;", string(ch.algo), getString(ch.value)) + } + } + if getString(part.ChecksumSHA512) != "" { + counter++ + } + if getString(part.ChecksumMD5) != "" { + counter++ + } + if getString(part.ChecksumXXHASH64) != "" { + counter++ + } + if getString(part.ChecksumXXHASH3) != "" { + counter++ + } + if getString(part.ChecksumXXHASH128) != "" { + counter++ + } + + return counter, builder.String() +} + +// validateUploadID returns NoSuchUpload for an upload ID this backend never +// generates. Upload IDs are UUIDs in canonical form, and each one names its +// upload's directory, so an ID holding a path separator or ".." would +// otherwise reach a path outside the bucket. +func validateUploadID(uploadID string) error { + id, err := uuid.Parse(uploadID) + if err != nil || id.String() != uploadID { + return s3err.GetNoSuchUploadErr(uploadID) + } + return nil +} + +func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) { + if err := validateUploadID(uploadID); err != nil { + return [32]byte{}, err + } + + sum := sha256.Sum256([]byte(object)) + objdir := filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) + + _, err := os.Stat(filepath.Join(objdir, uploadID)) + if errors.Is(err, fs.ErrNotExist) { + return [32]byte{}, s3err.GetNoSuchUploadErr(uploadID) + } + if err != nil { + return [32]byte{}, fmt.Errorf("stat upload: %w", err) + } + return sum, nil +} + +type metaProperties struct { + ContentType *string + ContentEncoding *string + ContentDisposition *string + ContentLanguage *string + CacheControl *string + Expires *string + WebsiteRedirectLocation *string + Metadata map[string]string +} + +// loadObjectMetadata loads the given object metadata, if it fails to load +// it falls back to the old metadata key -> xattr key mapping mechanism +// and converts the old metadata storing schema with the new one +func (p *Posix) loadObjectMetadata(f *os.File, bucket, object string) map[string]string { + m, err := p.meta.RetrieveAttribute(f, bucket, object, metadataHdr) + if err != nil { + if errors.Is(err, meta.ErrNoSuchKey) { + // fallback to the deprecated mechanism + ents, err := p.meta.ListAttributes(bucket, object) + if err != nil || len(ents) == 0 { + return nil + } + result := map[string]string{} + + legacyAttrs := make([]string, 0, len(ents)) + for _, e := range ents { + if !isValidMeta(e) { + continue + } + b, err := p.meta.RetrieveAttribute(f, bucket, object, e) + if err != nil { + continue + } + if b == nil { + b = []byte{} + } + + legacyAttrs = append(legacyAttrs, e) + result[strings.TrimPrefix(e, fmt.Sprintf("%v.", oldMetaHdr))] = string(b) + } + + if len(result) != 0 { + err = p.storeObjectMetadata(f, bucket, object, result) + if err == nil { + // if it succeeded to store the metadata as a json + // object in user.metadata, cleanup the legacy X-Amz-Meta. + // attributes + for _, attr := range legacyAttrs { + _ = p.meta.DeleteAttribute(bucket, object, attr) + } + } + } + + return result + } + + return nil + } + + var metadata map[string]string + _ = json.Unmarshal(m, &metadata) + return metadata +} + +// loadObjectMetaProperties loads the given object meta properties +// including Content-Type, Cache-Control ... and the object Metadata +func (p *Posix) loadObjectMetaProperties(f *os.File, bucket, object string, fi *os.FileInfo) metaProperties { + var result metaProperties + + b, err := p.meta.RetrieveAttribute(f, bucket, object, contentTypeHdr) + if err == nil { + result.ContentType = backend.GetPtrFromString(string(b)) + } + + if (result.ContentType == nil || *result.ContentType == "") && fi != nil { + if (*fi).IsDir() { + // this is the media type for directories in AWS and Nextcloud + result.ContentType = backend.GetPtrFromString("application/x-directory") + } + } + + b, err = p.meta.RetrieveAttribute(f, bucket, object, contentEncHdr) + if err == nil { + result.ContentEncoding = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(f, bucket, object, contentDispHdr) + if err == nil { + result.ContentDisposition = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(f, bucket, object, contentLangHdr) + if err == nil { + result.ContentLanguage = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(f, bucket, object, cacheCtrlHdr) + if err == nil { + result.CacheControl = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(f, bucket, object, expiresHdr) + if err == nil { + result.Expires = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(f, bucket, object, websiteRedirectHdr) + if err == nil { + result.WebsiteRedirectLocation = backend.GetPtrFromString(string(b)) + } + + result.Metadata = p.loadObjectMetadata(f, bucket, object) + + return result +} + +// storeObjectMetadata parses the given metadata map[string]string +// to raw json to store in file/directory meta storage and checks/cleans up +// any legacy metadata persent in the object meta storage +func (p *Posix) storeObjectMetadata(f *os.File, bucket, object string, meta map[string]string) error { + if len(meta) == 0 { + return nil + } + + parsed, err := json.Marshal(meta) + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + + err = p.meta.StoreAttribute(f, bucket, object, metadataHdr, parsed) + if err != nil { + return err + } + + // cleanup any previously set legacy metadata + ents, err := p.meta.ListAttributes(bucket, object) + if err != nil || len(ents) == 0 { + return nil + } + + for _, ent := range ents { + if !isValidMeta(ent) { + continue + } + + _ = p.meta.DeleteAttribute(bucket, object, ent) + } + + return nil +} + +// storeObjectMetaProperties stores the object meta properties like: +// Content-Type, Content-Encoding, object Metadata ... +func (p *Posix) storeObjectMetaProperties(f *os.File, bucket, object string, m metaProperties) error { + if getString(m.ContentType) != "" { + err := p.meta.StoreAttribute(f, bucket, object, contentTypeHdr, []byte(*m.ContentType)) + if err != nil { + return fmt.Errorf("set content-type: %w", err) + } + } + if getString(m.ContentEncoding) != "" { + err := p.meta.StoreAttribute(f, bucket, object, contentEncHdr, []byte(*m.ContentEncoding)) + if err != nil { + return fmt.Errorf("set content-encoding: %w", err) + } + } + if getString(m.ContentDisposition) != "" { + err := p.meta.StoreAttribute(f, bucket, object, contentDispHdr, []byte(*m.ContentDisposition)) + if err != nil { + return fmt.Errorf("set content-disposition: %w", err) + } + } + if getString(m.ContentLanguage) != "" { + err := p.meta.StoreAttribute(f, bucket, object, contentLangHdr, []byte(*m.ContentLanguage)) + if err != nil { + return fmt.Errorf("set content-language: %w", err) + } + } + if getString(m.CacheControl) != "" { + err := p.meta.StoreAttribute(f, bucket, object, cacheCtrlHdr, []byte(*m.CacheControl)) + if err != nil { + return fmt.Errorf("set cache-control: %w", err) + } + } + if getString(m.Expires) != "" { + err := p.meta.StoreAttribute(f, bucket, object, expiresHdr, []byte(*m.Expires)) + if err != nil { + return fmt.Errorf("set expires: %w", err) + } + } + if getString(m.WebsiteRedirectLocation) != "" { + err := p.meta.StoreAttribute(f, bucket, object, websiteRedirectHdr, []byte(*m.WebsiteRedirectLocation)) + if err != nil { + return fmt.Errorf("set website-redirect-location: %w", err) + } + } + if m.Metadata != nil { + err := p.storeObjectMetadata(f, bucket, object, m.Metadata) + if err != nil { + return fmt.Errorf("set metadata: %w", err) + } + } + + return nil +} + +func isValidMeta(val string) bool { + return strings.HasPrefix(val, oldMetaHdr) +} + +func (p *Posix) AbortMultipartUpload(ctx context.Context, mpu *s3.AbortMultipartUploadInput) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if mpu.Key == nil { + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if mpu.UploadId == nil { + return s3err.GetNoSuchUploadErr("") + } + + bucket := *mpu.Bucket + object := *mpu.Key + uploadID := *mpu.UploadId + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + if err := validateUploadID(uploadID); err != nil { + return err + } + + sum := sha256.Sum256([]byte(object)) + objdir := filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) + + f, err := os.Stat(filepath.Join(objdir, uploadID)) + if err != nil { + return s3err.GetNoSuchUploadErr(uploadID) + } + + if mpu.IfMatchInitiatedTime != nil { + if mpu.IfMatchInitiatedTime.Unix() != f.ModTime().Unix() { + return s3err.GetPreconditionFailedErr(s3err.ConditionIfMatchInitiatedTime) + } + } + + err = os.RemoveAll(filepath.Join(objdir, uploadID)) + if err != nil { + return fmt.Errorf("remove multipart upload container: %w", err) + } + os.Remove(objdir) + + // Clean up sidecar metadata for the aborted upload. With xattr this is + // a no-op; with sidecar the metadata directory would otherwise be orphaned. + uploadMetaPath := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadID) + _ = p.meta.DeleteAttributes(bucket, uploadMetaPath) + + return nil +} + +func (p *Posix) ListMultipartUploads(ctx context.Context, mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.ListMultipartUploadsResult{}, err + } + defer release() + + var lmu s3response.ListMultipartUploadsResult + + bucket := *mpu.Bucket + + if !p.isBucketValid(bucket) { + return lmu, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + var delimiter string + if mpu.Delimiter != nil { + delimiter = *mpu.Delimiter + } + var prefix string + if mpu.Prefix != nil { + prefix = *mpu.Prefix + } + var keyMarker string + if mpu.KeyMarker != nil { + keyMarker = *mpu.KeyMarker + } + var uploadIDMarker string + if mpu.UploadIdMarker != nil { + uploadIDMarker = *mpu.UploadIdMarker + } + maxUploads := int(*mpu.MaxUploads) + + err = p.doesBucketExist(bucket) + if err != nil { + return lmu, err + } + + // ignore readdir error and use the empty list returned + objs, _ := os.ReadDir(p.ObjectPath(bucket, MetaTmpMultipartDir)) + + var uploads []s3response.Upload + + for _, obj := range objs { + if !obj.IsDir() { + continue + } + + b, err := p.meta.RetrieveAttribute(nil, bucket, filepath.Join(MetaTmpMultipartDir, obj.Name()), onameAttr) + if err != nil { + continue + } + objectName := string(b) + // filter by prefix + if prefix != "" && !strings.HasPrefix(objectName, prefix) { + continue + } + // filter by keyMarker + if keyMarker != "" && objectName <= keyMarker { + continue + } + + upids, err := os.ReadDir(filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, obj.Name())) + if err != nil { + continue + } + + for _, upid := range upids { + if !upid.IsDir() { + continue + } + // skip directories that are currently being completed + if strings.HasSuffix(upid.Name(), inProgressSuffix) { + continue + } + + fi, err := upid.Info() + if err != nil { + return lmu, fmt.Errorf("stat %q: %w", upid.Name(), err) + } + + uploadID := upid.Name() + + checksum, err := p.retrieveChecksums(nil, bucket, filepath.Join(MetaTmpMultipartDir, obj.Name(), uploadID)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return lmu, fmt.Errorf("get mp checksum: %w", err) + } + + uploads = append(uploads, s3response.Upload{ + Key: objectName, + UploadID: uploadID, + StorageClass: types.StorageClassStandard, + Initiated: fi.ModTime(), + ChecksumAlgorithm: checksum.Algorithm, + ChecksumType: checksum.Type, + }) + } + } + + // Sort once: Key asc, Initiated asc, UploadID asc + // UploadID is a tie-breaker for uploads that share both key and + // initiated time, since directory mtime resolution isn't always + // fine-grained enough to distinguish uploads created back to back. + sort.Slice(uploads, func(i, j int) bool { + if uploads[i].Key != uploads[j].Key { + return uploads[i].Key < uploads[j].Key + } + if !uploads[i].Initiated.Equal(uploads[j].Initiated) { + return uploads[i].Initiated.Before(uploads[j].Initiated) + } + return uploads[i].UploadID < uploads[j].UploadID + }) + + result, err := backend.ListMultipartUploads(uploads, prefix, delimiter, keyMarker, uploadIDMarker, maxUploads) + if err != nil { + return lmu, err + } + + return s3response.ListMultipartUploadsResult{ + Bucket: bucket, + Delimiter: delimiter, + KeyMarker: keyMarker, + MaxUploads: maxUploads, + Prefix: prefix, + NextKeyMarker: result.NextKeyMarker, + NextUploadIDMarker: result.NextUploadIDMarker, + UploadIDMarker: uploadIDMarker, + IsTruncated: result.IsTruncated, + Uploads: result.Uploads, + CommonPrefixes: result.CommonPrefixes, + }, nil +} + +func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.ListPartsResult{}, err + } + defer release() + + var lpr s3response.ListPartsResult + + if input.Key == nil { + return lpr, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if input.UploadId == nil { + return lpr, s3err.GetNoSuchUploadErr("") + } + + bucket := *input.Bucket + object := *input.Key + uploadID := *input.UploadId + + if !p.isBucketValid(bucket) { + return lpr, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + stringMarker := "" + if input.PartNumberMarker != nil { + stringMarker = *input.PartNumberMarker + } + + maxParts := int(*input.MaxParts) + + var partNumberMarker int + if stringMarker != "" { + var err error + partNumberMarker, err = strconv.Atoi(stringMarker) + if err != nil { + return lpr, s3err.GetInvalidArgMaxLimiter("part-number-marker", *input.PartNumberMarker) + } + } + + err = p.doesBucketExist(bucket) + if err != nil { + return lpr, err + } + + sum, err := p.checkUploadIDExists(bucket, object, uploadID) + if err != nil { + return lpr, err + } + + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) + tmpdir := p.ObjectPath(bucket, objdir) + + ents, err := os.ReadDir(filepath.Join(tmpdir, uploadID)) + if errors.Is(err, fs.ErrNotExist) { + return lpr, s3err.GetNoSuchUploadErr(uploadID) + } + if err != nil { + return lpr, fmt.Errorf("readdir upload: %w", err) + } + + checksum, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return lpr, fmt.Errorf("get mp checksum: %w", err) + } + if checksum.Algorithm == "" { + checksum.Algorithm = types.ChecksumAlgorithm("null") + } + if checksum.Type == "" { + checksum.Type = types.ChecksumType("null") + } + + parts := make([]s3response.Part, 0, len(ents)) + for i, e := range ents { + if i%128 == 0 { + select { + case <-ctx.Done(): + return s3response.ListPartsResult{}, ctx.Err() + default: + } + } + pn, err := strconv.Atoi(e.Name()) + if err != nil { + // file is not a valid part file + continue + } + if pn <= partNumberMarker { + continue + } + + partPath := filepath.Join(objdir, uploadID, e.Name()) + b, err := p.meta.RetrieveAttribute(nil, bucket, partPath, etagkey) + etag := string(b) + if err != nil { + etag = "" + } + + checksum, err := p.retrieveChecksums(nil, bucket, partPath) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + continue + } + + fi, err := os.Lstat(p.ObjectPath(bucket, partPath)) + if err != nil { + continue + } + + parts = append(parts, s3response.Part{ + PartNumber: pn, + ETag: etag, + LastModified: fi.ModTime(), + Size: fi.Size(), + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, + ChecksumCRC64NVME: checksum.CRC64NVME, + ChecksumSHA512: checksum.SHA512, + ChecksumMD5: checksum.MD5, + ChecksumXXHASH64: checksum.XXHASH64, + ChecksumXXHASH3: checksum.XXHASH3, + ChecksumXXHASH128: checksum.XXHASH128, + }) + } + + sort.Slice(parts, + func(i int, j int) bool { return parts[i].PartNumber < parts[j].PartNumber }) + + oldLen := len(parts) + if maxParts > 0 && len(parts) > maxParts { + parts = parts[:maxParts] + } + newLen := len(parts) + + nextpart := 0 + if len(parts) != 0 { + nextpart = parts[len(parts)-1].PartNumber + } + + return s3response.ListPartsResult{ + Bucket: bucket, + IsTruncated: oldLen != newLen, + Key: object, + MaxParts: maxParts, + NextPartNumberMarker: nextpart, + PartNumberMarker: partNumberMarker, + Parts: parts, + UploadID: uploadID, + StorageClass: types.StorageClassStandard, + ChecksumAlgorithm: checksum.Algorithm, + ChecksumType: checksum.Type, + }, nil +} + +type hashConfig struct { + value *string + hashType utils.HashType +} + +func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + return p.UploadPartWithPostFunc(ctx, input, func(*os.File) error { return nil }) +} + +func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPartInput, postprocess func(f *os.File) error) (_ *s3.UploadPartOutput, err error) { + defer backend.MapPermissionErr(&err) + + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + bucket := *input.Bucket + object := *input.Key + uploadID := *input.UploadId + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + part := input.PartNumber + length := int64(0) + if input.ContentLength != nil { + length = *input.ContentLength + } + r := input.Body + + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + if err := validateUploadID(uploadID); err != nil { + return nil, err + } + + sum := sha256.Sum256([]byte(object)) + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) + mpPath := filepath.Join(objdir, uploadID) + + _, err = os.Stat(p.ObjectPath(bucket, mpPath)) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetNoSuchUploadErr(uploadID) + } + if err != nil { + return nil, fmt.Errorf("stat uploadid: %w", err) + } + + // A part of an upload with Object Lock parameters needs an integrity + // check of its body + if !backend.UploadPartHasIntegrityCheck(input) { + _, err := p.meta.RetrieveAttribute(nil, bucket, mpPath, mpObjectLockKey) + if err == nil { + return nil, s3err.GetAPIError(s3err.ErrObjectLockPartChecksumRequired) + } + if !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get mp object lock: %w", err) + } + } + + partPath := filepath.Join(mpPath, fmt.Sprintf("%v", *part)) + + f, err := p.openTmpFile(p.ObjectPath(bucket, objdir), + bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile, odirectAllowed) + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + drainBody(r) + return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + drainBody(r) + return nil, s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + return nil, fmt.Errorf("open temp file: %w", err) + } + defer f.cleanup() + + // When dataIntegrityEtag is enabled, MD5 is never used for the part ETag, + // so skip both md5.New() and the per-byte TeeReader overhead. + var md5hash hash.Hash + var tr io.Reader + if p.dataIntegrityEtag { + tr = r + } else { + md5hash = md5.New() + tr = io.TeeReader(r, md5hash) + } + + chRdr, chunkUpload := input.Body.(middlewares.ChecksumReader) + isTrailingChecksum := chunkUpload && chRdr.Algorithm() != "" + + // user input checksum algorithm: either with chunk uploads or with request headers + var inputChAlgo utils.HashType + // user input checksum value specified with request headers + var inputSum string + + if !isTrailingChecksum { + hashConfigs := []hashConfig{ + {input.ChecksumCRC32, utils.HashTypeCRC32}, + {input.ChecksumCRC32C, utils.HashTypeCRC32C}, + {input.ChecksumSHA1, utils.HashTypeSha1}, + {input.ChecksumSHA256, utils.HashTypeSha256}, + {input.ChecksumCRC64NVME, utils.HashTypeCRC64NVME}, + {input.ChecksumSHA512, utils.HashTypeSha512}, + {input.ChecksumMD5, utils.HashTypeMd5}, + {input.ChecksumXXHASH64, utils.HashTypeXXHASH64}, + {input.ChecksumXXHASH3, utils.HashTypeXXHASH3}, + {input.ChecksumXXHASH128, utils.HashTypeXXHASH128}, + } + + for _, config := range hashConfigs { + if config.value != nil { + inputChAlgo = config.hashType + inputSum = *config.value + break + } + } + } else { + inputChAlgo = utils.HashType(chRdr.Algorithm()) + } + + exposeChecksum := inputChAlgo != "" + + checksums, err := p.retrieveChecksums(nil, bucket, mpPath) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("retrieve mp checksum: %w", err) + } + + // If checksum isn't provided for the part, + // but it has been provided on mp initialization + // and checksum type is 'COMPOSITE', return mismatch error + if inputChAlgo == "" && checksums.Type == types.ChecksumTypeComposite { + return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithm, "null") + } + + // Check if the provided checksum algorithm match + // the one specified on mp initialization + if inputChAlgo != "" && checksums.Type != "" { + algo := types.ChecksumAlgorithm(strings.ToUpper(string(inputChAlgo))) + if checksums.Algorithm != algo { + return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithm, algo) + } + } + + if inputChAlgo == "" { + // default to crc64nvme + inputChAlgo = utils.HashTypeCRC64NVME + } + + // hashreader is responsible to calculate and validate + // user input checksums + var hashRdr *utils.HashReader + // crc64nvmeRdr is used to calculate the object crc64nvme + // only for internal usage + var crc64nvmeRdr *utils.HashReader + + if checksums.Type == "" { + if inputChAlgo != utils.HashTypeCRC64NVME { + // if the input checksum algorithm isn't crc64nvme, create a + // crc64nvme reader no matter if checksum is + // received from chunk reader or request headers. + crc64nvmeRdr, err = utils.NewHashReader(tr, "", utils.HashTypeCRC64NVME) + if err != nil { + return nil, fmt.Errorf("initialize crc64nvme hash reader: %w", err) + } + tr = crc64nvmeRdr + } + + if !isTrailingChecksum { + // create a new hash reader for the user input checksum for calculation + // if the checksum doesn't come from chunk readers + hashRdr, err = utils.NewHashReader(tr, inputSum, inputChAlgo) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + } else { + // if no checksum algorithm or precalculated checksum is + // provided, but one has been on multipart upload initialization, + // anyways calculate and store the uploaded part checksum + if !isTrailingChecksum { + chAlgo := utils.HashType(strings.ToLower(string(checksums.Algorithm))) + hashRdr, err = utils.NewHashReader(tr, inputSum, chAlgo) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + // When dataIntegrityEtag is enabled and the mp algo isn't CRC64NVME, + // wrap the reader chain with an internal CRC64NVME reader so the part + // ETag can be CRC64NVME-based (trailing or non-trailing). + if p.dataIntegrityEtag && checksums.Algorithm != types.ChecksumAlgorithmCrc64nvme { + crc64nvmeRdr, err = utils.NewHashReader(tr, "", utils.HashTypeCRC64NVME) + if err != nil { + return nil, fmt.Errorf("initialize internal crc64nvme reader: %w", err) + } + tr = crc64nvmeRdr + } + } + + buf := p.getIOBuffer() + defer p.putIOBuffer(buf) + + _, err = f.copyFrom(tr, buf) + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + drainBody(tr) + return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + drainBody(tr) + return nil, s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + // Return the error itself, if it implements the s3err.S3Error interface + if _, ok := err.(s3err.S3Error); ok { + return nil, err + } + return nil, fmt.Errorf("write part data: %w", err) + } + + // Generate the part ETag. + // When dataIntegrityEtag is enabled, use CRC64NVME instead of MD5. + // CRC64NVME is available from: hashRdr/chRdr when the mp or input algo + // is already CRC64NVME, or from crc64nvmeRdr which is always set up in + // the non-CRC64NVME paths (checksums.Type=="" branch) or was just added + // above in the storeChecksum branch. + var etag string + if p.dataIntegrityEtag { + var crc64Sum string + isCrc64 := checksums.Algorithm == types.ChecksumAlgorithmCrc64nvme || + (checksums.Type == "" && inputChAlgo == utils.HashTypeCRC64NVME) + if isCrc64 { + if isTrailingChecksum { + crc64Sum = chRdr.Checksum() + } else { + crc64Sum = hashRdr.Sum() + } + } else { + crc64Sum = crc64nvmeRdr.Sum() + } + etag = fmt.Sprintf("\"CRC64NVME-%s\"", crc64Sum) + } else { + etag = backend.GenerateEtag(md5hash) + } + err = p.meta.StoreAttribute(f.File(), bucket, partPath, etagkey, []byte(etag)) + if err != nil { + return nil, fmt.Errorf("set etag attr: %w", err) + } + + res := &s3.UploadPartOutput{ + ETag: &etag, + } + + // if a checksum algorithm has been provided on mp initiation + // the checksums should be stored, otherwise only returned + // in the response without storing + storeChecksum := checksums.Type != "" + + if storeChecksum { + checksum := s3response.Checksum{ + Algorithm: checksums.Algorithm, + } + + var sum string + if isTrailingChecksum { + sum = chRdr.Checksum() + } + if hashRdr != nil { + sum = hashRdr.Sum() + } + + checksum.SetSum(checksums.Algorithm, &sum) + setUploadPartChecksum(res, checksums.Algorithm, &sum) + + err := p.storeChecksums(f.File(), bucket, partPath, checksum) + if err != nil { + return nil, fmt.Errorf("store checksum: %w", err) + } + } else { + var internalCrc64NvmeSum string + if inputChAlgo == utils.HashTypeCRC64NVME { + if isTrailingChecksum { + internalCrc64NvmeSum = chRdr.Checksum() + } else { + internalCrc64NvmeSum = hashRdr.Sum() + } + } else { + internalCrc64NvmeSum = crc64nvmeRdr.Sum() + } + + err := p.meta.StoreAttribute(f.File(), bucket, partPath, partCrc64nvme, []byte(internalCrc64NvmeSum)) + if err != nil { + return nil, fmt.Errorf("store part internal crc64nvme: %w", err) + } + + if exposeChecksum { + var sumToReturn string + if isTrailingChecksum { + sumToReturn = chRdr.Checksum() + } else { + sumToReturn = hashRdr.Sum() + } + + switch inputChAlgo { + case utils.HashTypeCRC32: + res.ChecksumCRC32 = &sumToReturn + case utils.HashTypeCRC32C: + res.ChecksumCRC32C = &sumToReturn + case utils.HashTypeSha1: + res.ChecksumSHA1 = &sumToReturn + case utils.HashTypeSha256: + res.ChecksumSHA256 = &sumToReturn + case utils.HashTypeCRC64NVME: + res.ChecksumCRC64NVME = &sumToReturn + case utils.HashTypeSha512: + res.ChecksumSHA512 = &sumToReturn + case utils.HashTypeMd5: + res.ChecksumMD5 = &sumToReturn + case utils.HashTypeXXHASH64: + res.ChecksumXXHASH64 = &sumToReturn + case utils.HashTypeXXHASH3: + res.ChecksumXXHASH3 = &sumToReturn + case utils.HashTypeXXHASH128: + res.ChecksumXXHASH128 = &sumToReturn + } + } + } + + err = postprocess(f.File()) + if err != nil { + return nil, fmt.Errorf("upload part post process failed: %w", err) + } + + err = f.link() + if err != nil { + return nil, fmt.Errorf("link object in namespace: %w", err) + } + + return res, nil +} + +func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) (_ s3response.CopyPartResult, err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.CopyPartResult{}, err + } + defer release() + + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + if upi.Key == nil { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + if !p.isBucketValid(*upi.Bucket) { + return s3response.CopyPartResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, *upi.Bucket) + } + + err = p.doesBucketExist(*upi.Bucket) + if err != nil { + return s3response.CopyPartResult{}, err + } + + if err := validateUploadID(*upi.UploadId); err != nil { + return s3response.CopyPartResult{}, err + } + + sum := sha256.Sum256([]byte(*upi.Key)) + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) + + _, err = os.Stat(filepath.Join(p.BucketPath(*upi.Bucket), objdir, *upi.UploadId)) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyPartResult{}, s3err.GetNoSuchUploadErr(*upi.UploadId) + } + if isErrNameTooLong(err) { + return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(*upi.Key)), 1024) + } + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("stat uploadid: %w", err) + } + + partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber)) + + srcBucket, srcObject, srcVersionId, err := backend.ParseCopySource(*upi.CopySource) + if err != nil { + return s3response.CopyPartResult{}, err + } + if err := p.validateVersionId(srcVersionId); err != nil { + return s3response.CopyPartResult{}, err + } + if !p.isBucketValid(srcBucket) { + return s3response.CopyPartResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, srcBucket) + } + + err = p.doesBucketExist(srcBucket) + if err != nil { + return s3response.CopyPartResult{}, err + } + + if upi.ExpectedSourceBucketOwner != nil && *upi.ExpectedSourceBucketOwner != "" { + aclData, err := p.meta.RetrieveAttribute(nil, srcBucket, "", aclkey) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("get src bucket acl: %w", err) + } + srcAcl, err := auth.ParseACL(aclData) + if err != nil { + return s3response.CopyPartResult{}, err + } + if srcAcl.Owner != *upi.ExpectedSourceBucketOwner { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrAccessDenied) + } + } + + vStatus, err := p.getBucketVersioningStatus(ctx, srcBucket) + if err != nil { + return s3response.CopyPartResult{}, err + } + vEnabled := p.isBucketVersioningEnabled(vStatus) + + if srcVersionId != "" { + if !p.versioningEnabled() || !vEnabled { + return s3response.CopyPartResult{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, srcVersionId) + } + vId, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, versionIdKey) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyPartResult{}, fmt.Errorf("get src object version id: %w", err) + } + + if string(vId) != srcVersionId { + srcBucket = filepath.Join(p.versioningDir, srcBucket) + srcObject = filepath.Join(genObjVersionKey(srcObject), srcVersionId) + } + } + + objPath := p.ObjectPath(srcBucket, srcObject) + fi, err := os.Stat(objPath) + if errors.Is(err, fs.ErrNotExist) { + if p.versioningEnabled() && vEnabled { + return s3response.CopyPartResult{}, s3err.GetNoSuchVersionErr(srcObject, srcVersionId) + } + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if isErrNameTooLong(err) { + return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcObject)), 1024) + } + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("stat object: %w", err) + } + if strings.HasSuffix(srcObject, "/") != fi.IsDir() { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err := p.checkCopySourceDeleteMarker(srcBucket, srcObject, srcVersionId); err != nil { + return s3response.CopyPartResult{}, err + } + // a directory object holds no data + srcSize := fi.Size() + if fi.IsDir() { + isObj, err := p.isDirObject(srcBucket, srcObject) + if err != nil { + return s3response.CopyPartResult{}, err + } + if !isObj { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + srcSize = 0 + } + + startOffset, length, err := backend.ParseCopySourceRange(srcSize, *upi.CopySourceRange) + if err != nil { + return s3response.CopyPartResult{}, err + } + if length > p.copyObjectThreshold { + return s3response.CopyPartResult{}, s3err.GetCopySourceObjectTooLargeErr(p.copyObjectThreshold) + } + + srcf, err := os.Open(objPath) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err) + } + defer srcf.Close() + + // evaluate preconditions + b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey) + srcEtag := string(b) + if err != nil || srcEtag == "" { + srcEtag = p.defaultEtag + } + + err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{ + IfMatch: upi.CopySourceIfMatch, + IfNoneMatch: upi.CopySourceIfNoneMatch, + IfModSince: upi.CopySourceIfModifiedSince, + IfUnmodeSince: upi.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyPartResult{}, err + } + + f, err := p.openTmpFile(p.ObjectPath(*upi.Bucket, objdir), + *upi.Bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile, odirectNotAllowed) + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + return s3response.CopyPartResult{}, fmt.Errorf("open temp file: %w", err) + } + defer f.cleanup() + + rdr := io.NewSectionReader(srcf, startOffset, length) + var md5hash hash.Hash + var tr io.Reader = rdr + if !p.dataIntegrityEtag { + md5hash = md5.New() + tr = io.TeeReader(rdr, md5hash) + } + + mpChecksums, err := p.retrieveChecksums(nil, *upi.Bucket, filepath.Join(objdir, *upi.UploadId)) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyPartResult{}, fmt.Errorf("retrieve mp checksums: %w", err) + } + + checksums, err := p.retrieveChecksums(nil, srcBucket, srcObject) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyPartResult{}, fmt.Errorf("retrieve object part checksums: %w", err) + } + + // TODO: Should the checksum be recalculated or just copied ? + var hashRdr *utils.HashReader + var crc64nvmeRdr *utils.HashReader + if mpChecksums.Algorithm != "" { + if checksums.Algorithm == "" || mpChecksums.Algorithm != checksums.Algorithm { + hashRdr, err = utils.NewHashReader(tr, "", utils.HashType(strings.ToLower(string(mpChecksums.Algorithm)))) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } + } else { + // if no checksum has been specified on multipart upload initiation + // create an internal crc64nvme reader to calculate and stored the internal crc64nvme + crc64nvmeRdr, err = utils.NewHashReader(tr, "", utils.HashTypeCRC64NVME) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("initialize internal crc64nvme reader: %w", err) + } + tr = crc64nvmeRdr + } + + if p.dataIntegrityEtag && crc64nvmeRdr == nil { + // still need a crc64nvme-derived ETag even though a different checksum is already being computed + crc64nvmeRdr, err = utils.NewHashReader(tr, "", utils.HashTypeCRC64NVME) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("initialize etag crc64nvme reader: %w", err) + } + tr = crc64nvmeRdr + } + + _, err = io.Copy(f, tr) + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + return s3response.CopyPartResult{}, fmt.Errorf("copy part data: %w", err) + } + + if checksums.Algorithm != "" { + if mpChecksums.Algorithm == "" { + checksums = s3response.Checksum{} + } else { + if hashRdr == nil { + err := p.storeChecksums(f.File(), *upi.Bucket, partPath, checksums) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("store part checksum: %w", err) + } + } + } + } + + if hashRdr != nil { + algo := types.ChecksumAlgorithm(strings.ToUpper(string(hashRdr.Type()))) + checksums = s3response.Checksum{ + Algorithm: algo, + } + + sum := hashRdr.Sum() + checksums.SetSum(algo, &sum) + + err := p.storeChecksums(f.File(), *upi.Bucket, partPath, checksums) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("store part checksum: %w", err) + } + } + + if crc64nvmeRdr != nil { + // store the internal crc64nvme + internalCrc64NvmeSum := crc64nvmeRdr.Sum() + err := p.meta.StoreAttribute(f.File(), *upi.Bucket, partPath, partCrc64nvme, []byte(internalCrc64NvmeSum)) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("store part internal crc64nvme: %w", err) + } + } + + var etag string + if p.dataIntegrityEtag { + etag = fmt.Sprintf("\"CRC64NVME-%s\"", crc64nvmeRdr.Sum()) + } else { + etag = backend.GenerateEtag(md5hash) + } + err = p.meta.StoreAttribute(f.File(), *upi.Bucket, partPath, etagkey, []byte(etag)) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("set etag attr: %w", err) + } + + err = f.link() + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("link object in namespace: %w", err) + } + + fi, err = os.Stat(p.ObjectPath(*upi.Bucket, partPath)) + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("stat part path: %w", err) + } + + return s3response.CopyPartResult{ + ETag: &etag, + LastModified: fi.ModTime(), + CopySourceVersionId: srcVersionId, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, + }, nil +} + +// getEmptyChecksumValue returns the base64-encoded checksum +// for an empty payload for the given algorithm defaulting to crc64nvme +func getEmptyChecksumValue(algo types.ChecksumAlgorithm) string { + switch algo { + case types.ChecksumAlgorithmCrc32: + return "AAAAAA==" + case types.ChecksumAlgorithmCrc32c: + return "AAAAAA==" + case types.ChecksumAlgorithmSha1: + return "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" + case types.ChecksumAlgorithmSha256: + return "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + case types.ChecksumAlgorithmSha512: + return "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" + case types.ChecksumAlgorithmMd5: + return "1B2M2Y8AsgTpgAmY7PhCfg==" + case types.ChecksumAlgorithmXxhash64: + return "70bbN1HY6Zk=" + case types.ChecksumAlgorithmXxhash3: + return "LQaABTjTlMI=" + case types.ChecksumAlgorithmXxhash128: + return "maoG0wFHmNhgAcMkRo1Jfw==" + default: + // default to crc64nvme + return "AAAAAAAAAAA=" + } +} + +// checkPutPreconditions evaluates the conditional write headers against the +// object's current etag. Callers that publish an object must repeat this +// check while holding the object publish lock (lockObjectPublish) so that +// evaluation and publication are atomic per bucket/key. +func (p *Posix) checkPutPreconditions(bucket, object string, ifMatch, ifNoneMatch *string) error { + if ifMatch == nil && ifNoneMatch == nil { + return nil + } + if p.objectLockMode == ObjectLockModeNone { + return s3err.GetAPIError(s3err.ErrNotImplemented) + } + + // the etag at the object path may be the one of the key with or + // without the trailing slash + _, err := p.statLiveObject(bucket, object) + if errors.Is(err, fs.ErrNotExist) { + return backend.EvaluateObjectPutPreconditions("", ifMatch, ifNoneMatch, false) + } + + etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + if err == nil || errors.Is(err, fs.ErrNotExist) || errors.Is(err, meta.ErrNoSuchKey) { + return backend.EvaluateObjectPutPreconditions(string(etagBytes), ifMatch, ifNoneMatch, err == nil) + } + + return nil +} + +// snapshotObjVersion copies the current object at bucket/key into the +// versioning directory before the object is replaced, if versioning is +// configured for the bucket and there is an object to snapshot. +func (p *Posix) snapshotObjVersion(bucket, key string, vStatus types.BucketVersioningStatus, acct auth.Account) error { + if !p.versioningEnabled() || vStatus == "" { + return nil + } + + isDir := strings.HasSuffix(key, "/") + d, err := os.Stat(p.ObjectPath(bucket, key)) + if err != nil || d.IsDir() != isDir { + // nothing to snapshot + return nil + } + if isDir { + isObj, err := p.isDirObject(bucket, key) + if err != nil { + return err + } + if !isObj { + // nothing to snapshot + return nil + } + } + + var isVersionIdMissing bool + if p.isBucketVersioningSuspended(vStatus) { + vIdBytes, err := p.meta.RetrieveAttribute(nil, bucket, key, versionIdKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("get object versionId: %w", err) + } + isVersionIdMissing = len(vIdBytes) == 0 + } + if !isVersionIdMissing { + _, err := p.createObjVersion(bucket, key, d.Size(), acct, false) + if err != nil { + return fmt.Errorf("create object version: %w", err) + } + } + + return nil +} + +func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.PutObjectOutput{}, err + } + defer release() + + return p.PutObjectWithPostFunc(ctx, po, func(*os.File) error { return nil }) +} + +func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObjectInput, postprocess func(f *os.File) error) (_ s3response.PutObjectOutput, err error) { + defer backend.MapPermissionErr(&err) + + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + if po.Key == nil { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if !p.isBucketValid(*po.Bucket) { + return s3response.PutObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, *po.Bucket) + } + err = p.doesBucketExist(*po.Bucket) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + tags, err := backend.ParseObjectTags(getString(po.Tagging)) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + name := p.ObjectPath(*po.Bucket, *po.Key) + + // Fast-fail precondition check before the request body is staged. This + // is only advisory: the authoritative check is repeated while holding + // the object publish lock at commit time so that evaluation and + // publication are atomic under concurrent writers. + err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + uid, gid, doChown := p.getChownIDs(acct) + + contentLength := int64(0) + if po.ContentLength != nil { + contentLength = *po.ContentLength + } + + chRdr, chunkUpload := po.Body.(middlewares.ChecksumReader) + isTrailingChecksum := chunkUpload && chRdr.Algorithm() != "" + + checksumAlgorithm := po.ChecksumAlgorithm + checksumValue := "" + + if isTrailingChecksum { + checksumAlgorithm = types.ChecksumAlgorithm(strings.ToUpper(chRdr.Algorithm())) + checksumValue = chRdr.Checksum() + } else { + hashConfigs := []hashConfig{ + {po.ChecksumCRC32, utils.HashTypeCRC32}, + {po.ChecksumCRC32C, utils.HashTypeCRC32C}, + {po.ChecksumSHA1, utils.HashTypeSha1}, + {po.ChecksumSHA256, utils.HashTypeSha256}, + {po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME}, + {po.ChecksumSHA512, utils.HashTypeSha512}, + {po.ChecksumMD5, utils.HashTypeMd5}, + {po.ChecksumXXHASH64, utils.HashTypeXXHASH64}, + {po.ChecksumXXHASH3, utils.HashTypeXXHASH3}, + {po.ChecksumXXHASH128, utils.HashTypeXXHASH128}, + } + + for _, config := range hashConfigs { + if config.value != nil { + checksumAlgorithm = types.ChecksumAlgorithm(strings.ToUpper(string(config.hashType))) + checksumValue = *config.value + break + } + } + + if checksumAlgorithm == "" { + // default to crc64nvme + checksumAlgorithm = types.ChecksumAlgorithmCrc64nvme + } + } + + if strings.HasSuffix(*po.Key, "/") { + // object is directory + if contentLength != 0 { + // posix directories can't contain data, send error + // if reuests has a data payload associated with a + // directory object + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData) + } + + // serialize with competing writers to the same key and repeat the + // precondition check authoritatively under the publish lock + unlock, err := p.lockObjectPublish(ctx, *po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + defer unlock() + + err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + expectedSum := getEmptyChecksumValue(checksumAlgorithm) + if checksumValue != "" && expectedSum != checksumValue { + return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + + // reject object lock settings the bucket doesn't support before + // the directory object is changed + if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn || po.ObjectLockMode != "" { + err = p.isBucketObjectLockEnabled(*po.Bucket) + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + + vStatus, err := p.getBucketVersioningStatus(ctx, *po.Bucket) + if err != nil { + return s3response.PutObjectOutput{}, err + } + versioned := p.versioningEnabled() && vStatus != "" + + // In a versioned bucket the directory is kept across versions: its + // current version is copied to the versioning directory first. + err = p.snapshotObjVersion(*po.Bucket, *po.Key, vStatus, acct) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + err = p.mkdirAll(name, uid, gid, doChown) + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + return s3response.PutObjectOutput{}, err + } + + // The directory outlives the object it replaces, whether or not + // the bucket is versioned: the new object starts with none of the + // replaced object's attributes. + err = p.clearDirObjectAttrs(*po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + var versionID string + if versioned { + if p.isBucketVersioningSuspended(vStatus) { + err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + } else { + versionID = ulid.Make().String() + } + } + + err = p.storeObjectMetadata(nil, *po.Bucket, *po.Key, po.Metadata) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set object metadata: %w", err) + } + + dirETag := emptyMD5 + if p.dataIntegrityEtag { + dirETag = fmt.Sprintf("\"%s-%s\"", strings.ToUpper(string(checksumAlgorithm)), expectedSum) + } + + // set etag attribute to signify this dir was specifically put + err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, etagkey, + []byte(dirETag)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) + } + + // set "application/x-directory" content-type + err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, contentTypeHdr, + []byte(backend.DirContentType)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err) + } + + if getString(po.WebsiteRedirectLocation) != "" { + err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, websiteRedirectHdr, + []byte(*po.WebsiteRedirectLocation)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set website-redirect-location attr: %w", err) + } + } + + // set empty checksum + checksum := s3response.Checksum{ + Type: types.ChecksumTypeFullObject, + Algorithm: checksumAlgorithm, + } + + checksum.SetSum(checksumAlgorithm, &expectedSum) + + err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) + } + + if versionID != "" { + err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, versionIdKey, []byte(versionID)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) + } + } + + if versioned { + // The directory mtime is the version's last modified time. + // Setting it needs the directory to be owned by the gateway, + // otherwise it's left as is. + now := time.Now() + _ = os.Chtimes(name, now, now) + } + + // Set object tagging once the etag makes the directory an object + if tags != nil { + err := p.PutObjectTagging(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", tags) + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + + err = p.putObjectLockSettings(ctx, po) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + return s3response.PutObjectOutput{ + ETag: dirETag, + VersionID: versionID, + Size: &contentLength, + ChecksumType: checksum.Type, + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumCRC64NVME: checksum.CRC64NVME, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, + ChecksumSHA512: checksum.SHA512, + ChecksumMD5: checksum.MD5, + ChecksumXXHASH64: checksum.XXHASH64, + ChecksumXXHASH3: checksum.XXHASH3, + ChecksumXXHASH128: checksum.XXHASH128, + }, nil + } + + vStatus, err := p.getBucketVersioningStatus(ctx, *po.Bucket) + if err != nil { + return s3response.PutObjectOutput{}, err + } + vEnabled := p.isBucketVersioningEnabled(vStatus) + + // object is file + d, err := os.Stat(name) + if err == nil && d.IsDir() { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } + + // Version snapshots are taken under the object publish lock below. This + // keeps concurrent versioned overwrites ordered with their publications. + conditional := po.IfMatch != nil || po.IfNoneMatch != nil + if isErrNameTooLong(err) { + return s3response.PutObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(*po.Key)), 1024) + } + if isErrNotDir(err) { + parentErr := handleParentDirError(name) + if parentErr != nil { + return s3response.PutObjectOutput{}, parentErr + } + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return s3response.PutObjectOutput{}, fmt.Errorf("stat object: %w", err) + } + + f, err := p.openTmpFile(p.ObjectPath(*po.Bucket, MetaTmpDir), + *po.Bucket, *po.Key, contentLength, acct, doFalloc, p.forceNoTmpFile, odirectAllowed) + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + drainBody(po.Body) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + drainBody(po.Body) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + return s3response.PutObjectOutput{}, fmt.Errorf("open temp file: %w", err) + } + defer f.cleanup() + + objsize := f.size + + // When dataIntegrityEtag is enabled the MD5 is never used, so skip + // both the allocating md5.New() and the per-byte TeeReader overhead. + var md5hash hash.Hash + var rdr io.Reader + if p.dataIntegrityEtag { + rdr = po.Body + } else { + md5hash = md5.New() + rdr = io.TeeReader(po.Body, md5hash) + } + + var hashRdr *utils.HashReader + if !isTrailingChecksum { + hashRdr, err = utils.NewHashReader(rdr, checksumValue, utils.HashType(strings.ToLower(string(checksumAlgorithm)))) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } + + buf := p.getIOBuffer() + defer p.putIOBuffer(buf) + + _, err = f.copyFrom(rdr, buf) + if err != nil { + if errors.Is(err, syscall.EDQUOT) { + drainBody(rdr) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) + } + if errors.Is(err, syscall.ENOSPC) { + drainBody(rdr) + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSpaceLeftOnDevice) + } + // Return the error itself, if it implements the s3err.S3Error interface + if _, ok := err.(s3err.S3Error); ok { + return s3response.PutObjectOutput{}, err + } + return s3response.PutObjectOutput{}, fmt.Errorf("write object data: %w", err) + } + + // If the file was pre-allocated (via fallocate) to a size larger than the + // bytes actually written, truncate it to the real content size. This can + // happen when Content-Length includes epilogue bytes after the multipart + // final boundary (e.g. browser-based POST Object with trailing data). + if f.size > 0 { + objsize -= f.size + if err := f.File().Truncate(objsize); err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("truncate object to actual size: %w", err) + } + } + + dir := filepath.Dir(name) + if dir != "" { + err = p.mkdirAll(dir, uid, gid, doChown) + if backend.IsPermissionErr(err) { + return s3response.PutObjectOutput{}, fmt.Errorf("make parent dir: %w", err) + } + if err != nil { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } + } + + // if the versioning is enabled, generate a new versionID for the object + var versionID string + if p.versioningEnabled() && vEnabled { + versionID = ulid.Make().String() + } + + // Run the backend post-process hook on the staged file before taking + // the publish lock and before any path-based metadata is written, so + // that a failing hook neither holds up other writers nor leaves + // partial metadata behind. + err = postprocess(f.File()) + if err != nil { + return s3response.PutObjectOutput{}, + fmt.Errorf("put object post process failed: %w", err) + } + + // The body is fully staged; serialize the commit phase (condition + // evaluation, metadata stores, and final link) with competing writers + // to the same key. + unlock, err := p.lockObjectPublish(ctx, *po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + defer unlock() + + if conditional { + // authoritative precondition check under the publish lock + err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + } + + // The path may have become a directory while the body was staged: the + // attributes there belong to the directory, not to this object. + d, err = os.Stat(name) + if err == nil && d.IsDir() { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } + + // Snapshot the object that is about to be replaced only after the + // conditional check has succeeded and while holding the publish lock. + verr := p.snapshotObjVersion(*po.Bucket, *po.Key, vStatus, acct) + if verr != nil { + return s3response.PutObjectOutput{}, verr + } + + // The new object starts with none of the replaced object's attributes, + // whether or not they were archived above: the null version and the + // unversioned object are replaced without a snapshot. + keep := []string{etagkey, checksumsKey} + if versionID != "" { + keep = append(keep, versionIdKey) + } + err = p.meta.ReplaceObject(*po.Bucket, *po.Key, keep) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("remove replaced object attributes: %w", err) + } + + // Before finalizing the object creation remove + // null versionId object from versioning directory + // if it exists and the versioning status is Suspended + if p.isBucketVersioningSuspended(vStatus) { + err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key) + if err != nil { + return s3response.PutObjectOutput{}, err + } + versionID = nullVersionId + } + + var sum string + + if isTrailingChecksum { + sum = chRdr.Checksum() + } else { + sum = hashRdr.Sum() + } + + // Generate the ETag for this object. + // When dataIntegrityEtag is enabled, embed the checksum algorithm name and + // value directly in the ETag (e.g. "CRC64NVME-") instead of the + // standard MD5 digest. The checksum is always available here: either + // supplied by the client or computed internally as CRC64NVME by default. + var etag string + if p.dataIntegrityEtag { + etag = fmt.Sprintf("\"%s-%s\"", strings.ToUpper(string(checksumAlgorithm)), sum) + } else { + etag = backend.GenerateEtag(md5hash) + } + + checksum := s3response.Checksum{ + Type: types.ChecksumTypeFullObject, + Algorithm: checksumAlgorithm, + } + + checksum.SetSum(checksumAlgorithm, &sum) + err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) + } + + err = p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, etagkey, []byte(etag)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) + } + + err = p.storeObjectMetaProperties(f.File(), *po.Bucket, *po.Key, + metaProperties{ + ContentType: po.ContentType, + ContentEncoding: po.ContentEncoding, + ContentLanguage: po.ContentLanguage, + ContentDisposition: po.ContentDisposition, + CacheControl: po.CacheControl, + Expires: po.Expires, + WebsiteRedirectLocation: po.WebsiteRedirectLocation, + Metadata: po.Metadata, + }) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + if versionID != "" && versionID != nullVersionId { + err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, versionIdKey, []byte(versionID)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) + } + } + + // "null" identifies the current object internally while versioning is + // suspended, but S3 omits the version ID from the PutObject response. + if versionID == nullVersionId { + versionID = "" + } + + err = f.link() + if errors.Is(err, syscall.EEXIST) { + return s3response.PutObjectOutput{ + ETag: etag, + VersionID: versionID, + }, nil + } + if backend.IsPermissionErr(err) { + return s3response.PutObjectOutput{}, fmt.Errorf("link object in namespace: %w", err) + } + if err != nil { + return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory) + } + + // Set object tagging + if tags != nil { + err := p.PutObjectTagging(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", tags) + if errors.Is(err, fs.ErrNotExist) { + return s3response.PutObjectOutput{ + ETag: etag, + VersionID: versionID, + }, nil + } + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + + err = p.putObjectLockSettings(ctx, po) + if err != nil { + return s3response.PutObjectOutput{}, err + } + + return s3response.PutObjectOutput{ + ETag: etag, + VersionID: versionID, + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, + ChecksumCRC64NVME: checksum.CRC64NVME, + ChecksumSHA512: checksum.SHA512, + ChecksumMD5: checksum.MD5, + ChecksumXXHASH64: checksum.XXHASH64, + ChecksumXXHASH3: checksum.XXHASH3, + ChecksumXXHASH128: checksum.XXHASH128, + Size: &objsize, + ChecksumType: checksum.Type, + }, nil +} + +// putObjectLockSettings sets the legal hold and retention requested with +// the put on the object that was just published +func (p *Posix) putObjectLockSettings(ctx context.Context, po s3response.PutObjectInput) error { + // Set object legal hold + if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { + err := p.PutObjectLegalHold(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", true) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return err + } + } + + // Set object retention + if po.ObjectLockMode != "" { + retention := types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionMode(po.ObjectLockMode), + RetainUntilDate: po.ObjectLockRetainUntilDate, + } + retParsed, err := json.Marshal(retention) + if err != nil { + return fmt.Errorf("parse object lock retention: %w", err) + } + err = p.PutObjectRetention(withCtxNoSlot(ctx), *po.Bucket, *po.Key, "", retParsed) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return err + } + } + + return nil +} + +func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (_ *s3.DeleteObjectOutput, err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + bucket := *input.Bucket + object := *input.Key + isDir := strings.HasSuffix(object, "/") + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + if err := p.validateVersionId(backend.GetStringFromPtr(input.VersionId)); err != nil { + return nil, err + } + + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + objpath := p.ObjectPath(bucket, object) + + vStatus, err := p.getBucketVersioningStatus(ctx, bucket) + if err != nil { + return nil, err + } + + evalPreconditions := func(f os.FileInfo, bucket, object string) error { + var err error + if f == nil { + f, err = os.Stat(p.ObjectPath(bucket, object)) + if err != nil { + return nil + } + } + + b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + etag := string(b) + if err != nil || etag == "" { + etag = p.defaultEtag + } + + // evaluate preconditions + return backend.EvaluateObjectDeletePreconditions(etag, f.ModTime(), f.Size(), + backend.ObjectDeletePreconditions{ + IfMatch: input.IfMatch, + IfMatchLastModTime: input.IfMatchLastModifiedTime, + IfMatchSize: input.IfMatchSize, + }) + } + + if p.versioningEnabled() && vStatus != "" { + if getString(input.VersionId) == "" { + // if the versionId is not specified, make the current version a delete marker + fi, err := os.Stat(objpath) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + } + if err != nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + // the entry at the object path may be the object of the key + // with or without the trailing slash + isObj, err := p.isLiveObject(fi, bucket, object) + if err != nil { + return nil, err + } + if !isObj { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } + + err = evalPreconditions(fi, bucket, object) + if err != nil { + return nil, err + } + + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + // Get object versionId + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + if errors.Is(err, meta.ErrNoSuchKey) { + vId = []byte(nullVersionId) + } + + // Creates a new object version in the versioning directory + if p.isBucketVersioningEnabled(vStatus) || string(vId) != nullVersionId { + _, err = p.createObjVersion(bucket, object, fi.Size(), acct, true) + if err != nil { + return nil, err + } + } + + // Mark the object as a delete marker + err = p.meta.StoreAttribute(nil, bucket, object, deleteMarkerKey, []byte{}) + if err != nil { + return nil, fmt.Errorf("set delete marker: %w", err) + } + + versionId := nullVersionId + if p.isBucketVersioningEnabled(vStatus) { + // Generate & set a unique versionId for the delete marker + versionId = ulid.Make().String() + err = p.meta.StoreAttribute(nil, bucket, object, versionIdKey, []byte(versionId)) + if err != nil { + return nil, fmt.Errorf("set versionId: %w", err) + } + } else { + err = p.meta.DeleteAttribute(bucket, object, versionIdKey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("delete versionId: %w", err) + } + } + + // The delete marker is last modified when it's created, not + // when the version it hides was. Setting the time needs the + // entry to be owned by the gateway, otherwise it's left as is. + now := time.Now() + _ = os.Chtimes(objpath, now, now) + + return &s3.DeleteObjectOutput{ + DeleteMarker: getBoolPtr(true), + VersionId: &versionId, + }, nil + } else { + versionPath := p.genObjVersionPath(bucket, object) + + // the attributes at the object path may belong to the key with + // or without the trailing slash, or to a directory that isn't an + // object + _, err := p.statLiveObject(bucket, object) + if errors.Is(err, fs.ErrNotExist) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil + } + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + } + if err != nil { + return nil, fmt.Errorf("stat object: %w", err) + } + + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{ + VersionId: input.VersionId, + }, nil + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + if errors.Is(err, meta.ErrNoSuchKey) { + vId = []byte(nullVersionId) + } + + if string(vId) == *input.VersionId { + // evaluate preconditions + err := evalPreconditions(nil, bucket, object) + if err != nil { + return nil, err + } + // if the specified VersionId is the same as in the latest version, + // remove the latest version, find the latest version from the versioning + // directory and move to the place of the deleted object, to make it the latest + isDelMarker, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return nil, err + } + if isDir { + err = p.deleteDirObjectLatestVersion(bucket, object) + if err != nil { + return nil, err + } + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + err = os.Remove(objpath) + if err != nil && !errors.Is(err, fs.ErrNotExist) && !isErrNotDir(err) { + return nil, fmt.Errorf("remove obj version: %w", err) + } + + ents, err := os.ReadDir(versionPath) + if errors.Is(err, fs.ErrNotExist) { + if err := p.meta.DeleteAttributes(bucket, object); err != nil && + !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("delete object attributes: %w", err) + } + p.removeParents(bucket, object) + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + if err != nil { + return nil, fmt.Errorf("read version dir: %w", err) + } + + if len(ents) == 0 { + if err := p.meta.DeleteAttributes(bucket, object); err != nil && + !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("delete object attributes: %w", err) + } + p.removeParents(bucket, object) + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + + srcObjVersion, err := p.latestObjVersion(versionPath, ents) + if err != nil { + return nil, fmt.Errorf("get latest obj version: %w", err) + } + srcVersionId := srcObjVersion.Name() + sf, err := os.Open(filepath.Join(versionPath, srcVersionId)) + if err != nil { + return nil, fmt.Errorf("open obj version: %w", err) + } + acct, ok := ctx.Value("account").(auth.Account) + if !ok { + acct = auth.Account{} + } + + f, err := p.openTmpFile(p.ObjectPath(bucket, MetaTmpDir), + bucket, object, srcObjVersion.Size(), acct, doFalloc, + p.forceNoTmpFile, odirectNotAllowed) + if err != nil { + return nil, fmt.Errorf("open tmp file: %w", err) + } + defer f.cleanup() + + // Prioritize copy_file_range for internal file-to-file version restores. + _, err = io.Copy(f.File(), sf) + if err != nil { + _ = sf.Close() + return nil, fmt.Errorf("copy object %w", err) + } + + if err := sf.Close(); err != nil { + return nil, fmt.Errorf("close obj version: %w", err) + } + + // the restored version keeps its last modified time + f.setModTime(srcObjVersion.ModTime()) + + if err := f.link(); err != nil { + return nil, fmt.Errorf("link tmp file: %w", err) + } + + attrs, err := p.meta.ListAttributes(versionPath, srcVersionId) + if err != nil { + return nil, fmt.Errorf("list object attributes: %w", err) + } + + // the restored version has only its own attributes: the + // ones of the deleted version it doesn't have are removed + err = p.meta.ReplaceObject(bucket, object, attrs) + if err != nil { + return nil, fmt.Errorf("remove deleted version attributes: %w", err) + } + + for _, attr := range attrs { + // the place of the null version only applies in + // the versioning directory + if attr == nullVersionPrevKey { + continue + } + data, err := p.meta.RetrieveAttribute(nil, versionPath, srcVersionId, attr) + if err != nil { + return nil, fmt.Errorf("load %v attribute", attr) + } + + err = p.meta.StoreAttribute(nil, bucket, object, attr, data) + if err != nil { + return nil, fmt.Errorf("store %v attribute", attr) + } + } + + err = os.Remove(filepath.Join(versionPath, srcVersionId)) + if err != nil { + return nil, fmt.Errorf("remove obj version %w", err) + } + + _ = p.meta.DeleteAttributes(versionPath, srcVersionId) + p.removeParents(filepath.Join(p.versioningDir, bucket), filepath.Join(genObjVersionKey(object), *input.VersionId)) + + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + + err = evalPreconditions(nil, versionPath, *input.VersionId) + if err != nil { + return nil, err + } + + isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId) + + err = os.Remove(filepath.Join(versionPath, *input.VersionId)) + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + } + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, *input.VersionId) + } + if err != nil { + return nil, fmt.Errorf("delete object: %w", err) + } + + _ = p.meta.DeleteAttributes(versionPath, *input.VersionId) + p.removeParents(filepath.Join(p.versioningDir, bucket), filepath.Join(genObjVersionKey(object), *input.VersionId)) + + return &s3.DeleteObjectOutput{ + DeleteMarker: &isDelMarker, + VersionId: input.VersionId, + }, nil + } + } + + // a bucket that has never been versioned only holds null versions, so + // any other versionId names a version that doesn't exist: AWS returns + // success and deletes nothing + if versionId := getString(input.VersionId); versionId != "" && versionId != nullVersionId { + return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil + } + + fi, err := os.Stat(objpath) + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + } + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } + if err != nil { + return &s3.DeleteObjectOutput{}, fmt.Errorf("stat object: %w", err) + } + if strings.HasSuffix(object, "/") && !fi.IsDir() { + // requested object is expecting a directory with a trailing + // slash, but the object is not a directory. treat this as + // a non-existent object. + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } + if !strings.HasSuffix(object, "/") && fi.IsDir() { + // requested object is expecting a file, but the object is a + // directory. treat this as a non-existent object. + // AWS returns success if the object does not exist + return &s3.DeleteObjectOutput{}, nil + } + + err = evalPreconditions(fi, bucket, object) + if err != nil { + return nil, err + } + + err = os.Remove(objpath) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if isErrDirNotEmpty(err) { + // If the directory object has been uploaded explicitly + // remove the directory object (remove the ETag) + _, err = p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object etag: %w", err) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return nil, s3err.GetAPIError(s3err.ErrDirectoryNotEmpty) + } + + err = p.meta.DeleteAttribute(bucket, object, etagkey) + if err != nil { + return nil, fmt.Errorf("delete object etag: %w", err) + } + + return &s3.DeleteObjectOutput{}, nil + } + if err != nil { + return nil, fmt.Errorf("delete object: %w", err) + } + + err = p.meta.DeleteAttributes(bucket, object) + if err != nil { + return nil, fmt.Errorf("delete object attributes: %w", err) + } + + p.removeParents(bucket, object) + + return &s3.DeleteObjectOutput{}, nil +} + +// newestObjVersionId returns the id of the newest version, other than the +// null version, in the object versioning directory at versionPath, or an +// empty string if it has none. The entries are named after the version id +// and os.ReadDir sorts them by name, which puts the ulid version ids in +// creation order. +func newestObjVersionId(versionPath string) (string, error) { + ents, err := os.ReadDir(versionPath) + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + for _, ent := range slices.Backward(ents) { + if ent.Name() != nullVersionId { + return ent.Name(), nil + } + } + return "", nil +} + +// nullVersionPos is the place of the null version in the version history of +// an object, among the versions ordered by their ulid version ids +type nullVersionPos struct { + // prev is the id of the newest version older than the null version, + // empty if the null version is the oldest one + prev string + // modTime places a null version that was moved to the versioning + // directory before its place was recorded + modTime *time.Time +} + +// isNewerThan reports whether the null version is newer than the version +// fi, an entry of the object versioning directory named after its version id +func (n nullVersionPos) isNewerThan(fi fs.FileInfo) bool { + if n.modTime != nil { + return n.modTime.After(fi.ModTime()) + } + return fi.Name() <= n.prev +} + +// getNullVersionPos returns the place of the null version, nullInfo, in the +// object versioning directory at versionPath +func (p *Posix) getNullVersionPos(versionPath string, nullInfo fs.FileInfo) (nullVersionPos, error) { + prev, err := p.meta.RetrieveAttribute(nil, versionPath, nullVersionId, nullVersionPrevKey) + if errors.Is(err, meta.ErrNoSuchKey) { + modTime := nullInfo.ModTime() + return nullVersionPos{modTime: &modTime}, nil + } + if err != nil { + return nullVersionPos{}, fmt.Errorf("get %v attribute: %w", nullVersionPrevKey, err) + } + return nullVersionPos{prev: string(prev)}, nil +} + +// latestObjVersion returns the version, among the entries of the object +// versioning directory at versionPath, that becomes the latest one when the +// latest version of the object is deleted +func (p *Posix) latestObjVersion(versionPath string, ents []fs.DirEntry) (fs.FileInfo, error) { + var latest, nullEnt fs.DirEntry + for _, ent := range ents { + if ent.Name() == nullVersionId { + nullEnt = ent + continue + } + latest = ent + } + + switch { + case latest == nil && nullEnt == nil: + return nil, fs.ErrNotExist + case latest == nil: + return nullEnt.Info() + case nullEnt == nil: + return latest.Info() + } + + latestInfo, err := latest.Info() + if err != nil { + return nil, err + } + nullInfo, err := nullEnt.Info() + if err != nil { + return nil, err + } + + pos, err := p.getNullVersionPos(versionPath, nullInfo) + if err != nil { + return nil, err + } + if pos.isNewerThan(latestInfo) { + return nullInfo, nil + } + + return latestInfo, nil +} + +// deleteDirObjectLatestVersion removes the latest version of the directory +// object at bucket/key. The newest remaining version is restored onto the +// directory. With no versions left the directory stops being an object and +// is removed, unless other objects are under it. +func (p *Posix) deleteDirObjectLatestVersion(bucket, key string) error { + objpath := p.ObjectPath(bucket, key) + versionPath := p.genObjVersionPath(bucket, key) + + ents, err := os.ReadDir(versionPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("read version dir: %w", err) + } + + if len(ents) == 0 { + err := os.Remove(objpath) + if isErrDirNotEmpty(err) { + // the directory stays as the parent of the objects under it + err = p.clearDirObjectAttrs(bucket, key) + if err != nil { + return err + } + err = p.meta.DeleteAttribute(bucket, key, etagkey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove etag attribute: %w", err) + } + return nil + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove dir object: %w", err) + } + err = p.meta.DeleteAttributes(bucket, key) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("delete object attributes: %w", err) + } + p.removeParents(bucket, key) + return nil + } + + srcInfo, err := p.latestObjVersion(versionPath, ents) + if err != nil { + return fmt.Errorf("get latest obj version: %w", err) + } + srcVersionId := srcInfo.Name() + + // replace the attributes in place, so that the directory keeps its etag + for _, attr := range dirObjectAttrs { + data, err := p.meta.RetrieveAttribute(nil, versionPath, srcVersionId, attr) + if errors.Is(err, meta.ErrNoSuchKey) { + err = p.meta.DeleteAttribute(bucket, key, attr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove %v attribute: %w", attr, err) + } + continue + } + if err != nil { + return fmt.Errorf("load %v attribute: %w", attr, err) + } + + err = p.meta.StoreAttribute(nil, bucket, key, attr, data) + if err != nil { + return fmt.Errorf("store %v attribute: %w", attr, err) + } + } + + err = os.Remove(filepath.Join(versionPath, srcVersionId)) + if err != nil { + return fmt.Errorf("remove obj version: %w", err) + } + + _ = p.meta.DeleteAttributes(versionPath, srcVersionId) + p.removeParents(filepath.Join(p.versioningDir, bucket), filepath.Join(genObjVersionKey(key), srcVersionId)) + + // The restored version keeps its last modified time. Setting it needs + // the directory to be owned by the gateway, otherwise it's left as is. + _ = os.Chtimes(objpath, time.Now(), srcInfo.ModTime()) + + return nil +} + +func (p *Posix) removeParents(bucket, object string) { + // this will remove all parent directories that were not + // specifically uploaded with a put object. we detect + // this with a special attribute to indicate these. stop + // at either the bucket or the first parent we encounter + // with the attribute, whichever comes first. + + // Remove the last path separator for the directory objects + // to correctly detect the parent in the loop + objPath := strings.TrimSuffix(object, "/") + for { + parent := filepath.Dir(objPath) + if parent == string(filepath.Separator) || parent == "." { + // stop removing parents if we hit the bucket directory. + break + } + + _, err := p.meta.RetrieveAttribute(nil, bucket, parent, etagkey) + if err == nil { + // a directory with a valid etag means this was specifically + // uploaded with a put object, so stop here and leave this + // directory in place. + break + } + + err = os.Remove(p.ObjectPath(bucket, parent)) + if err != nil { + break + } + + objPath = parent + } +} + +func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.DeleteResult{}, err + } + defer release() + + bucket := *input.Bucket + if !p.isBucketValid(bucket) { + return s3response.DeleteResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + // a missing bucket fails the whole request rather than each key + err = p.doesBucketExist(bucket) + if err != nil { + return s3response.DeleteResult{}, err + } + + delResult, errs := []types.DeletedObject{}, []types.Error{} + for _, obj := range input.Delete.Objects { + //TODO: Make the delete operation concurrent + // once concurrency is implemented, the posix rate limiter should + // be taken into account. + res, err := p.DeleteObject(withCtxNoSlot(ctx), &s3.DeleteObjectInput{ + Bucket: input.Bucket, + Key: obj.Key, + VersionId: obj.VersionId, + }) + if err == nil { + delEntity := types.DeletedObject{ + Key: obj.Key, + DeleteMarker: res.DeleteMarker, + VersionId: obj.VersionId, + } + if delEntity.DeleteMarker != nil && *delEntity.DeleteMarker { + delEntity.DeleteMarkerVersionId = res.VersionId + } + + delResult = append(delResult, delEntity) + } else { + errs = append(errs, s3err.ObjectDeleteError(obj.Key, obj.VersionId, err)) + } + } + + return s3response.DeleteResult{ + Deleted: delResult, + Error: errs, + }, nil +} + +func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + var versionId string + if input.VersionId != nil { + versionId = *input.VersionId + } + + if err := p.validateVersionId(versionId); err != nil { + return nil, err + } + + if !p.versioningEnabled() && versionId != "" { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + + bucket := *input.Bucket + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + object := *input.Key + if versionId != "" { + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + if errors.Is(err, meta.ErrNoSuchKey) { + vId = []byte(nullVersionId) + } + + if string(vId) != versionId { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), versionId) + } + } + + objPath := p.ObjectPath(bucket, object) + + fid, err := os.Stat(objPath) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId) + } + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) + } + if err != nil { + return nil, fmt.Errorf("stat object: %w", err) + } + + if strings.HasSuffix(object, "/") && !fid.IsDir() { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if !strings.HasSuffix(object, "/") && fid.IsDir() { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if fid.IsDir() { + // Only directories explicitly created via S3 (put-object with key ending + // in '/') have an etag attribute. Directories created incidentally on the + // filesystem or as parent directories during object upload should not be + // accessible via get-object. + isObj, err := p.isDirObject(bucket, object) + if err != nil { + return nil, err + } + if !isObj { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + } + + if p.versioningEnabled() { + isDelMarker, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return nil, err + } + + // if the specified object version is a delete marker, return MethodNotAllowed + if isDelMarker { + if versionId != "" { + err = s3err.GetAPIError(s3err.ErrMethodNotAllowed) + } else { + err = s3err.GetAPIError(s3err.ErrNoSuchKey) + } + return &s3.GetObjectOutput{ + DeleteMarker: getBoolPtr(true), + LastModified: backend.GetTimePtr(fid.ModTime()), + }, err + } + } + + b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + etag := string(b) + if err != nil || etag == "" { + etag = p.defaultEtag + } + + // evaluate preconditions + err = backend.EvaluatePreconditions(etag, fid.ModTime(), backend.PreConditions{ + IfMatch: input.IfMatch, + IfNoneMatch: input.IfNoneMatch, + IfModSince: input.IfModifiedSince, + IfUnmodeSince: input.IfUnmodifiedSince, + }) + if err != nil { + return nil, err + } + + // If versioning is configured get the object versionId + if p.versioningEnabled() && versionId == "" { + versionId, err = p.liveObjVersionId(ctx, bucket, object) + if err != nil { + return nil, err + } + } + + if fid.IsDir() { + _, _, _, err := backend.ParseObjectRange(0, *input.Range) + if err != nil { + return nil, err + } + + objMeta := p.loadObjectMetaProperties(nil, bucket, object, &fid) + + var tagCount *int32 + tags, err := p.getAttrTags(bucket, object, versionId) + if err != nil { + return nil, err + } + if len(tags) != 0 { + tgCount := int32(len(tags)) + tagCount = &tgCount + } + + var checksums s3response.Checksum + if input.ChecksumMode == types.ChecksumModeEnabled { + checksums, err = p.retrieveChecksums(nil, bucket, object) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object checksums: %w", err) + } + } + + var length int64 = 0 + return &s3.GetObjectOutput{ + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, + ChecksumType: checksums.Type, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + ContentEncoding: objMeta.ContentEncoding, + ContentType: objMeta.ContentType, + ContentLanguage: objMeta.ContentLanguage, + ContentDisposition: objMeta.ContentDisposition, + CacheControl: objMeta.CacheControl, + ExpiresString: objMeta.Expires, + WebsiteRedirectLocation: objMeta.WebsiteRedirectLocation, + ETag: &etag, + LastModified: backend.GetTimePtr(fid.ModTime()), + Metadata: objMeta.Metadata, + TagCount: tagCount, + ContentRange: nil, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, + }, nil + } + + // openForRead opens with FILE_SHARE_DELETE on Windows so that a concurrent + // DeleteObject can call os.Remove on this file while the GET response body + // is still being streamed. On POSIX, os.Open is sufficient. + f, err := openForRead(objPath, p.enableODirect) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return nil, fmt.Errorf("open object: %w", err) + } + + fi, err := f.Stat() + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) + } + if err != nil { + return nil, fmt.Errorf("stat object: %w", err) + } + + objSize := fi.Size() + if fi.IsDir() { + objSize = 0 + } + + var contentRange *string + var startOffset, length int64 + var partsCount *int32 + + // If partNumber is requested and mp-metadata exists, serve that specific part. + // For non-multipart objects (no mp-metadata), partNumber=1 returns the full + // object with no Content-Range; any other partNumber is out of range. + // Both range read and partNumber can't be used together. + if input.PartNumber != nil { + mpMetaBytes, metaErr := p.meta.RetrieveAttribute(nil, bucket, object, mpMetaKey) + if metaErr == nil { + mpMeta, err := backend.UnmarshalMpUploadMetadata(mpMetaBytes, false) + if err != nil { + return nil, fmt.Errorf("parse object multipart metadata: %w", err) + } + + partNum := *input.PartNumber + totalParts := int32(len(mpMeta.Parts)) + partsCount = &totalParts + if partNum > totalParts { + return nil, s3err.GetInvalidPartNumberRangeErr(totalParts, partNum) + } + + // Parts holds cumulative sizes: Parts[i] = sum of sizes 1..i+1 + if partNum > 1 { + startOffset = mpMeta.Parts[partNum-2] + } + length = mpMeta.Parts[partNum-1] - startOffset + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %d-%d/%d", startOffset, startOffset+length-1, objSize)) + } else if errors.Is(metaErr, meta.ErrNoSuchKey) { + // Non-multipart object: partNumber=1 means the whole object; anything + // higher is out of range + if *input.PartNumber > 1 { + return nil, s3err.GetInvalidPartNumberRangeErr(1, *input.PartNumber) + } + length = objSize + if objSize != 0 { + // if object size is 0, the whole object is served, no content range should be set + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes 0-%d/%d", objSize-1, objSize)) + } + } else { + return nil, fmt.Errorf("retrieve mp metadata: %w", metaErr) + } + } else { + start, lgth, isValid, err := backend.ParseObjectRange(objSize, getString(input.Range)) + if err != nil { + return nil, err + } + startOffset, length = start, lgth + + if isValid { + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %v-%v/%v", start, start+lgth-1, objSize)) + } + } + + objMeta := p.loadObjectMetaProperties(f, bucket, object, &fi) + + var tagCount *int32 + tags, err := p.getAttrTags(bucket, object, versionId) + if err != nil { + return nil, err + } + if len(tags) != 0 { + tgCount := int32(len(tags)) + tagCount = &tgCount + } + + var checksums s3response.Checksum + // Return checksums only when the full object is requested + if input.ChecksumMode == types.ChecksumModeEnabled && length-startOffset == objSize { + checksums, err = p.retrieveChecksums(f, bucket, object) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object checksums: %w", err) + } + } + + // Buffered full-object responses keep the underlying *os.File for sendfile. + body, err := buildGetObjectBody(f, objPath, startOffset, length, objSize, p.enableODirect, p.ioBufferSize) + if err != nil { + return nil, fmt.Errorf("build get object body: %w", err) + } + + return &s3.GetObjectOutput{ + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + ContentEncoding: objMeta.ContentEncoding, + ContentType: objMeta.ContentType, + ContentDisposition: objMeta.ContentDisposition, + ContentLanguage: objMeta.ContentLanguage, + CacheControl: objMeta.CacheControl, + ExpiresString: objMeta.Expires, + WebsiteRedirectLocation: objMeta.WebsiteRedirectLocation, + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + Metadata: objMeta.Metadata, + TagCount: tagCount, + ContentRange: contentRange, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, + Body: body, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, + ChecksumType: checksums.Type, + PartsCount: partsCount, + }, nil +} + +func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + versionId := backend.GetStringFromPtr(input.VersionId) + + if err := p.validateVersionId(versionId); err != nil { + return nil, err + } + + if !p.versioningEnabled() && versionId != "" { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + + bucket := *input.Bucket + object := *input.Key + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + if versionId != "" { + vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get obj versionId: %w", err) + } + if errors.Is(err, meta.ErrNoSuchKey) { + // an object without a versionId attribute is the null version + vId = []byte(nullVersionId) + } + + if string(vId) != versionId { + bucket = filepath.Join(p.versioningDir, bucket) + object = filepath.Join(genObjVersionKey(object), versionId) + } + } + + objPath := p.ObjectPath(bucket, object) + + fi, err := os.Stat(objPath) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId) + } + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if isErrNameTooLong(err) { + return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) + } + if err != nil { + return nil, fmt.Errorf("stat object: %w", err) + } + if strings.HasSuffix(object, "/") && !fi.IsDir() { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if !strings.HasSuffix(object, "/") && fi.IsDir() { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if fi.IsDir() { + // Only directories explicitly created via S3 (put-object with key ending + // in '/') have an etag attribute. Directories created incidentally on the + // filesystem or as parent directories during object upload should not be + // accessible via head-object. + isObj, err := p.isDirObject(bucket, object) + if err != nil { + return nil, err + } + if !isObj { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + } + + if p.versioningEnabled() { + isDelMarker, err := p.isObjDeleteMarker(bucket, object) + if err != nil { + return nil, err + } + + // if the specified object version is a delete marker, return MethodNotAllowed + if isDelMarker { + if versionId != "" { + return &s3.HeadObjectOutput{ + DeleteMarker: getBoolPtr(true), + LastModified: backend.GetTimePtr(fi.ModTime()), + }, s3err.GetAPIError(s3err.ErrMethodNotAllowed) + } else { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + } + } + + if p.versioningEnabled() && versionId == "" { + versionId, err = p.liveObjVersionId(ctx, bucket, object) + if err != nil { + return nil, err + } + } + + objMeta := p.loadObjectMetaProperties(nil, bucket, object, &fi) + + b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + etag := string(b) + if err != nil || etag == "" { + etag = p.defaultEtag + } + + // evaluate preconditions + err = backend.EvaluatePreconditions(etag, fi.ModTime(), backend.PreConditions{ + IfMatch: input.IfMatch, + IfNoneMatch: input.IfNoneMatch, + IfModSince: input.IfModifiedSince, + IfUnmodeSince: input.IfUnmodifiedSince, + }) + if err != nil { + return nil, err + } + + size := fi.Size() + if fi.IsDir() { + size = 0 + } + + var contentRange *string + var startOffset, length int64 + var partsCount *int32 + + // If partNumber is requested and mp-metadata exists, serve that specific part. + // For non-multipart objects (no mp-metadata), partNumber=1 returns the full + // object with no Content-Range; any other partNumber is out of range. + // Both range read and partNumber can't be used together. + if input.PartNumber != nil { + mpMetaBytes, metaErr := p.meta.RetrieveAttribute(nil, bucket, object, mpMetaKey) + if metaErr == nil { + mpMeta, err := backend.UnmarshalMpUploadMetadata(mpMetaBytes, false) + if err != nil { + return nil, fmt.Errorf("parse object multipart metadata: %w", err) + } + + partNum := *input.PartNumber + totalParts := int32(len(mpMeta.Parts)) + partsCount = &totalParts + if partNum > totalParts { + return nil, s3err.GetInvalidPartNumberRangeErr(totalParts, partNum) + } + + // Parts holds cumulative sizes: Parts[i] = sum of sizes 1..i+1 + if partNum > 1 { + startOffset = mpMeta.Parts[partNum-2] + } + length = mpMeta.Parts[partNum-1] - startOffset + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %d-%d/%d", startOffset, startOffset+length-1, size)) + } else if errors.Is(metaErr, meta.ErrNoSuchKey) { + // Non-multipart object: partNumber=1 means the whole object; anything + // higher is out of range + if *input.PartNumber > 1 { + return nil, s3err.GetInvalidPartNumberRangeErr(1, *input.PartNumber) + } + length = size + if length != 0 { + // if object size is 0, the whole object is served, no content range should be set + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes 0-%d/%d", length-1, length)) + } + } else { + return nil, fmt.Errorf("retrieve mp metadata: %w", metaErr) + } + } else { + start, lgth, isValid, err := backend.ParseObjectRange(size, getString(input.Range)) + if err != nil { + return nil, err + } + startOffset, length = start, lgth + if isValid { + contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %v-%v/%v", start, start+lgth-1, size)) + } + } + + var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus + status, err := p.GetObjectLegalHold(withCtxNoSlot(ctx), bucket, object, versionId) + if err == nil { + if *status { + objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn + } else { + objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff + } + } + + var objectLockMode types.ObjectLockMode + var objectLockRetainUntilDate *time.Time + retention, err := p.GetObjectRetention(withCtxNoSlot(ctx), bucket, object, versionId) + if err == nil { + var config types.ObjectLockRetention + if err := json.Unmarshal(retention, &config); err == nil { + objectLockMode = types.ObjectLockMode(config.Mode) + objectLockRetainUntilDate = config.RetainUntilDate + } + } + + var checksums s3response.Checksum + // Return checksums only when the full object is requested + if input.ChecksumMode == types.ChecksumModeEnabled && length-startOffset == size { + checksums, err = p.retrieveChecksums(nil, bucket, object) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object checksums: %w", err) + } + } + + var tagCount *int32 + tags, err := p.getAttrTags(bucket, object, versionId) + if err != nil { + return nil, err + } + if len(tags) != 0 { + tc := int32(len(tags)) + tagCount = &tc + } + + return &s3.HeadObjectOutput{ + ContentLength: &length, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentRange: contentRange, + ContentType: objMeta.ContentType, + ContentEncoding: objMeta.ContentEncoding, + ContentDisposition: objMeta.ContentDisposition, + ContentLanguage: objMeta.ContentLanguage, + CacheControl: objMeta.CacheControl, + ExpiresString: objMeta.Expires, + WebsiteRedirectLocation: objMeta.WebsiteRedirectLocation, + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + Metadata: objMeta.Metadata, + ObjectLockLegalHoldStatus: objectLockLegalHoldStatus, + ObjectLockMode: objectLockMode, + ObjectLockRetainUntilDate: objectLockRetainUntilDate, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, + ChecksumType: checksums.Type, + TagCount: tagCount, + PartsCount: partsCount, + }, nil +} + +func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.GetObjectAttributesResponse{}, err + } + defer release() + + data, err := p.HeadObject(withCtxNoSlot(ctx), &s3.HeadObjectInput{ + Bucket: input.Bucket, + Key: input.Key, + VersionId: input.VersionId, + ChecksumMode: types.ChecksumModeEnabled, + }) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMethodNotAllowed)) && data != nil { + return s3response.GetObjectAttributesResponse{ + DeleteMarker: data.DeleteMarker, + VersionId: data.VersionId, + }, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + + return s3response.GetObjectAttributesResponse{}, err + } + + return s3response.GetObjectAttributesResponse{ + ETag: backend.TrimEtag(data.ETag), + ObjectSize: data.ContentLength, + StorageClass: data.StorageClass, + LastModified: data.LastModified, + VersionId: data.VersionId, + DeleteMarker: data.DeleteMarker, + Checksum: &types.Checksum{ + ChecksumCRC32: data.ChecksumCRC32, + ChecksumCRC32C: data.ChecksumCRC32C, + ChecksumSHA1: data.ChecksumSHA1, + ChecksumSHA256: data.ChecksumSHA256, + ChecksumCRC64NVME: data.ChecksumCRC64NVME, + ChecksumSHA512: data.ChecksumSHA512, + ChecksumMD5: data.ChecksumMD5, + ChecksumXXHASH64: data.ChecksumXXHASH64, + ChecksumXXHASH3: data.ChecksumXXHASH3, + ChecksumXXHASH128: data.ChecksumXXHASH128, + ChecksumType: data.ChecksumType, + }, + }, nil +} + +func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (_ s3response.CopyObjectOutput, err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + defer release() + + if input.Key == nil { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) + } + if input.ExpectedBucketOwner == nil { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + srcBucket, srcObject, srcVersionId, err := backend.ParseCopySource(*input.CopySource) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + if err := p.validateVersionId(srcVersionId); err != nil { + return s3response.CopyObjectOutput{}, err + } + if !p.isBucketValid(srcBucket) { + return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, srcBucket) + } + dstBucket := *input.Bucket + dstObject := *input.Key + + if !p.isBucketValid(dstBucket) { + return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, dstBucket) + } + + err = p.doesBucketExist(srcBucket) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + if input.ExpectedSourceBucketOwner != nil && *input.ExpectedSourceBucketOwner != "" { + aclData, err := p.meta.RetrieveAttribute(nil, srcBucket, "", aclkey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectOutput{}, fmt.Errorf("get src bucket acl: %w", err) + } + srcAcl, err := auth.ParseACL(aclData) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + if srcAcl.Owner != *input.ExpectedSourceBucketOwner { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrAccessDenied) + } + } + + vStatus, err := p.getBucketVersioningStatus(ctx, srcBucket) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + vEnabled := p.isBucketVersioningEnabled(vStatus) + + origSrcObject := srcObject + if srcVersionId != "" { + if !p.versioningEnabled() || !vEnabled { + return s3response.CopyObjectOutput{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, srcVersionId) + } + vId, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, versionIdKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectOutput{}, fmt.Errorf("get src object version id: %w", err) + } + + if string(vId) != srcVersionId { + srcBucket = joinPathWithTrailer(p.versioningDir, srcBucket) + srcObject = joinPathWithTrailer(genObjVersionKey(srcObject), srcVersionId) + } + } + + err = p.doesBucketExist(dstBucket) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + objPath := joinPathWithTrailer(p.BucketPath(srcBucket), srcObject) + f, err := os.Open(objPath) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if p.versioningEnabled() && vEnabled { + return s3response.CopyObjectOutput{}, s3err.GetNoSuchVersionErr(origSrcObject, srcVersionId) + } + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if isErrNameTooLong(err) { + return s3response.CopyObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(origSrcObject)), 1024) + } + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("open object: %w", err) + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("stat object: %w", err) + } + if strings.HasSuffix(srcObject, "/") && !fi.IsDir() { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if !strings.HasSuffix(srcObject, "/") && fi.IsDir() { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err := p.checkCopySourceDeleteMarker(srcBucket, srcObject, srcVersionId); err != nil { + return s3response.CopyObjectOutput{}, err + } + // a directory object holds no data + srcSize := fi.Size() + var srcBody io.Reader = f + if fi.IsDir() { + isObj, err := p.isDirObject(srcBucket, srcObject) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + if !isObj { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + srcSize = 0 + srcBody = strings.NewReader("") + } + if srcSize > p.copyObjectThreshold { + return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(p.copyObjectThreshold) + } + + b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey) + srcEtag := string(b) + if err != nil || srcEtag == "" { + srcEtag = p.defaultEtag + } + + err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{ + IfMatch: input.CopySourceIfMatch, + IfNoneMatch: input.CopySourceIfNoneMatch, + IfModSince: input.CopySourceIfModifiedSince, + IfUnmodeSince: input.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + var etag string + var version *string + var crc32 *string + var crc32c *string + var sha1 *string + var sha256 *string + var crc64nvme *string + var sha512 *string + var md5sum *string + var xxhash64 *string + var xxhash3 *string + var xxhash128 *string + var chType types.ChecksumType + + dstObjdPath := joinPathWithTrailer(p.BucketPath(dstBucket), dstObject) + // A copy of an object onto itself is rejected unless it replaces the + // object metadata. Naming a source version makes it a regular copy. + selfCopy := dstObjdPath == objPath + if selfCopy && srcVersionId == "" && + input.MetadataDirective == types.MetadataDirectiveCopy { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) + } + + // In a versioned bucket a self copy creates a new version like any other + // write, so only unversioned buckets are rewritten in place. + versioned := p.versioningEnabled() && vStatus != "" + if selfCopy && !versioned { + // Delete the object metadata + err = p.meta.DeleteAttribute(dstBucket, dstObject, metadataHdr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectOutput{}, fmt.Errorf("delete object metadata: %w", err) + } + + checksums, err := p.retrieveChecksums(nil, dstBucket, dstObject) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectOutput{}, fmt.Errorf("get obj checksums: %w", err) + } + + chType = checksums.Type + + if input.ChecksumAlgorithm != "" { + // If a different checksum algorith is specified + // first caclculate and store the checksum + if checksums.Algorithm != input.ChecksumAlgorithm { + f, err := os.Open(dstObjdPath) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("open obj file: %w", err) + } + defer f.Close() + + hashReader, err := utils.NewHashReader(f, "", utils.HashType(strings.ToLower(string(input.ChecksumAlgorithm)))) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + _, err = hashReader.Read(nil) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("read err: %w", err) + } + + checksums = s3response.Checksum{} + + sum := hashReader.Sum() + switch hashReader.Type() { + case utils.HashTypeCRC32: + checksums.CRC32 = &sum + crc32 = &sum + case utils.HashTypeCRC32C: + checksums.CRC32C = &sum + crc32c = &sum + case utils.HashTypeSha1: + checksums.SHA1 = &sum + sha1 = &sum + case utils.HashTypeSha256: + checksums.SHA256 = &sum + sha256 = &sum + case utils.HashTypeCRC64NVME: + checksums.CRC64NVME = &sum + crc64nvme = &sum + case utils.HashTypeSha512: + checksums.SHA512 = &sum + sha512 = &sum + case utils.HashTypeMd5: + checksums.MD5 = &sum + md5sum = &sum + case utils.HashTypeXXHASH64: + checksums.XXHASH64 = &sum + xxhash64 = &sum + case utils.HashTypeXXHASH3: + checksums.XXHASH3 = &sum + xxhash3 = &sum + case utils.HashTypeXXHASH128: + checksums.XXHASH128 = &sum + xxhash128 = &sum + } + + // If a new checksum is calculated, the checksum type + // should be FULL_OBJECT + chType = types.ChecksumTypeFullObject + + err = p.storeChecksums(f, dstBucket, dstObject, checksums) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("store checksum: %w", err) + } + } + } + + b, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, etagkey) + etag = string(b) + vId, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, versionIdKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + version = backend.GetPtrFromString(string(vId)) + + // Store the provided object meta properties + err = p.storeObjectMetaProperties(nil, dstBucket, dstObject, + metaProperties{ + ContentType: input.ContentType, + ContentEncoding: input.ContentEncoding, + ContentLanguage: input.ContentLanguage, + ContentDisposition: input.ContentDisposition, + CacheControl: input.CacheControl, + Expires: input.Expires, + WebsiteRedirectLocation: input.WebsiteRedirectLocation, + Metadata: input.Metadata, + }) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + // explicitly delete the website redirect location, as if it's not + // provided as CopyObject input, it should not be copied + if getString(input.WebsiteRedirectLocation) == "" { + err := p.meta.DeleteAttribute(dstBucket, dstObject, websiteRedirectHdr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectOutput{}, fmt.Errorf("delete website-redirect-location: %w", err) + } + } + + if input.TaggingDirective == types.TaggingDirectiveReplace { + tags, err := backend.ParseObjectTags(getString(input.Tagging)) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + err = p.PutObjectTagging(withCtxNoSlot(ctx), dstBucket, dstObject, "", tags) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + } + } else { + contentLength := srcSize + + checksums, err := p.retrieveChecksums(f, srcBucket, srcObject) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectOutput{}, fmt.Errorf("get obj checksum: %w", err) + } + + // If any checksum algorithm is provided, replace, otherwise + // use the existing one + if input.ChecksumAlgorithm != "" { + checksums.Algorithm = input.ChecksumAlgorithm + } + + // A self copy publishes the new version over the source path, which + // on Windows can't be renamed over while the source is still open. + // PutObject reads the body before publishing, so the handle is + // released as soon as the data has been staged. + if selfCopy { + srcBody = &closeOnEOFReader{r: srcBody, c: f} + } + + putObjectInput := s3response.PutObjectInput{ + Bucket: &dstBucket, + Key: &dstObject, + Body: srcBody, + ContentLength: &contentLength, + ChecksumAlgorithm: checksums.Algorithm, + ContentType: input.ContentType, + ContentEncoding: input.ContentEncoding, + ContentDisposition: input.ContentDisposition, + ContentLanguage: input.ContentLanguage, + CacheControl: input.CacheControl, + Expires: input.Expires, + WebsiteRedirectLocation: input.WebsiteRedirectLocation, + Metadata: input.Metadata, + ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate, + ObjectLockMode: input.ObjectLockMode, + ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus, + } + + // load and pass the source object meta properties, if metadata directive is "COPY" + if input.MetadataDirective != types.MetadataDirectiveReplace { + metaProps := p.loadObjectMetaProperties(nil, srcBucket, srcObject, &fi) + putObjectInput.ContentEncoding = metaProps.ContentEncoding + putObjectInput.ContentDisposition = metaProps.ContentDisposition + putObjectInput.ContentLanguage = metaProps.ContentLanguage + putObjectInput.ContentType = metaProps.ContentType + putObjectInput.CacheControl = metaProps.CacheControl + putObjectInput.Expires = metaProps.Expires + putObjectInput.Metadata = metaProps.Metadata + } + + // pass the input tagging to PutObject, if tagging directive is "REPLACE" + if input.TaggingDirective == types.TaggingDirectiveReplace { + putObjectInput.Tagging = input.Tagging + } + + // read the source tagging before the destination is written, as a + // self copy replaces the source object's metadata + var srcTagging []byte + var hasSrcTagging bool + if input.TaggingDirective == types.TaggingDirectiveCopy { + tagging, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, tagHdr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.CopyObjectOutput{}, fmt.Errorf("get source object tagging: %w", err) + } + srcTagging, hasSrcTagging = tagging, err == nil + } + + res, err := p.PutObject(withCtxNoSlot(ctx), putObjectInput) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + // the source tagging is stored after the destination object creation, + // if tagging directive is "COPY" + if hasSrcTagging { + err := p.meta.StoreAttribute(nil, dstBucket, dstObject, tagHdr, srcTagging) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("set destination object tagging: %w", err) + } + } + + etag = res.ETag + version = &res.VersionID + crc32 = res.ChecksumCRC32 + crc32c = res.ChecksumCRC32C + sha1 = res.ChecksumSHA1 + sha256 = res.ChecksumSHA256 + crc64nvme = res.ChecksumCRC64NVME + sha512 = res.ChecksumSHA512 + md5sum = res.ChecksumMD5 + xxhash64 = res.ChecksumXXHASH64 + xxhash3 = res.ChecksumXXHASH3 + xxhash128 = res.ChecksumXXHASH128 + chType = res.ChecksumType + } + + fi, err = os.Stat(dstObjdPath) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("stat dst object: %w", err) + } + + return s3response.CopyObjectOutput{ + CopyObjectResult: &s3response.CopyObjectResult{ + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + ChecksumCRC32: crc32, + ChecksumCRC32C: crc32c, + ChecksumSHA1: sha1, + ChecksumSHA256: sha256, + ChecksumCRC64NVME: crc64nvme, + ChecksumSHA512: sha512, + ChecksumMD5: md5sum, + ChecksumXXHASH64: xxhash64, + ChecksumXXHASH3: xxhash3, + ChecksumXXHASH128: xxhash128, + ChecksumType: chType, + }, + VersionId: version, + CopySourceVersionId: &srcVersionId, + }, nil +} + +func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.ListObjectsResult{}, err + } + defer release() + + return p.ListObjectsParametrized(ctx, input, p.FileToObj) +} + +func (p *Posix) ListObjectsParametrized(ctx context.Context, input *s3.ListObjectsInput, customFileToObj func(string, bool) backend.GetObjFunc) (s3response.ListObjectsResult, error) { + bucket := *input.Bucket + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.Marker != nil { + marker = *input.Marker + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } + + if !p.isBucketValid(bucket) { + return s3response.ListObjectsResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + err := p.doesBucketExist(bucket) + if err != nil { + return s3response.ListObjectsResult{}, err + } + + fileSystem := os.DirFS(p.BucketPath(bucket)) + results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, + customFileToObj(bucket, true), []string{MetaTmpDir}) + if err != nil { + return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) + } + + return s3response.ListObjectsResult{ + CommonPrefixes: results.CommonPrefixes, + Contents: results.Objects, + Delimiter: backend.GetPtrFromString(delim), + Marker: backend.GetPtrFromString(marker), + NextMarker: backend.GetPtrFromString(results.NextMarker), + Prefix: backend.GetPtrFromString(prefix), + IsTruncated: &results.Truncated, + MaxKeys: &maxkeys, + Name: &bucket, + }, nil +} + +func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { + if !p.isBucketValid(bucket) { + return func(string, fs.DirEntry) (s3response.Object, error) { + return s3response.Object{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + } + + return func(path string, d fs.DirEntry) (s3response.Object, error) { + var owner *types.Owner + // Retrieve the object owner data from bucket ACL, if fetchOwner is true + // All the objects in the bucket are owned by the bucket owner + if fetchOwner { + aclJSON, err := p.meta.RetrieveAttribute(nil, bucket, "", aclkey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.Object{}, fmt.Errorf("get bucket acl: %w", err) + } + + acl, err := auth.ParseACL(aclJSON) + if err != nil { + return s3response.Object{}, err + } + + owner = &types.Owner{ + ID: &acl.Owner, + } + } + if d.IsDir() { + // a directory is listed only if it's a directory object + // check to see if this is a directory object by checking etag + etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) + if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { + return s3response.Object{}, backend.ErrSkipObj + } + if err != nil { + return s3response.Object{}, fmt.Errorf("get etag: %w", err) + } + etag := string(etagBytes) + + // If the directory object is a delete marker, skip + isDel, _ := p.isObjDeleteMarker(bucket, path) + if isDel { + return s3response.Object{}, backend.ErrSkipObj + } + + fi, err := d.Info() + if errors.Is(err, fs.ErrNotExist) { + return s3response.Object{}, backend.ErrSkipObj + } + if err != nil { + return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) + } + + size := int64(0) + mtime := fi.ModTime() + + return s3response.Object{ + ETag: &etag, + Key: &path, + LastModified: &mtime, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + Owner: owner, + }, nil + } + + // If the object is a delete marker, skip + isDel, _ := p.isObjDeleteMarker(bucket, path) + if isDel { + return s3response.Object{}, backend.ErrSkipObj + } + + // Retrieve the object checksum algorithm + checksums, err := p.retrieveChecksums(nil, bucket, path) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.Object{}, backend.ErrSkipObj + } + + // file object, get object info and fill out object data + etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) + if errors.Is(err, fs.ErrNotExist) { + return s3response.Object{}, backend.ErrSkipObj + } + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return s3response.Object{}, fmt.Errorf("get etag: %w", err) + } + // note: meta.ErrNoSuchKey will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + + etag := string(etagBytes) + if etag == "" { + etag = p.defaultEtag + } + + fi, err := d.Info() + if errors.Is(err, fs.ErrNotExist) { + return s3response.Object{}, backend.ErrSkipObj + } + if err != nil { + return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) + } + + size := fi.Size() + mtime := fi.ModTime() + + return s3response.Object{ + ETag: &etag, + Key: &path, + LastModified: &mtime, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{checksums.Algorithm}, + ChecksumType: checksums.Type, + Owner: owner, + }, nil + } +} + +func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return s3response.ListObjectsV2Result{}, err + } + defer release() + + return p.ListObjectsV2Parametrized(ctx, input, p.FileToObj) +} + +func (p *Posix) ListObjectsV2Parametrized(ctx context.Context, input *s3.ListObjectsV2Input, customFileToObj func(string, bool) backend.GetObjFunc) (s3response.ListObjectsV2Result, error) { + bucket := *input.Bucket + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.ContinuationToken != nil { + if input.StartAfter != nil { + marker = max(*input.StartAfter, *input.ContinuationToken) + } else { + marker = *input.ContinuationToken + } + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } + var fetchOwner bool + if input.FetchOwner != nil { + fetchOwner = *input.FetchOwner + } + + if !p.isBucketValid(bucket) { + return s3response.ListObjectsV2Result{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + + err := p.doesBucketExist(bucket) + if err != nil { + return s3response.ListObjectsV2Result{}, err + } + + fileSystem := os.DirFS(p.BucketPath(bucket)) + results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, + customFileToObj(bucket, fetchOwner), []string{MetaTmpDir}) + if err != nil { + return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) + } + + count := int32(len(results.Objects) + len(results.CommonPrefixes)) + + return s3response.ListObjectsV2Result{ + CommonPrefixes: results.CommonPrefixes, + Contents: results.Objects, + IsTruncated: &results.Truncated, + MaxKeys: &maxkeys, + Name: &bucket, + KeyCount: &count, + Delimiter: backend.GetPtrFromString(delim), + ContinuationToken: backend.GetPtrFromString(marker), + NextContinuationToken: backend.GetPtrFromString(results.NextMarker), + Prefix: backend.GetPtrFromString(prefix), + StartAfter: backend.GetPtrFromString(*input.StartAfter), + }, nil +} + +func (p *Posix) PutBucketAcl(ctx context.Context, bucket string, data []byte) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + err = p.meta.StoreAttribute(nil, bucket, "", aclkey, data) + if err != nil { + return fmt.Errorf("set acl: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(*input.Bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, *input.Bucket) + } + err = p.doesBucketExist(*input.Bucket) + if err != nil { + return nil, err + } + + b, err := p.meta.RetrieveAttribute(nil, *input.Bucket, "", aclkey) + if errors.Is(err, meta.ErrNoSuchKey) { + return []byte{}, nil + } + if err != nil { + return nil, fmt.Errorf("get acl: %w", err) + } + return b, nil +} + +func (p *Posix) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + if tags == nil { + err = p.meta.DeleteAttribute(bucket, "", tagHdr) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove tags: %w", err) + } + + return nil + } + + b, err := json.Marshal(tags) + if err != nil { + return fmt.Errorf("marshal tags: %w", err) + } + + err = p.meta.StoreAttribute(nil, bucket, "", tagHdr, b) + if err != nil { + return fmt.Errorf("set tags: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + tags, err := p.getAttrTags(bucket, "", "") + if err != nil { + return nil, err + } + + return tags, nil +} + +func (p *Posix) DeleteBucketTagging(ctx context.Context, bucket string) error { + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + return p.PutBucketTagging(ctx, bucket, nil) +} + +func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId string) (map[string]string, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + if err := p.validateVersionId(versionId); err != nil { + return nil, err + } + + if versionId != "" && !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err + } + + err = p.ensureNotDeleteMarker(bucket, object, versionId) + if err != nil { + return nil, err + } + + return p.getAttrTags(bucket, object, versionId) +} + +func (p *Posix) getAttrTags(bucket, object, versionId string) (map[string]string, error) { + tags := make(map[string]string) + b, err := p.meta.RetrieveAttribute(nil, bucket, object, tagHdr) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return nil, s3err.GetNoSuchVersionErr(object, versionId) + } + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + if object != "" { + // return empty tag set for object tagging + return tags, nil + } + return nil, s3err.GetBucketErr(s3err.ErrBucketTaggingNotFound, bucket) + } + if err != nil { + return nil, fmt.Errorf("get tags: %w", err) + } + + err = json.Unmarshal(b, &tags) + if err != nil { + return nil, fmt.Errorf("unmarshal tags: %w", err) + } + + return tags, nil +} + +func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId string, tags map[string]string) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + if err := p.validateVersionId(versionId); err != nil { + return err + } + + if versionId != "" && !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err + } + + err = p.ensureNotDeleteMarker(bucket, object, versionId) + if err != nil { + return err + } + + if tags == nil { + err = p.meta.DeleteAttribute(bucket, object, tagHdr) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return s3err.GetNoSuchVersionErr(object, versionId) + } + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return nil + } + if err != nil { + return fmt.Errorf("remove tags: %w", err) + } + return nil + } + + err = p.storeObjectTags(bucket, object, tags) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return s3err.GetNoSuchVersionErr(object, versionId) + } + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return fmt.Errorf("set tags: %w", err) + } + + return nil +} + +// storeObjectTags stores tags as the tagging attribute of bucket/object +func (p *Posix) storeObjectTags(bucket, object string, tags map[string]string) error { + b, err := json.Marshal(tags) + if err != nil { + return fmt.Errorf("marshal tags: %w", err) + } + + return p.meta.StoreAttribute(nil, bucket, object, tagHdr, b) +} + +func (p *Posix) DeleteObjectTagging(ctx context.Context, bucket, object, versionId string) error { + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + return p.PutObjectTagging(ctx, bucket, object, versionId, nil) +} + +func (p *Posix) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + if policy == nil { + err := p.meta.DeleteAttribute(bucket, "", policykey) + if err != nil { + if errors.Is(err, meta.ErrNoSuchKey) { + return nil + } + + return fmt.Errorf("remove policy: %w", err) + } + + return nil + } + + err = p.meta.StoreAttribute(nil, bucket, "", policykey, policy) + if err != nil { + return fmt.Errorf("set policy: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + policy, err := p.meta.RetrieveAttribute(nil, bucket, "", policykey) + if errors.Is(err, meta.ErrNoSuchKey) { + return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucketPolicy, bucket) + } + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket) + } + if err != nil { + return nil, fmt.Errorf("get bucket policy: %w", err) + } + + return policy, nil +} + +func (p *Posix) DeleteBucketPolicy(ctx context.Context, bucket string) error { + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + return p.PutBucketPolicy(ctx, bucket, nil) +} + +func (p *Posix) PutBucketCors(ctx context.Context, bucket string, cors []byte) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + if cors == nil { + err = p.meta.DeleteAttribute(bucket, "", corskey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove cors: %w", err) + } + + return nil + } + + err = p.meta.StoreAttribute(nil, bucket, "", corskey, cors) + if err != nil { + return fmt.Errorf("set cors: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketCors(ctx context.Context, bucket string) ([]byte, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + cors, err := p.meta.RetrieveAttribute(nil, bucket, "", corskey) + if errors.Is(err, meta.ErrNoSuchKey) { + return nil, s3err.GetBucketErr(s3err.ErrNoSuchCORSConfiguration, bucket) + } + if err != nil { + return nil, err + } + + return cors, nil +} + +func (p *Posix) DeleteBucketCors(ctx context.Context, bucket string) error { + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + return p.PutBucketCors(ctx, bucket, nil) +} + +func (p *Posix) PutBucketWebsite(ctx context.Context, bucket string, website []byte) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + if website == nil { + err = p.meta.DeleteAttribute(bucket, "", websitekey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return fmt.Errorf("remove website: %w", err) + } + + return nil + } + + // The website configuration can be up to 128KB + // compress the data to fit in 64KB xattr limits + encoded, err := backend.MarshalWebsiteConfig(website, false) + if err != nil { + return err + } + + err = p.meta.StoreAttribute(nil, bucket, "", websitekey, encoded) + if err != nil { + return fmt.Errorf("set website: %w", err) + } + + return nil +} + +func (p *Posix) GetBucketWebsite(ctx context.Context, bucket string) ([]byte, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + website, err := p.meta.RetrieveAttribute(nil, bucket, "", websitekey) + if errors.Is(err, meta.ErrNoSuchKey) { + return nil, s3err.GetBucketErr(s3err.ErrNoSuchWebsiteConfiguration, bucket) + } + if err != nil { + return nil, err + } + + decoded, err := backend.UnmarshalWebsiteConfig(website, false) + if err != nil { + return nil, err + } + + return decoded, nil +} + +func (p *Posix) DeleteBucketWebsite(ctx context.Context, bucket string) error { + if !p.isBucketValid(bucket) { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + return p.PutBucketWebsite(ctx, bucket, nil) +} + +// bucketHasDefaultRetention reports whether bucket has Object Lock enabled +// with a default retention rule, which gives every object written to it +// Object Lock parameters. +func (p *Posix) bucketHasDefaultRetention(bucket string) (bool, error) { + cfg, err := p.meta.RetrieveAttribute(nil, bucket, "", bucketLockKey) + if errors.Is(err, meta.ErrNoSuchKey) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get object lock config: %w", err) + } + + var bucketLockConfig auth.BucketLockConfig + if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil { + return false, fmt.Errorf("parse bucket lock config: %w", err) + } + + return bucketLockConfig.Enabled && bucketLockConfig.DefaultRetention != nil, nil +} + +func (p *Posix) isBucketObjectLockEnabled(bucket string) error { + cfg, err := p.meta.RetrieveAttribute(nil, bucket, "", bucketLockKey) + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration) + } + if err != nil { + return fmt.Errorf("get object lock config: %w", err) + } + + var bucketLockConfig auth.BucketLockConfig + if err := json.Unmarshal(cfg, &bucketLockConfig); err != nil { + return fmt.Errorf("parse bucket lock config: %w", err) + } + + if !bucketLockConfig.Enabled { + return s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration) + } + + return nil +} + +func (p *Posix) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return err + } + + if p.versioningEnabled() { + // if versioning is enabled on gateway level and bucket versioning + // status is not `Enabled`, object lock can't be enabled. + // if object lock has been enabled on bucket creation + // it means the versioning has been enabled alongside with object lock + // and it can't be suspended ever again + status, err := p.getBucketVersioningStatus(ctx, bucket) + if err != nil { + return err + } + + if status != types.BucketVersioningStatusEnabled { + // if versioning is enabled on gateway level + return s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotAllowed) + } + } + + err = p.meta.StoreAttribute(nil, bucket, "", bucketLockKey, config) + if err != nil { + return fmt.Errorf("set object lock config: %w", err) + } + + return nil +} + +func (p *Posix) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketExist(bucket) + if err != nil { + return nil, err + } + + cfg, err := p.meta.RetrieveAttribute(nil, bucket, "", bucketLockKey) + if errors.Is(err, meta.ErrNoSuchKey) { + return nil, s3err.GetBucketErr(s3err.ErrObjectLockConfigurationNotFound, bucket) + } + if err != nil { + return nil, fmt.Errorf("get object lock config: %w", err) + } + + return cfg, nil +} + +func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketAndObjectExist(bucket, object, versionId) + if err != nil { + return err + } + err = p.isBucketObjectLockEnabled(bucket) + if err != nil { + return err + } + + if err := p.validateVersionId(versionId); err != nil { + return err + } + + var statusData []byte + if status { + statusData = []byte{1} + } else { + statusData = []byte{0} + } + + if versionId != "" { + if !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err + } + } + + err = p.ensureNotDeleteMarker(bucket, object, versionId) + if err != nil { + return err + } + + err = p.meta.StoreAttribute(nil, bucket, object, objectLegalHoldKey, statusData) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return s3err.GetNoSuchVersionErr(object, versionId) + } + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return fmt.Errorf("set object lock config: %w", err) + } + + return nil +} + +func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketAndObjectExist(bucket, object, versionId) + if err != nil { + return nil, err + } + err = p.isBucketObjectLockEnabled(bucket) + if err != nil { + return nil, err + } + + if err := p.validateVersionId(versionId); err != nil { + return nil, err + } + + if versionId != "" { + if !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err + } + } + + err = p.ensureNotDeleteMarker(bucket, object, versionId) + if err != nil { + return nil, err + } + + data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectLegalHoldKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return nil, s3err.GetNoSuchVersionErr(object, versionId) + } + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration) + } + if err != nil { + return nil, fmt.Errorf("get object lock config: %w", err) + } + + result := data[0] == 1 + + return &result, nil +} + +func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionId string, retention []byte) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketAndObjectExist(bucket, object, versionId) + if err != nil { + return err + } + err = p.isBucketObjectLockEnabled(bucket) + if err != nil { + return err + } + + if err := p.validateVersionId(versionId); err != nil { + return err + } + + if versionId != "" { + if !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return err + } + } + + err = p.ensureNotDeleteMarker(bucket, object, versionId) + if err != nil { + return err + } + + err = p.meta.StoreAttribute(nil, bucket, object, objectRetentionKey, retention) + if err != nil { + return fmt.Errorf("set object lock config: %w", err) + } + + return nil +} + +func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + if !p.isBucketValid(bucket) { + return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + err = p.doesBucketAndObjectExist(bucket, object, versionId) + if err != nil { + return nil, err + } + err = p.isBucketObjectLockEnabled(bucket) + if err != nil { + return nil, err + } + + if err := p.validateVersionId(versionId); err != nil { + return nil, err + } + + if versionId != "" { + if !p.versioningEnabled() { + //TODO: Maybe we need to return our custom error here? + return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) + } + bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) + if err != nil { + return nil, err + } + } + + err = p.ensureNotDeleteMarker(bucket, object, versionId) + if err != nil { + return nil, err + } + + data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectRetentionKey) + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { + if versionId != "" { + return nil, s3err.GetNoSuchVersionErr(object, versionId) + } + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if errors.Is(err, meta.ErrNoSuchKey) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration) + } + if err != nil { + return nil, fmt.Errorf("get object lock config: %w", err) + } + + return data, nil +} + +func (p *Posix) ChangeBucketOwner(ctx context.Context, bucket, owner string) (err error) { + defer backend.MapPermissionErr(&err) + + release, err := p.acquireActionSlot(ctx) + if err != nil { + return err + } + defer release() + + if !p.isBucketValid(bucket) { + return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) + } + return auth.UpdateBucketACLOwner(ctx, p, bucket, owner) +} + +// listBucketFileInfos returns the file info of every bucket directory under +// rootdir. With bucketlinks, a symlink to a directory counts as a bucket too. +func listBucketFileInfos(rootdir string, bucketlinks bool) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(rootdir) + if err != nil { + return nil, fmt.Errorf("readdir buckets: %w", err) + } + + var fis []fs.FileInfo + for _, entry := range entries { + if entry.Name() == objLockDir { + continue + } + + fi, err := entry.Info() + if err != nil { + continue + } + + if bucketlinks && entry.Type() == fs.ModeSymlink { + fi, err = os.Stat(filepath.Join(rootdir, entry.Name())) + if err != nil { + // skip entries returning errors + continue + } + } + + if !fi.IsDir() { + // buckets must be a directory + continue + } + + fis = append(fis, fi) + } + + return fis, nil +} + +func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) { + release, err := p.acquireActionSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + fis, err := listBucketFileInfos(p.rootPath(), p.bucketlinks) + if err != nil { + return buckets, fmt.Errorf("listBucketFileInfos: %w", err) + } + + for _, fi := range fis { + aclJSON, err := p.meta.RetrieveAttribute(nil, fi.Name(), "", aclkey) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return buckets, fmt.Errorf("get acl tag: %w", err) + } + + acl, err := auth.ParseACL(aclJSON) + if err != nil { + return buckets, fmt.Errorf("parse acl tag: %w", err) + } + + buckets = append(buckets, s3response.Bucket{ + Name: fi.Name(), + Owner: acl.Owner, + }) + } + + sort.SliceStable(buckets, func(i, j int) bool { + return buckets[i].Name < buckets[j].Name + }) + + return buckets, nil +} + +// NormalizeObjectKey resolves object relative to bucket the same way the +// filesystem will (collapsing ".."/"." segments, catching a traversal +// attempt that escapes bucket), but the result names an S3 key, not a host +// path: on Windows filepath.Join/Rel would return it with "\" separators, +// which callers building a policy-resource ARN or match string must never +// see, so it's converted back to "/" before returning. +func (p *Posix) NormalizeObjectKey(bucket, object string) string { + fullPath := filepath.Join(bucket, object) + key, err := filepath.Rel(filepath.Clean(bucket), fullPath) + if err != nil { + return filepath.ToSlash(fullPath) + } + if key == "." { + return "" + } + + return filepath.ToSlash(key) +} + +func (p *Posix) storeChecksums(f *os.File, bucket, object string, chs s3response.Checksum) error { + checksums, err := json.Marshal(chs) + if err != nil { + return fmt.Errorf("parse checksum: %w", err) + } + + return p.meta.StoreAttribute(f, bucket, object, checksumsKey, checksums) +} + +func (p *Posix) retrieveChecksums(f *os.File, bucket, object string) (checksums s3response.Checksum, err error) { + checksumsAtr, err := p.meta.RetrieveAttribute(f, bucket, object, checksumsKey) + if err != nil { + return checksums, err + } + + err = json.Unmarshal(checksumsAtr, &checksums) + return checksums, err +} + +func getString(str *string) string { + if str == nil { + return "" + } + return *str +} + +// drainBody consumes and discards all remaining bytes from r. +// It is called after a server error is detected mid-stream so that +// the client can read the error response before the write-side of the +// connection is shut down. +func drainBody(r io.Reader) { + if r == nil { + return + } + _, _ = io.Copy(io.Discard, r) +} + +func joinPathWithTrailer(paths ...string) string { + joined := filepath.Join(paths...) + if strings.HasSuffix(paths[len(paths)-1], "/") { + joined += "/" + } + return joined +} diff --git a/tests/integration/ListMultipartUploads.go b/tests/integration/ListMultipartUploads.go index 136e15d..f163875 100644 --- a/tests/integration/ListMultipartUploads.go +++ b/tests/integration/ListMultipartUploads.go @@ -206,9 +206,6 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error { return fmt.Errorf("expected multipart uploads to be %v, instead got %v", uploads, out.Uploads) } - if getString(out.UploadIdMarker) != "" { - return fmt.Errorf("expected empty upload id marker, instead got %s", getString(out.UploadIdMarker)) - } // should ignore invalid uploaId marker ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) @@ -224,9 +221,6 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error { return fmt.Errorf("expected multipart uploads to be %v, instead got %v", uploads, out.Uploads) } - if getString(out.UploadIdMarker) != "" { - return fmt.Errorf("expected empty upload id marker, instead got %s", getString(out.UploadIdMarker)) - } return nil }) diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 4bc4257..0b5eb90 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -542,7 +542,6 @@ func TestListMultipartUploads(ts *TestState) { ts.Run(ListMultipartUploads_empty_result) ts.Run(ListMultipartUploads_invalid_max_uploads) ts.Run(ListMultipartUploads_max_uploads) - ts.Run(ListMultipartUploads_next_markers_not_truncated) ts.Run(ListMultipartUploads_exceeding_max_uploads) ts.Run(ListMultipartUploads_ignore_upload_id_marker) ts.Run(ListMultipartUploads_invalid_uploadId_marker) @@ -3251,7 +3250,6 @@ func GetIntTests() IntTests { "ListMultipartUploads_empty_result": ListMultipartUploads_empty_result, "ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads, "ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads, - "ListMultipartUploads_next_markers_not_truncated": ListMultipartUploads_next_markers_not_truncated, "ListMultipartUploads_exceeding_max_uploads": ListMultipartUploads_exceeding_max_uploads, "ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker, "ListMultipartUploads_invalid_uploadId_marker": ListMultipartUploads_invalid_uploadId_marker, diff --git a/tests/test_rest_list_multipart_uploads.sh b/tests/test_rest_list_multipart_uploads.sh index 9ea8d44..6038dad 100644 --- a/tests/test_rest_list_multipart_uploads.sh +++ b/tests/test_rest_list_multipart_uploads.sh @@ -50,6 +50,9 @@ source ./tests/setup.sh # tags: curl,ListMultipartUploads,multipart,uploadId,next-key-marker,next-upload-id-marker @test "REST - ListMultipartUploads - NextKeyMarker and NextUploadIdMarker set to last item" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2144" + fi run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 assert_success read -r bucket_name test_file_one test_file_two <<< "$output"