dualfroz
08d5acf963
NextKeyMarker and NextUploadIdMarker were only set for a truncated listing, so a ListMultipartUploads response that fit in one page returned them empty. S3 still sets them to the key and upload ID of the last upload in the response. Fill both markers from the last listed upload when the listing is not truncated, including the case where the page ends exactly at max-uploads. A page of common prefixes only keeps them empty, as before. Enable the REST test that covers it. Fixes #2144
726 lines
22 KiB
Go
726 lines
22 KiB
Go
// Copyright 2023 Versity Software
|
|
// This file is licensed under the Apache License, Version 2.0
|
|
// (the "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
package integration
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/google/uuid"
|
|
"github.com/versity/versitygw/s3err"
|
|
)
|
|
|
|
func ListMultipartUploads_non_existing_bucket(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_non_existing_bucket"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
bucketName := getBucketName()
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucketName,
|
|
})
|
|
cancel()
|
|
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_empty_result(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_empty_result"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(out.Uploads) != 0 {
|
|
return fmt.Errorf("expected empty uploads, instead got %+v",
|
|
out.Uploads)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_invalid_max_uploads(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_invalid_max_uploads"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
maxUploads := int32(-3)
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
MaxUploads: &maxUploads,
|
|
})
|
|
cancel()
|
|
if err := checkApiErr(err, s3err.GetInvalidArgNegativeMaxLimiter("max-uploads", fmt.Sprint(maxUploads))); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_max_uploads(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_max_uploads"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
uploads := []types.MultipartUpload{}
|
|
for i := 1; i < 6; i++ {
|
|
out, err := createMp(s3client, bucket, fmt.Sprintf("obj%v", i))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
UploadId: out.UploadId,
|
|
Key: out.Key,
|
|
StorageClass: types.StorageClassStandard,
|
|
})
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
maxUploads := int32(2)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
MaxUploads: &maxUploads,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if out.IsTruncated == nil {
|
|
return fmt.Errorf("unexpected nil is-truncated")
|
|
}
|
|
if out.MaxUploads == nil {
|
|
return fmt.Errorf("unexpected nil max-uploads")
|
|
}
|
|
if !*out.IsTruncated {
|
|
return fmt.Errorf("expected the output to be truncated")
|
|
}
|
|
if *out.MaxUploads != 2 {
|
|
return fmt.Errorf("expected max-uploads to be 2, instead got %v",
|
|
out.MaxUploads)
|
|
}
|
|
if ok := compareMultipartUploads(out.Uploads, uploads[:2]); !ok {
|
|
return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
|
|
uploads[:2], out.Uploads)
|
|
}
|
|
if getString(out.NextKeyMarker) != getString(uploads[1].Key) {
|
|
return fmt.Errorf("expected next-key-marker to be %v, instead got %v",
|
|
getString(uploads[1].Key), getString(out.NextKeyMarker))
|
|
}
|
|
if getString(out.NextUploadIdMarker) != getString(uploads[1].UploadId) {
|
|
return fmt.Errorf("expected next-upload-id-marker to be %v, instead got %v",
|
|
getString(uploads[1].UploadId), getString(out.NextUploadIdMarker))
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
KeyMarker: out.NextKeyMarker,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if ok := compareMultipartUploads(out.Uploads, uploads[2:]); !ok {
|
|
return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
|
|
uploads[2:], out.Uploads)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_next_markers_not_truncated(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_next_markers_not_truncated"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
uploads := []types.MultipartUpload{}
|
|
for _, key := range []string{"bar", "baz", "foo"} {
|
|
out, err := createMp(s3client, bucket, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
UploadId: out.UploadId,
|
|
Key: out.Key,
|
|
StorageClass: types.StorageClassStandard,
|
|
})
|
|
}
|
|
last := uploads[len(uploads)-1]
|
|
|
|
// a single page still reports the last upload in the next markers,
|
|
// with the default max-uploads and with one equal to the upload count
|
|
for _, maxUploads := range []*int32{nil, getPtr(int32(len(uploads)))} {
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
MaxUploads: maxUploads,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if out.IsTruncated == nil || *out.IsTruncated {
|
|
return fmt.Errorf("expected a non-truncated response")
|
|
}
|
|
if ok := compareMultipartUploads(out.Uploads, uploads); !ok {
|
|
return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
|
|
uploads, out.Uploads)
|
|
}
|
|
if getString(out.NextKeyMarker) != getString(last.Key) {
|
|
return fmt.Errorf("expected next-key-marker to be %v, instead got %v",
|
|
getString(last.Key), getString(out.NextKeyMarker))
|
|
}
|
|
if getString(out.NextUploadIdMarker) != getString(last.UploadId) {
|
|
return fmt.Errorf("expected next-upload-id-marker to be %v, instead got %v",
|
|
getString(last.UploadId), getString(out.NextUploadIdMarker))
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_exceeding_max_uploads(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_exceeding_max_uploads"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
maxUploads := int32(1343235)
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
MaxUploads: &maxUploads,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if res.MaxUploads == nil {
|
|
return fmt.Errorf("unexpected nil max-uploads")
|
|
}
|
|
if *res.MaxUploads != 1000 {
|
|
return fmt.Errorf("expected max-uploads to be %v, instaed got %v",
|
|
1000, *res.MaxUploads)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_ignore_upload_id_marker"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
uploads := []types.MultipartUpload{}
|
|
for i := 1; i < 6; i++ {
|
|
out, err := createMp(s3client, bucket, fmt.Sprintf("obj%v", i))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
UploadId: out.UploadId,
|
|
Key: out.Key,
|
|
StorageClass: types.StorageClassStandard,
|
|
})
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
UploadIdMarker: uploads[2].UploadId,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !compareMultipartUploads(out.Uploads, uploads) {
|
|
return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
|
|
uploads, out.Uploads)
|
|
}
|
|
|
|
// should ignore invalid uploaId marker
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
UploadIdMarker: getPtr("invalid_uploadId_marker"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !compareMultipartUploads(out.Uploads, uploads) {
|
|
return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
|
|
uploads, out.Uploads)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_invalid_uploadId_marker(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_invalid_uploadId_marker"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
uploads := make([]types.MultipartUpload, 0, 5)
|
|
for i := range 5 {
|
|
out, err := createMp(s3client, bucket, fmt.Sprintf("obj-%v", i))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
UploadId: out.UploadId,
|
|
Key: out.Key,
|
|
StorageClass: types.StorageClassStandard,
|
|
})
|
|
}
|
|
|
|
// invalid UUID
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
KeyMarker: getPtr("obj-2"),
|
|
UploadIdMarker: getPtr("invalid_uploadId_marker"),
|
|
})
|
|
cancel()
|
|
if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, "invalid_uploadId_marker")); err != nil {
|
|
return err
|
|
}
|
|
|
|
// valid UUID, but not from the list
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
uploadIDMarker := uuid.New().String()
|
|
_, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
KeyMarker: getPtr("obj-2"),
|
|
UploadIdMarker: &uploadIDMarker,
|
|
})
|
|
cancel()
|
|
if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, uploadIDMarker)); err != nil {
|
|
return err
|
|
}
|
|
|
|
// uploadId marker and key marker mismatch
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
KeyMarker: getPtr("obj-2"),
|
|
UploadIdMarker: uploads[4].UploadId,
|
|
})
|
|
cancel()
|
|
return checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, *uploads[4].UploadId))
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_keyMarker_not_from_list(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_keyMarker_not_from_list"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
uploads := make([]types.MultipartUpload, 0, 9)
|
|
for _, mp := range []struct {
|
|
key string
|
|
count int
|
|
}{
|
|
{"bar", 3},
|
|
{"baz", 4},
|
|
{"foo", 2},
|
|
} {
|
|
for range mp.count {
|
|
out, err := createMp(s3client, bucket, mp.key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
Key: out.Key,
|
|
UploadId: out.UploadId,
|
|
StorageClass: types.StorageClassStandard,
|
|
})
|
|
// add an artificial delay, since some backends
|
|
// (e.g. azure, or posix on filesystems with coarse
|
|
// mtime resolution) can report the same identical
|
|
// creation/initiated time for uploads created back
|
|
// to back, making the resulting order non-deterministic
|
|
time.Sleep(time.Second)
|
|
}
|
|
}
|
|
|
|
// without uploadId marker
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
KeyMarker: getPtr("bat"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !compareMultipartUploads(uploads[3:], out.Uploads) {
|
|
return fmt.Errorf("expected the mp list to be %v, instead got %v", uploads[3:], out.Uploads)
|
|
}
|
|
|
|
// should start the listing after the specified uploadId marker
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
KeyMarker: getPtr("bat"),
|
|
UploadIdMarker: uploads[4].UploadId,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !compareMultipartUploads(uploads[5:], out.Uploads) {
|
|
return fmt.Errorf("expected the mp list to be %v, instead got %v", uploads[5:], out.Uploads)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_delimiter_truncated(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_delimiter_truncated"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
uploads := make([]types.MultipartUpload, 0, 6)
|
|
for _, key := range []string{
|
|
"abc/something",
|
|
"foo/bar/baz",
|
|
"foo/quxx",
|
|
"xyz/hello",
|
|
"zzz/bca",
|
|
"some/very/nested/mp/object",
|
|
} {
|
|
out, err := createMp(s3client, bucket, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
Key: out.Key,
|
|
UploadId: out.UploadId,
|
|
StorageClass: types.StorageClassStandard,
|
|
})
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
Delimiter: getPtr("/"),
|
|
MaxUploads: getPtr(int32(2)),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(out.Uploads) != 0 {
|
|
return fmt.Errorf("expected empty uplodas list, instead got %v", out.Uploads)
|
|
}
|
|
expectedCps := []string{"abc/", "foo/"}
|
|
if !comparePrefixes(expectedCps, out.CommonPrefixes) {
|
|
return fmt.Errorf("expected the common prefixes to be %v, instead got %v", expectedCps, out.CommonPrefixes)
|
|
}
|
|
if getString(out.NextKeyMarker) != "foo/" {
|
|
return fmt.Errorf("expected the next key marker to be 'foo/', instead got %s", getString(out.NextKeyMarker))
|
|
}
|
|
if getString(out.NextUploadIdMarker) != getString(uploads[1].UploadId) {
|
|
return fmt.Errorf("expected the next upload id marker to be %s, instead got %s", getString(uploads[1].UploadId), getString(out.NextUploadIdMarker))
|
|
}
|
|
if !*out.IsTruncated {
|
|
return fmt.Errorf("expected a truncated response")
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
out2, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
Delimiter: getPtr("/"),
|
|
UploadIdMarker: out.NextUploadIdMarker,
|
|
KeyMarker: out.NextKeyMarker,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(out2.Uploads) != 0 {
|
|
return fmt.Errorf("expected empty uplodas list, instead got %v", out2.Uploads)
|
|
}
|
|
expectedCps = []string{"foo/", "some/", "xyz/", "zzz/"}
|
|
if !comparePrefixes(expectedCps, out2.CommonPrefixes) {
|
|
return fmt.Errorf("expected the common prefixes to be %v, instead got %v", expectedCps, out2.CommonPrefixes)
|
|
}
|
|
if getString(out2.KeyMarker) != "foo/" {
|
|
return fmt.Errorf("expected key marker to be 'foo/', instead got %s", getString(out2.KeyMarker))
|
|
}
|
|
if getString(out2.UploadIdMarker) != getString(uploads[1].UploadId) {
|
|
return fmt.Errorf("expected the upload id marker to be %s, instead got %s", getString(uploads[1].UploadId), getString(out2.UploadIdMarker))
|
|
}
|
|
if getString(out2.NextKeyMarker) != "" {
|
|
return fmt.Errorf("expected empty next key marker, instead got %s", getString(out2.NextKeyMarker))
|
|
}
|
|
if getString(out2.NextUploadIdMarker) != "" {
|
|
return fmt.Errorf("expected empty next upload id marker, instead got %s", getString(out2.NextUploadIdMarker))
|
|
}
|
|
if *out2.IsTruncated {
|
|
return fmt.Errorf("expected a non-truncated response")
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_prefix(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_prefix"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
prefix := "foo"
|
|
uploads := make([]types.MultipartUpload, 0, 8)
|
|
for _, key := range []string{
|
|
"abc/something",
|
|
"foo/bar/baz",
|
|
"foo/quxx",
|
|
"hello/world",
|
|
"xyz/hello",
|
|
"zzz/bca",
|
|
"some/very/nested/mp/object",
|
|
"foo/xyz",
|
|
} {
|
|
out, err := createMp(s3client, bucket, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if strings.HasPrefix(key, prefix) {
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
Key: out.Key,
|
|
UploadId: out.UploadId,
|
|
StorageClass: types.StorageClassStandard,
|
|
})
|
|
}
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
Prefix: &prefix,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if getString(out.Prefix) != prefix {
|
|
return fmt.Errorf("expected the prefix to be %s, instead got %s", prefix, getString(out.Prefix))
|
|
}
|
|
if !compareMultipartUploads(out.Uploads, uploads) {
|
|
return fmt.Errorf("expected the uploads list to be %v, instead got %v", uploads, out.Uploads)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_both_delimiter_and_prefix(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_both_delimiter_and_prefix"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
for _, key := range []string{
|
|
"foo/abc/bbb/aaa/c",
|
|
"abc/something",
|
|
"foo/bar/baz",
|
|
"foo/quxx",
|
|
"hello/world",
|
|
"foo/random/object",
|
|
"foo/random/another/object",
|
|
"xyz/hello",
|
|
"zzz/bca",
|
|
"some/very/nested/mp/object",
|
|
"foo/xyz",
|
|
} {
|
|
_, err := createMp(s3client, bucket, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
Delimiter: getPtr("/"),
|
|
Prefix: getPtr("foo/"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
expectedCps := []string{"foo/abc/", "foo/bar/", "foo/random/"}
|
|
if !comparePrefixes(expectedCps, out.CommonPrefixes) {
|
|
return fmt.Errorf("expected the common prefixes to be %v, instead got %v", expectedCps, out.CommonPrefixes)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_delimiter_no_matches(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_delimiter_no_matches"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
uploads := make([]types.MultipartUpload, 0, 8)
|
|
for _, key := range []string{
|
|
"abc/something",
|
|
"foo/bar/baz",
|
|
"foo/quxx",
|
|
"hello/world",
|
|
"xyz/hello",
|
|
"zzz/bca",
|
|
"some/very/nested/mp/object",
|
|
"foo/xyz",
|
|
} {
|
|
out, err := createMp(s3client, bucket, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
Key: out.Key,
|
|
UploadId: out.UploadId,
|
|
StorageClass: types.StorageClassStandard,
|
|
})
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
Delimiter: getPtr("delim"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
sort.SliceStable(uploads, func(i, j int) bool {
|
|
return *uploads[i].Key < *uploads[j].Key
|
|
})
|
|
|
|
if !compareMultipartUploads(uploads, out.Uploads) {
|
|
return fmt.Errorf("expected the uploads to be %v, instead got %v", uploads, out.Uploads)
|
|
}
|
|
if len(out.CommonPrefixes) != 0 {
|
|
return fmt.Errorf("expected empty common prefixes, instead got %v", out.CommonPrefixes)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListMultipartUploads_with_checksums(s *S3Conf) error {
|
|
testName := "ListMultipartUploads_with_checksums"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
uploads := []types.MultipartUpload{}
|
|
for _, el := range []struct {
|
|
obj string
|
|
algo types.ChecksumAlgorithm
|
|
t types.ChecksumType
|
|
}{
|
|
{
|
|
obj: "obj-1",
|
|
algo: types.ChecksumAlgorithmCrc32,
|
|
t: types.ChecksumTypeComposite,
|
|
},
|
|
{
|
|
obj: "obj-2",
|
|
algo: types.ChecksumAlgorithmCrc32c,
|
|
t: types.ChecksumTypeFullObject,
|
|
},
|
|
{
|
|
obj: "obj-3",
|
|
algo: types.ChecksumAlgorithmSha1,
|
|
t: types.ChecksumTypeComposite,
|
|
},
|
|
{
|
|
obj: "obj-4",
|
|
algo: types.ChecksumAlgorithmSha256,
|
|
t: types.ChecksumTypeComposite,
|
|
},
|
|
{
|
|
obj: "obj-5",
|
|
algo: types.ChecksumAlgorithmCrc64nvme,
|
|
t: types.ChecksumTypeFullObject,
|
|
},
|
|
{
|
|
obj: "obj-6",
|
|
algo: types.ChecksumAlgorithmSha512,
|
|
t: types.ChecksumTypeComposite,
|
|
},
|
|
{
|
|
obj: "obj-7",
|
|
algo: types.ChecksumAlgorithmMd5,
|
|
t: types.ChecksumTypeComposite,
|
|
},
|
|
{
|
|
obj: "obj-8",
|
|
algo: types.ChecksumAlgorithmXxhash64,
|
|
t: types.ChecksumTypeComposite,
|
|
},
|
|
{
|
|
obj: "obj-9",
|
|
algo: types.ChecksumAlgorithmXxhash3,
|
|
t: types.ChecksumTypeComposite,
|
|
},
|
|
// obj-91 to preserve the lexicographical order
|
|
{
|
|
obj: "obj-91",
|
|
algo: types.ChecksumAlgorithmXxhash128,
|
|
t: types.ChecksumTypeComposite,
|
|
},
|
|
} {
|
|
key := el.obj
|
|
mp, err := createMp(s3client, bucket, key, withChecksum(el.algo), withChecksumType(el.t))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
uploads = append(uploads, types.MultipartUpload{
|
|
Key: &key,
|
|
UploadId: mp.UploadId,
|
|
StorageClass: types.StorageClassStandard,
|
|
ChecksumAlgorithm: el.algo,
|
|
ChecksumType: el.t,
|
|
})
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
|
Bucket: &bucket,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !compareMultipartUploads(res.Uploads, uploads) {
|
|
return fmt.Errorf("expected the final multipart uploads to be %v, instead got %v",
|
|
uploads, res.Uploads)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|