Upstream versity/versitygw state of the touched files before 592e91b10c

backend/mp-lister.go

@@ -127,8 +127,9 @@ func (l *MultipartUploadLister) Run() (*ListMultipartUploadsPage, error) {
 						if out.IsTruncated {
 							out.NextKeyMarker = lastKey
 							out.NextUploadIDMarker = up.UploadID
+							return out, nil
 						}
-						return out, nil
+						break
 					}
 				}
 				continue
@@ -140,11 +141,20 @@ func (l *MultipartUploadLister) Run() (*ListMultipartUploadsPage, error) {
 			if out.IsTruncated {
 				out.NextKeyMarker = lastKey
 				out.NextUploadIDMarker = up.UploadID
+				return out, nil
 			}
-			return out, nil
+			break
 		}
 	}
 
+	// a listing that is not truncated still reports its last upload
+	// in the next markers
+	if len(out.Uploads) != 0 {
+		last := out.Uploads[len(out.Uploads)-1]
+		out.NextKeyMarker = last.Key
+		out.NextUploadIDMarker = last.UploadID
+	}
+
 	return out, nil
 }
 

tests/integration/ListMultipartUploads.go

@@ -17,6 +17,7 @@ package integration
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"sort"
 	"strings"
 	"time"
@@ -152,6 +153,57 @@ func ListMultipartUploads_max_uploads(s *S3Conf) error {
 	})
 }
 
+func ListMultipartUploads_next_markers_not_truncated(s *S3Conf) error {
+	testName := "ListMultipartUploads_next_markers_not_truncated"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		uploads := []types.MultipartUpload{}
+		for _, key := range []string{"bar", "baz", "foo"} {
+			out, err := createMp(s3client, bucket, key)
+			if err != nil {
+				return err
+			}
+			uploads = append(uploads, types.MultipartUpload{
+				UploadId:     out.UploadId,
+				Key:          out.Key,
+				StorageClass: types.StorageClassStandard,
+			})
+		}
+		last := uploads[len(uploads)-1]
+
+		// a single page still reports the last upload in the next markers,
+		// with the default max-uploads and with one equal to the upload count
+		for _, maxUploads := range []*int32{nil, getPtr(int32(len(uploads)))} {
+			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+			out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
+				Bucket:     &bucket,
+				MaxUploads: maxUploads,
+			})
+			cancel()
+			if err != nil {
+				return err
+			}
+
+			if out.IsTruncated == nil || *out.IsTruncated {
+				return fmt.Errorf("expected a non-truncated response")
+			}
+			if ok := compareMultipartUploads(out.Uploads, uploads); !ok {
+				return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
+					uploads, out.Uploads)
+			}
+			if getString(out.NextKeyMarker) != getString(last.Key) {
+				return fmt.Errorf("expected next-key-marker to be %v, instead got %v",
+					getString(last.Key), getString(out.NextKeyMarker))
+			}
+			if getString(out.NextUploadIdMarker) != getString(last.UploadId) {
+				return fmt.Errorf("expected next-upload-id-marker to be %v, instead got %v",
+					getString(last.UploadId), getString(out.NextUploadIdMarker))
+			}
+		}
+
+		return nil
+	})
+}
+
 func ListMultipartUploads_exceeding_max_uploads(s *S3Conf) error {
 	testName := "ListMultipartUploads_exceeding_max_uploads"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -206,6 +258,9 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error {
 			return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
 				uploads, out.Uploads)
 		}
+		if getString(out.UploadIdMarker) != "" {
+			return fmt.Errorf("expected empty upload id marker, instead got %s", getString(out.UploadIdMarker))
+		}
 
 		// should ignore invalid uploaId marker
 		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
@@ -221,6 +276,9 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error {
 			return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
 				uploads, out.Uploads)
 		}
+		if getString(out.UploadIdMarker) != "" {
+			return fmt.Errorf("expected empty upload id marker, instead got %s", getString(out.UploadIdMarker))
+		}
 
 		return nil
 	})
@@ -622,6 +680,30 @@ func ListMultipartUploads_delimiter_no_matches(s *S3Conf) error {
 	})
 }
 
+func ListMultipartUploads_with_upload_id(s *S3Conf) error {
+	testName := "ListMultipartUploads_with_upload_id"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		out, err := createMp(s3client, bucket, "my-obj")
+		if err != nil {
+			return err
+		}
+
+		req, err := createSignedReq(http.MethodGet, s.endpoint,
+			fmt.Sprintf("%v?uploads&uploadId=%v", bucket, *out.UploadId),
+			s.awsID, s.awsSecret, "s3", s.awsRegion, "", nil, time.Now(), nil)
+		if err != nil {
+			return err
+		}
+
+		resp, err := s.httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		return checkHTTPResponseApiErr(resp, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdWithUploads, "uploadId"))
+	})
+}
+
 func ListMultipartUploads_with_checksums(s *S3Conf) error {
 	testName := "ListMultipartUploads_with_checksums"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {

tests/integration/group-tests.go

@@ -205,6 +205,8 @@ func TestPutObject(ts *TestState) {
 		ts.Run(PutObject_with_metadata)
 	}
 	ts.Run(PutObject_success)
+	ts.Run(PutObject_sse_c_invalid_headers)
+	ts.Run(PutObject_unsupported_sse_not_implemented)
 	ts.Run(PutObject_default_content_type)
 	ts.Run(PutObject_overwrite_resets_attributes)
 	ts.Run(PutObject_overwrite_multipart_object)
@@ -238,6 +240,7 @@ func TestHeadObject(ts *TestState) {
 		ts.Run(HeadObject_ranged_with_checksum_mode)
 	}
 	ts.Run(HeadObject_success)
+	ts.Run(HeadObject_sse_c_invalid_headers)
 	ts.Run(HeadObject_overrides_success)
 	ts.Run(HeadObject_overrides_presign_success)
 	ts.Run(HeadObject_overrides_fail_public)
@@ -417,6 +420,7 @@ func TestCopyObject(ts *TestState) {
 	}
 	ts.Run(CopyObject_with_special_characters)
 	ts.Run(CopyObject_success)
+	ts.Run(CopyObject_sse_c_invalid_copy_source_headers)
 	ts.Run(CopyObject_cross_bucket_server_side_copy)
 	ts.Run(CopyObject_incorrect_source_bucket_expected_owner)
 }
@@ -542,6 +546,7 @@ func TestListMultipartUploads(ts *TestState) {
 	ts.Run(ListMultipartUploads_empty_result)
 	ts.Run(ListMultipartUploads_invalid_max_uploads)
 	ts.Run(ListMultipartUploads_max_uploads)
+	ts.Run(ListMultipartUploads_next_markers_not_truncated)
 	ts.Run(ListMultipartUploads_exceeding_max_uploads)
 	ts.Run(ListMultipartUploads_ignore_upload_id_marker)
 	ts.Run(ListMultipartUploads_invalid_uploadId_marker)
@@ -551,6 +556,7 @@ func TestListMultipartUploads(ts *TestState) {
 	ts.Run(ListMultipartUploads_prefix)
 	ts.Run(ListMultipartUploads_both_delimiter_and_prefix)
 	ts.Run(ListMultipartUploads_delimiter_no_matches)
+	ts.Run(ListMultipartUploads_with_upload_id)
 	//TODO: remove the condition after implementing checksums in azure
 	if !ts.conf.azureTests {
 		ts.Run(ListMultipartUploads_with_checksums)
@@ -904,6 +910,28 @@ func TestNotImplementedActions(ts *TestState) {
 	ts.Run(GetObjectAcl_not_implemented)
 }
 
+// TestSSECNotImplemented is for backends that reject SSE-C rather than forward it.
+func TestSSECNotImplemented(ts *TestState) {
+	ts.Run(PutObject_sse_c_not_implemented)
+	ts.Run(PostObject_sse_c_not_implemented)
+	ts.Run(GetObject_sse_c_not_implemented)
+	ts.Run(HeadObject_sse_c_not_implemented)
+	ts.Run(GetObjectAttributes_sse_c_not_implemented)
+	ts.Run(CopyObject_sse_c_not_implemented)
+	ts.Run(CreateMultipartUpload_sse_c_not_implemented)
+	ts.Run(UploadPart_sse_c_not_implemented)
+	ts.Run(UploadPartCopy_sse_c_not_implemented)
+	ts.Run(CompleteMultipartUpload_sse_c_not_implemented)
+}
+
+func TestSSECTransport(ts *TestState) {
+	if ts.conf.tlsStatus {
+		TestSSECNotImplemented(ts)
+		return
+	}
+	ts.Run(PutObject_sse_c_requires_tls)
+}
+
 func TestWORMProtection(ts *TestState) {
 	ts.Run(WORMProtection_bucket_object_lock_configuration_compliance_mode)
 	ts.Run(WORMProtection_bucket_object_lock_configuration_governance_mode)
@@ -1004,6 +1032,9 @@ func TestFullFlow(ts *TestState) {
 	if ts.conf.versioningEnabled {
 		TestVersioning(ts)
 	}
+	if ts.conf.azureTests {
+		TestSSECTransport(ts)
+	}
 	TestGatewayIAM(ts)
 	TestServer(ts)
 }
@@ -1034,6 +1065,7 @@ func TestPosix(ts *TestState) {
 	if !ts.conf.versioningEnabled {
 		TestVersioningDisabled(ts)
 	}
+	TestSSECTransport(ts)
 }
 
 func TestScoutfs(ts *TestState) {
@@ -1135,6 +1167,7 @@ func TestScoutfs(ts *TestState) {
 	ts.Run(ObjectTagging_trailing_slash_counterpart)
 	ts.Run(ObjectLock_trailing_slash_counterpart)
 	ts.Run(DeleteObject_directory_not_empty)
+	TestSSECTransport(ts)
 }
 
 func TestGatewayIAM(ts *TestState) {
@@ -2160,9 +2193,11 @@ func TestRouter(ts *TestState) {
 	ts.Run(RouterGetUploadsWithKey)
 	ts.Run(RouterCopySourceNotAllowed)
 	ts.Run(RouterListVersionsWithKey)
+	ts.Run(RouterUploadsWithUploadId)
 	ts.Run(RouterObjectAnnotationNotImplemented)
 	ts.Run(RouterRenameObjectNotImplemented)
 	ts.Run(RouterObjectTorrentMethodNotAllowed)
+	ts.Run(RouterInvalidURLEscape)
 }
 
 func TestPostObject(ts *TestState) {
@@ -3251,6 +3286,7 @@ func GetIntTests() IntTests {
 		"ListMultipartUploads_empty_result":                                                ListMultipartUploads_empty_result,
 		"ListMultipartUploads_invalid_max_uploads":                                         ListMultipartUploads_invalid_max_uploads,
 		"ListMultipartUploads_max_uploads":                                                 ListMultipartUploads_max_uploads,
+		"ListMultipartUploads_next_markers_not_truncated":                                  ListMultipartUploads_next_markers_not_truncated,
 		"ListMultipartUploads_exceeding_max_uploads":                                       ListMultipartUploads_exceeding_max_uploads,
 		"ListMultipartUploads_ignore_upload_id_marker":                                     ListMultipartUploads_ignore_upload_id_marker,
 		"ListMultipartUploads_invalid_uploadId_marker":                                     ListMultipartUploads_invalid_uploadId_marker,
@@ -3259,6 +3295,7 @@ func GetIntTests() IntTests {
 		"ListMultipartUploads_delimiter_truncated":                                         ListMultipartUploads_delimiter_truncated,
 		"ListMultipartUploads_prefix":                                                      ListMultipartUploads_prefix,
 		"ListMultipartUploads_both_delimiter_and_prefix":                                   ListMultipartUploads_both_delimiter_and_prefix,
+		"ListMultipartUploads_with_upload_id":                                              ListMultipartUploads_with_upload_id,
 		"ListMultipartUploads_with_checksums":                                              ListMultipartUploads_with_checksums,
 		"AbortMultipartUpload_non_existing_bucket":                                         AbortMultipartUpload_non_existing_bucket,
 		"AbortMultipartUpload_incorrect_uploadId":                                          AbortMultipartUpload_incorrect_uploadId,
@@ -3541,6 +3578,21 @@ func GetIntTests() IntTests {
 		"PutObject_with_slashes":                                                           PutObject_with_slashes,
 		"PutObject_race_with_delete":                                                       PutObject_race_with_delete,
 		"CreateMultipartUpload_dir_obj":                                                    CreateMultipartUpload_dir_obj,
+		"PutObject_sse_c_not_implemented":                                                  PutObject_sse_c_not_implemented,
+		"PutObject_unsupported_sse_not_implemented":                                        PutObject_unsupported_sse_not_implemented,
+		"PutObject_sse_c_requires_tls":                                                     PutObject_sse_c_requires_tls,
+		"PostObject_sse_c_not_implemented":                                                 PostObject_sse_c_not_implemented,
+		"GetObject_sse_c_not_implemented":                                                  GetObject_sse_c_not_implemented,
+		"HeadObject_sse_c_not_implemented":                                                 HeadObject_sse_c_not_implemented,
+		"GetObjectAttributes_sse_c_not_implemented":                                        GetObjectAttributes_sse_c_not_implemented,
+		"CopyObject_sse_c_not_implemented":                                                 CopyObject_sse_c_not_implemented,
+		"CreateMultipartUpload_sse_c_not_implemented":                                      CreateMultipartUpload_sse_c_not_implemented,
+		"UploadPart_sse_c_not_implemented":                                                 UploadPart_sse_c_not_implemented,
+		"UploadPartCopy_sse_c_not_implemented":                                             UploadPartCopy_sse_c_not_implemented,
+		"CompleteMultipartUpload_sse_c_not_implemented":                                    CompleteMultipartUpload_sse_c_not_implemented,
+		"PutObject_sse_c_invalid_headers":                                                  PutObject_sse_c_invalid_headers,
+		"HeadObject_sse_c_invalid_headers":                                                 HeadObject_sse_c_invalid_headers,
+		"CopyObject_sse_c_invalid_copy_source_headers":                                     CopyObject_sse_c_invalid_copy_source_headers,
 		"IAM_user_access_denied":                                                           IAM_user_access_denied,
 		"IAM_userplus_access_denied":                                                       IAM_userplus_access_denied,
 		"IAM_userplus_CreateBucket":                                                        IAM_userplus_CreateBucket,
@@ -3748,9 +3800,11 @@ func GetIntTests() IntTests {
 		"RouterGetUploadsWithKey":                                                          RouterGetUploadsWithKey,
 		"RouterCopySourceNotAllowed":                                                       RouterCopySourceNotAllowed,
 		"RouterListVersionsWithKey":                                                        RouterListVersionsWithKey,
+		"RouterUploadsWithUploadId":                                                        RouterUploadsWithUploadId,
 		"RouterObjectAnnotationNotImplemented":                                             RouterObjectAnnotationNotImplemented,
 		"RouterRenameObjectNotImplemented":                                                 RouterRenameObjectNotImplemented,
 		"RouterObjectTorrentMethodNotAllowed":                                              RouterObjectTorrentMethodNotAllowed,
+		"RouterInvalidURLEscape":                                                           RouterInvalidURLEscape,
 		"UnsignedStreaminPayloadTrailer_malformed_trailer":                                 UnsignedStreaminPayloadTrailer_malformed_trailer,
 		"UnsignedStreamingPayloadTrailer_aborted_connection":                               UnsignedStreamingPayloadTrailer_aborted_connection,
 		"UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length":               UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length,