Upstream versity/versitygw state of the touched files before 95a6da83b9

backend/mp-lister.go

@@ -0,0 +1,205 @@
+// Copyright 2026 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package backend
+
+import (
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/versity/versitygw/s3err"
+	"github.com/versity/versitygw/s3response"
+)
+
+// ListMultipartUploads initializes a multipart upload lister and calls Run()
+func ListMultipartUploads(uploads []s3response.Upload, prefix, delimiter, keyMarker, uploadIdMarker string, maxUploads int) (*ListMultipartUploadsPage, error) {
+	lister := &MultipartUploadLister{
+		Uploads:        uploads,
+		Prefix:         prefix,
+		Delimiter:      delimiter,
+		KeyMarker:      keyMarker,
+		UploadIDMarker: uploadIdMarker,
+		MaxUploads:     maxUploads,
+	}
+
+	return lister.Run()
+}
+
+// MultipartUploadLister emits a ListMultipartUploads-compatible page from an
+// already-sorted, already prefix- and key-marker-filtered upload list.
+//
+// Assumptions about input Uploads:
+//   - Sorted by (Key asc, Initiated asc)
+//   - Filtered by Prefix
+//   - Filtered to start strictly after key-marker when key-marker was provided.
+type MultipartUploadLister struct {
+	Uploads        []s3response.Upload
+	Prefix         string
+	Delimiter      string
+	MaxUploads     int
+	KeyMarker      string
+	UploadIDMarker string
+}
+
+// ListMultipartUploadsPage is the lister output
+type ListMultipartUploadsPage struct {
+	Uploads            []s3response.Upload
+	CommonPrefixes     []s3response.CommonPrefix
+	IsTruncated        bool
+	NextKeyMarker      string
+	NextUploadIDMarker string
+}
+
+// Run validates marker constraints, then performs a single-pass list that:
+// - collapses uploads into CommonPrefixes when delimiter is set
+// - enforces MaxUploads over (Uploads + CommonPrefixes)
+// - computes truncation and next markers
+func (l *MultipartUploadLister) Run() (*ListMultipartUploadsPage, error) {
+	out := &ListMultipartUploadsPage{}
+
+	var startIndex int
+
+	// if upload-id-marker is provided without a corresponding key-marker, ignore it.
+	uploadIDMarker := l.UploadIDMarker
+	if l.KeyMarker == "" {
+		uploadIDMarker = ""
+	}
+
+	if uploadIDMarker != "" {
+		// any invalid uuid is considered as an invalid uploadIdMarker
+		_, err := uuid.Parse(uploadIDMarker)
+		if err != nil {
+			return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, uploadIDMarker)
+		}
+		startIndex = l.findUploadIdMarkerIndex(uploadIDMarker)
+		if startIndex == -1 {
+			return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, uploadIDMarker)
+		}
+		if startIndex >= len(l.Uploads) {
+			return out, nil
+		}
+	}
+
+	// Common prefix uniqueness tracking.
+	seenCP := make(map[string]struct{})
+
+	emitted := 0
+	var lastKey string
+
+	// emitUpload appends a new upload to out.Uplodas
+	emitUpload := func(up s3response.Upload) bool {
+		out.Uploads = append(out.Uploads, up)
+		emitted++
+		lastKey = up.Key
+		return emitted == l.MaxUploads
+	}
+	// emitCp appends a new common prefix to out.CommonPrefixes
+	emitCP := func(cpref string) bool {
+		out.CommonPrefixes = append(out.CommonPrefixes, s3response.CommonPrefix{Prefix: cpref})
+		emitted++
+		lastKey = cpref
+		return emitted == l.MaxUploads
+	}
+
+	for i, up := range l.Uploads[startIndex:] {
+		if l.Delimiter != "" {
+			// delimiter check
+			suffix := strings.TrimPrefix(up.Key, l.Prefix)
+			before, _, found := strings.Cut(suffix, l.Delimiter)
+			if found {
+				cpref := l.Prefix + before + l.Delimiter
+				if _, ok := seenCP[cpref]; !ok {
+					seenCP[cpref] = struct{}{}
+					if emitCP(cpref) {
+						out.IsTruncated = l.hasMoreAfter(i+1, seenCP)
+						if out.IsTruncated {
+							out.NextKeyMarker = lastKey
+							out.NextUploadIDMarker = up.UploadID
+						}
+						return out, nil
+					}
+				}
+				continue
+			}
+		}
+
+		if emitUpload(up) {
+			out.IsTruncated = l.hasMoreAfter(i+1, seenCP)
+			if out.IsTruncated {
+				out.NextKeyMarker = lastKey
+				out.NextUploadIDMarker = up.UploadID
+			}
+			return out, nil
+		}
+	}
+
+	return out, nil
+}
+
+// findUploadIdMarkerIndex finds the index of given uploadId marker in uploads
+// uploadIDMarker must match an upload-id among uploads with the first key after KeyMarker.
+// Since caller filtered to Key > KeyMarker and the list is sorted by key/time,
+// the first key after KeyMarker is Uploads[0].Key (if any).
+// -1 is returned if no uploadId is found
+func (l *MultipartUploadLister) findUploadIdMarkerIndex(uploadIDMarker string) int {
+	if len(l.Uploads) == 0 {
+		// key-marker provided but nothing after it => upload-id-marker can never be valid.
+		return -1
+	}
+	firstKey := l.Uploads[0].Key
+
+	// it must match an upload-id under firstKey only.
+	// If firstKey has multiple uploads, any of those IDs is valid.
+	for i, up := range l.Uploads {
+		if up.Key != firstKey {
+			// sorted by key, so we're past firstKey group
+			break
+		}
+		if up.UploadID == uploadIDMarker {
+			// the listing should start from the next index
+			// to skip the uploadId marker
+			return i + 1
+		}
+	}
+	return -1
+}
+
+// hasMoreAfter checks if there exists at least one more effective item after idx,
+// considering delimiter collapse and already-emitted common prefixes.
+func (l *MultipartUploadLister) hasMoreAfter(idx int, seenCP map[string]struct{}) bool {
+	if idx >= len(l.Uploads) {
+		return false
+	}
+	if l.Delimiter == "" {
+		// any remaining upload would be emitted
+		return true
+	}
+
+	for i := idx; i < len(l.Uploads); i++ {
+		up := l.Uploads[i]
+		suffix := strings.TrimPrefix(up.Key, l.Prefix)
+		before, _, found := strings.Cut(suffix, l.Delimiter)
+		if !found {
+			// would emit an upload
+			return true
+		}
+		cpref := l.Prefix + before + l.Delimiter
+		if _, ok := seenCP[cpref]; ok {
+			continue
+		}
+		// would emit a new common prefix
+		return true
+	}
+	return false
+}

tests/integration/ListMultipartUploads.go

@@ -17,7 +17,6 @@ package integration
 import (
 	"context"
 	"fmt"
-	"net/http"
 	"sort"
 	"strings"
 	"time"
@@ -580,30 +579,6 @@ func ListMultipartUploads_delimiter_no_matches(s *S3Conf) error {
 	})
 }
 
-func ListMultipartUploads_with_upload_id(s *S3Conf) error {
-	testName := "ListMultipartUploads_with_upload_id"
-	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
-		out, err := createMp(s3client, bucket, "my-obj")
-		if err != nil {
-			return err
-		}
-
-		req, err := createSignedReq(http.MethodGet, s.endpoint,
-			fmt.Sprintf("%v?uploads&uploadId=%v", bucket, *out.UploadId),
-			s.awsID, s.awsSecret, "s3", s.awsRegion, "", nil, time.Now(), nil)
-		if err != nil {
-			return err
-		}
-
-		resp, err := s.httpClient.Do(req)
-		if err != nil {
-			return err
-		}
-
-		return checkHTTPResponseApiErr(resp, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdWithUploads, "uploadId"))
-	})
-}
-
 func ListMultipartUploads_with_checksums(s *S3Conf) error {
 	testName := "ListMultipartUploads_with_checksums"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {

tests/integration/group-tests.go

@@ -110,8 +110,6 @@ func TestListBuckets(ts *TestState) {
 	ts.Sync(ListBuckets_as_admin)
 	ts.Sync(ListBuckets_with_prefix)
 	ts.Sync(ListBuckets_invalid_max_buckets)
-	ts.Sync(ListBuckets_invalid_bucket_region)
-	ts.Sync(ListBuckets_bucket_region)
 	ts.Sync(ListBuckets_truncated)
 	ts.Sync(ListBuckets_success)
 	ts.Sync(ListBuckets_empty_success)
@@ -207,8 +205,6 @@ func TestPutObject(ts *TestState) {
 		ts.Run(PutObject_with_metadata)
 	}
 	ts.Run(PutObject_success)
-	ts.Run(PutObject_sse_c_invalid_headers)
-	ts.Run(PutObject_unsupported_sse_not_implemented)
 	ts.Run(PutObject_default_content_type)
 	ts.Run(PutObject_overwrite_resets_attributes)
 	ts.Run(PutObject_overwrite_multipart_object)
@@ -242,7 +238,6 @@ func TestHeadObject(ts *TestState) {
 		ts.Run(HeadObject_ranged_with_checksum_mode)
 	}
 	ts.Run(HeadObject_success)
-	ts.Run(HeadObject_sse_c_invalid_headers)
 	ts.Run(HeadObject_overrides_success)
 	ts.Run(HeadObject_overrides_presign_success)
 	ts.Run(HeadObject_overrides_fail_public)
@@ -422,7 +417,6 @@ func TestCopyObject(ts *TestState) {
 	}
 	ts.Run(CopyObject_with_special_characters)
 	ts.Run(CopyObject_success)
-	ts.Run(CopyObject_sse_c_invalid_copy_source_headers)
 	ts.Run(CopyObject_cross_bucket_server_side_copy)
 	ts.Run(CopyObject_incorrect_source_bucket_expected_owner)
 }
@@ -548,17 +542,14 @@ func TestListMultipartUploads(ts *TestState) {
 	ts.Run(ListMultipartUploads_empty_result)
 	ts.Run(ListMultipartUploads_invalid_max_uploads)
 	ts.Run(ListMultipartUploads_max_uploads)
-	ts.Run(ListMultipartUploads_next_markers_not_truncated)
 	ts.Run(ListMultipartUploads_exceeding_max_uploads)
 	ts.Run(ListMultipartUploads_ignore_upload_id_marker)
 	ts.Run(ListMultipartUploads_invalid_uploadId_marker)
 	ts.Run(ListMultipartUploads_keyMarker_not_from_list)
-	ts.Run(ListMultipartUploads_upload_id_marker_with_key_marker)
 	ts.Run(ListMultipartUploads_delimiter_truncated)
 	ts.Run(ListMultipartUploads_prefix)
 	ts.Run(ListMultipartUploads_both_delimiter_and_prefix)
 	ts.Run(ListMultipartUploads_delimiter_no_matches)
-	ts.Run(ListMultipartUploads_with_upload_id)
 	//TODO: remove the condition after implementing checksums in azure
 	if !ts.conf.azureTests {
 		ts.Run(ListMultipartUploads_with_checksums)
@@ -912,28 +903,6 @@ func TestNotImplementedActions(ts *TestState) {
 	ts.Run(GetObjectAcl_not_implemented)
 }
 
-// TestSSECNotImplemented is for backends that reject SSE-C rather than forward it.
-func TestSSECNotImplemented(ts *TestState) {
-	ts.Run(PutObject_sse_c_not_implemented)
-	ts.Run(PostObject_sse_c_not_implemented)
-	ts.Run(GetObject_sse_c_not_implemented)
-	ts.Run(HeadObject_sse_c_not_implemented)
-	ts.Run(GetObjectAttributes_sse_c_not_implemented)
-	ts.Run(CopyObject_sse_c_not_implemented)
-	ts.Run(CreateMultipartUpload_sse_c_not_implemented)
-	ts.Run(UploadPart_sse_c_not_implemented)
-	ts.Run(UploadPartCopy_sse_c_not_implemented)
-	ts.Run(CompleteMultipartUpload_sse_c_not_implemented)
-}
-
-func TestSSECTransport(ts *TestState) {
-	if ts.conf.tlsStatus {
-		TestSSECNotImplemented(ts)
-		return
-	}
-	ts.Run(PutObject_sse_c_requires_tls)
-}
-
 func TestWORMProtection(ts *TestState) {
 	ts.Run(WORMProtection_bucket_object_lock_configuration_compliance_mode)
 	ts.Run(WORMProtection_bucket_object_lock_configuration_governance_mode)
@@ -1034,9 +1003,6 @@ func TestFullFlow(ts *TestState) {
 	if ts.conf.versioningEnabled {
 		TestVersioning(ts)
 	}
-	if ts.conf.azureTests {
-		TestSSECTransport(ts)
-	}
 	TestGatewayIAM(ts)
 	TestServer(ts)
 }
@@ -1067,7 +1033,6 @@ func TestPosix(ts *TestState) {
 	if !ts.conf.versioningEnabled {
 		TestVersioningDisabled(ts)
 	}
-	TestSSECTransport(ts)
 }
 
 func TestScoutfs(ts *TestState) {
@@ -1169,7 +1134,6 @@ func TestScoutfs(ts *TestState) {
 	ts.Run(ObjectTagging_trailing_slash_counterpart)
 	ts.Run(ObjectLock_trailing_slash_counterpart)
 	ts.Run(DeleteObject_directory_not_empty)
-	TestSSECTransport(ts)
 }
 
 func TestGatewayIAM(ts *TestState) {
@@ -2195,11 +2159,9 @@ func TestRouter(ts *TestState) {
 	ts.Run(RouterGetUploadsWithKey)
 	ts.Run(RouterCopySourceNotAllowed)
 	ts.Run(RouterListVersionsWithKey)
-	ts.Run(RouterUploadsWithUploadId)
 	ts.Run(RouterObjectAnnotationNotImplemented)
 	ts.Run(RouterRenameObjectNotImplemented)
 	ts.Run(RouterObjectTorrentMethodNotAllowed)
-	ts.Run(RouterInvalidURLEscape)
 }
 
 func TestPostObject(ts *TestState) {
@@ -2987,8 +2949,6 @@ func GetIntTests() IntTests {
 		"ListBuckets_as_admin":                                                             ListBuckets_as_admin,
 		"ListBuckets_with_prefix":                                                          ListBuckets_with_prefix,
 		"ListBuckets_invalid_max_buckets":                                                  ListBuckets_invalid_max_buckets,
-		"ListBuckets_invalid_bucket_region":                                                ListBuckets_invalid_bucket_region,
-		"ListBuckets_bucket_region":                                                        ListBuckets_bucket_region,
 		"ListBuckets_truncated":                                                            ListBuckets_truncated,
 		"ListBuckets_success":                                                              ListBuckets_success,
 		"ListBuckets_empty_success":                                                        ListBuckets_empty_success,
@@ -3290,16 +3250,13 @@ func GetIntTests() IntTests {
 		"ListMultipartUploads_empty_result":                                                ListMultipartUploads_empty_result,
 		"ListMultipartUploads_invalid_max_uploads":                                         ListMultipartUploads_invalid_max_uploads,
 		"ListMultipartUploads_max_uploads":                                                 ListMultipartUploads_max_uploads,
-		"ListMultipartUploads_next_markers_not_truncated":                                  ListMultipartUploads_next_markers_not_truncated,
 		"ListMultipartUploads_exceeding_max_uploads":                                       ListMultipartUploads_exceeding_max_uploads,
 		"ListMultipartUploads_ignore_upload_id_marker":                                     ListMultipartUploads_ignore_upload_id_marker,
 		"ListMultipartUploads_invalid_uploadId_marker":                                     ListMultipartUploads_invalid_uploadId_marker,
 		"ListMultipartUploads_keyMarker_not_from_list":                                     ListMultipartUploads_keyMarker_not_from_list,
-		"ListMultipartUploads_upload_id_marker_with_key_marker":                            ListMultipartUploads_upload_id_marker_with_key_marker,
 		"ListMultipartUploads_delimiter_truncated":                                         ListMultipartUploads_delimiter_truncated,
 		"ListMultipartUploads_prefix":                                                      ListMultipartUploads_prefix,
 		"ListMultipartUploads_both_delimiter_and_prefix":                                   ListMultipartUploads_both_delimiter_and_prefix,
-		"ListMultipartUploads_with_upload_id":                                              ListMultipartUploads_with_upload_id,
 		"ListMultipartUploads_with_checksums":                                              ListMultipartUploads_with_checksums,
 		"AbortMultipartUpload_non_existing_bucket":                                         AbortMultipartUpload_non_existing_bucket,
 		"AbortMultipartUpload_incorrect_uploadId":                                          AbortMultipartUpload_incorrect_uploadId,
@@ -3582,21 +3539,6 @@ func GetIntTests() IntTests {
 		"PutObject_with_slashes":                                                           PutObject_with_slashes,
 		"PutObject_race_with_delete":                                                       PutObject_race_with_delete,
 		"CreateMultipartUpload_dir_obj":                                                    CreateMultipartUpload_dir_obj,
-		"PutObject_sse_c_not_implemented":                                                  PutObject_sse_c_not_implemented,
-		"PutObject_unsupported_sse_not_implemented":                                        PutObject_unsupported_sse_not_implemented,
-		"PutObject_sse_c_requires_tls":                                                     PutObject_sse_c_requires_tls,
-		"PostObject_sse_c_not_implemented":                                                 PostObject_sse_c_not_implemented,
-		"GetObject_sse_c_not_implemented":                                                  GetObject_sse_c_not_implemented,
-		"HeadObject_sse_c_not_implemented":                                                 HeadObject_sse_c_not_implemented,
-		"GetObjectAttributes_sse_c_not_implemented":                                        GetObjectAttributes_sse_c_not_implemented,
-		"CopyObject_sse_c_not_implemented":                                                 CopyObject_sse_c_not_implemented,
-		"CreateMultipartUpload_sse_c_not_implemented":                                      CreateMultipartUpload_sse_c_not_implemented,
-		"UploadPart_sse_c_not_implemented":                                                 UploadPart_sse_c_not_implemented,
-		"UploadPartCopy_sse_c_not_implemented":                                             UploadPartCopy_sse_c_not_implemented,
-		"CompleteMultipartUpload_sse_c_not_implemented":                                    CompleteMultipartUpload_sse_c_not_implemented,
-		"PutObject_sse_c_invalid_headers":                                                  PutObject_sse_c_invalid_headers,
-		"HeadObject_sse_c_invalid_headers":                                                 HeadObject_sse_c_invalid_headers,
-		"CopyObject_sse_c_invalid_copy_source_headers":                                     CopyObject_sse_c_invalid_copy_source_headers,
 		"IAM_user_access_denied":                                                           IAM_user_access_denied,
 		"IAM_userplus_access_denied":                                                       IAM_userplus_access_denied,
 		"IAM_userplus_CreateBucket":                                                        IAM_userplus_CreateBucket,
@@ -3804,11 +3746,9 @@ func GetIntTests() IntTests {
 		"RouterGetUploadsWithKey":                                                          RouterGetUploadsWithKey,
 		"RouterCopySourceNotAllowed":                                                       RouterCopySourceNotAllowed,
 		"RouterListVersionsWithKey":                                                        RouterListVersionsWithKey,
-		"RouterUploadsWithUploadId":                                                        RouterUploadsWithUploadId,
 		"RouterObjectAnnotationNotImplemented":                                             RouterObjectAnnotationNotImplemented,
 		"RouterRenameObjectNotImplemented":                                                 RouterRenameObjectNotImplemented,
 		"RouterObjectTorrentMethodNotAllowed":                                              RouterObjectTorrentMethodNotAllowed,
-		"RouterInvalidURLEscape":                                                           RouterInvalidURLEscape,
 		"UnsignedStreaminPayloadTrailer_malformed_trailer":                                 UnsignedStreaminPayloadTrailer_malformed_trailer,
 		"UnsignedStreamingPayloadTrailer_aborted_connection":                               UnsignedStreamingPayloadTrailer_aborted_connection,
 		"UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length":               UnsignedStreamingPayloadTrailer_missing_invalid_dec_content_length,

tests/test_rest_list_multipart_uploads.sh

@@ -215,6 +215,9 @@ source ./tests/setup.sh
 
 # tags: curl,ListMultipartUploads,multipart,uploadId,invalid-query
 @test "REST - ListMultipartUploads - uploads/uploadId combo" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/2155"
+  fi
   run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
   assert_success
   read -r bucket_name test_file <<< "$output"