Files
versity-versitygw/tests/integration/general.go
T
dualfroz e0fef79b7b fix: return InvalidURI for a malformed escape in the request path
A request path with an invalid percent escape, such as "/bucket/%zz",
failed url.PathUnescape in the DecodeURL middleware and the raw error
was answered with 500 InternalError. S3 rejects such a request with
400 InvalidURI.

Return InvalidURI from DecodeURL, so the s3 and admin routers answer
with it as well, and let the website handler pass the error through
instead of mapping it on its own.

Fixes #2070
2026-10-03 15:13:16 +02:00

600 lines
18 KiB
Go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package integration
import (
"context"
"fmt"
"net/http"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
)
var (
shortTimeout = 30 * time.Second
longTimeout = 60 * time.Second
iso8601Format = "20060102T150405Z"
timefmt = "Mon, 02 Jan 2006 15:04:05 GMT"
nullVersionId = "null"
)
// router tests
func RouterPutPartNumberWithoutUploadId(s *S3Conf) error {
testName := "RouterPutPartNumberWithoutUploadId"
return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error {
req, err := http.NewRequest(http.MethodPut, s.endpoint+"/bucket/object", nil)
if err != nil {
return err
}
query := req.URL.Query()
query.Add("partNumber", "1")
req.URL.RawQuery = query.Encode()
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
if err := checkHTTPResponseApiErr(resp, s3err.GetInvalidArgumentErr(s3err.InvalidArgMissingUploadId, "partNumber")); err != nil {
return err
}
return nil
})
}
func RouterPostRoot(s *S3Conf) error {
testName := "RouterPostRoot"
return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error {
req, err := http.NewRequest(http.MethodPost, s.endpoint+"/", nil)
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
if err := checkHTTPResponseApiErr(resp, s3err.GetMethodNotAllowedErr(http.MethodPost, s3err.ResourceTypeService, nil)); err != nil {
return err
}
return nil
})
}
func RouterPostObjectWithoutQuery(s *S3Conf) error {
testName := "RouterPostObjectWithoutQuery"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
req, err := http.NewRequest(http.MethodPost, s.endpoint+"/bucket/object", nil)
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
if err := checkHTTPResponseApiErr(resp, s3err.GetMethodNotAllowedErr(http.MethodPost, s3err.ResourceTypeService, nil)); err != nil {
return err
}
return nil
})
}
func RouterPUTObjectOnlyUploadId(s *S3Conf) error {
testName := "RouterPUTObjectOnlyUploadId"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
req, err := http.NewRequest(http.MethodPut, s.endpoint+"/bucket/object", nil)
if err != nil {
return err
}
query := req.URL.Query()
query.Add("uploadId", "my-upload-id")
req.URL.RawQuery = query.Encode()
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
if err := checkHTTPResponseApiErr(resp, s3err.GetMethodNotAllowedErr(
http.MethodPut,
s3err.ResourceTypeUpload,
[]string{http.MethodDelete, http.MethodPost, http.MethodGet},
)); err != nil {
return err
}
return nil
})
}
func RouterGetUploadsWithKey(s *S3Conf) error {
testName := "RouterGetUploadsWithKey"
return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error {
req, err := http.NewRequest(http.MethodGet, s.endpoint+"/bucket/object?uploads", nil)
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
return checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrGetUploadsWithKey))
})
}
func RouterCopySourceNotAllowed(s *S3Conf) error {
testName := "RouterCopySourceNotAllowed"
return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error {
for _, method := range []string{
http.MethodPost,
http.MethodDelete,
http.MethodGet,
http.MethodHead,
} {
for _, path := range []string{
"/bucket",
"/bucket/object",
} {
if method == http.MethodPost {
// the error for POST request occurs only when uploadId is there
path += "?uploadId=something"
}
req, err := http.NewRequest(method, s.endpoint+path, nil)
if err != nil {
return fmt.Errorf("failed to make %s request to %s", method, path)
}
req.Header.Add("x-amz-copy-source", "bucket/object")
resp, err := s.httpClient.Do(req)
if err != nil {
return fmt.Errorf("failed to send %s request to %s", method, path)
}
if method == http.MethodHead {
// for head requests only check the status code
if resp.StatusCode != http.StatusBadRequest {
return fmt.Errorf("expected 400 status code for HEAD %s request, instead got %v", path, resp.StatusCode)
}
} else {
if err := checkHTTPResponseApiErr(resp, s3err.GetInvalidArgumentErr(s3err.InvalidArgCopySource, "bucket/object")); err != nil {
return fmt.Errorf("%s %s: %w", method, path, err)
}
}
}
}
return nil
})
}
func RouterListVersionsWithKey(s *S3Conf) error {
testName := "RouterListVersionsWithKey"
return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error {
req, err := http.NewRequest(http.MethodGet, s.endpoint+"/bucket/object?versions", nil)
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
return checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrVersionsWithKey))
})
}
func RouterObjectAnnotationNotImplemented(s *S3Conf) error {
testName := "RouterObjectAnnotationNotImplemented"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj, name := "my-obj", "label"
_, err := putObjectWithData(10, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.PutObjectAnnotation(ctx, &s3.PutObjectAnnotationInput{
Bucket: &bucket,
Key: &obj,
AnnotationName: &name,
AnnotationPayload: strings.NewReader(`{"label":"x"}`),
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNotImplemented)); err != nil {
return fmt.Errorf("PutObjectAnnotation: %w", err)
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.GetObjectAnnotation(ctx, &s3.GetObjectAnnotationInput{
Bucket: &bucket,
Key: &obj,
AnnotationName: &name,
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNotImplemented)); err != nil {
return fmt.Errorf("GetObjectAnnotation: %w", err)
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.ListObjectAnnotations(ctx, &s3.ListObjectAnnotationsInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNotImplemented)); err != nil {
return fmt.Errorf("ListObjectAnnotations: %w", err)
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.DeleteObjectAnnotation(ctx, &s3.DeleteObjectAnnotationInput{
Bucket: &bucket,
Key: &obj,
AnnotationName: &name,
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNotImplemented)); err != nil {
return fmt.Errorf("DeleteObjectAnnotation: %w", err)
}
return nil
})
}
func RouterRenameObjectNotImplemented(s *S3Conf) error {
testName := "RouterRenameObjectNotImplemented"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
src, dst := "src-obj", "dst-obj"
_, err := putObjectWithData(10, &s3.PutObjectInput{
Bucket: &bucket,
Key: &src,
}, s3client)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.RenameObject(ctx, &s3.RenameObjectInput{
Bucket: &bucket,
Key: &dst,
RenameSource: &src,
})
cancel()
return checkApiErr(err, s3err.GetAPIError(s3err.ErrNotImplemented))
})
}
func RouterObjectTorrentMethodNotAllowed(s *S3Conf) error {
testName := "RouterObjectTorrentMethodNotAllowed"
return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error {
for _, method := range []string{
http.MethodGet,
http.MethodHead,
http.MethodPut,
http.MethodDelete,
http.MethodPost,
} {
req, err := http.NewRequest(method, s.endpoint+"/bucket/object?torrent", nil)
if err != nil {
return fmt.Errorf("failed to make %s request: %w", method, err)
}
resp, err := s.httpClient.Do(req)
if err != nil {
return fmt.Errorf("failed to send %s request: %w", method, err)
}
var allowed []string
if method != http.MethodGet {
allowed = []string{http.MethodGet}
}
expected := s3err.GetMethodNotAllowedErr(method, s3err.ResourceTypeTorrent, allowed)
if got, want := resp.Header.Get("Allow"), expected.AllowedMethodsString(); got != want {
return fmt.Errorf("%s: expected Allow header %q, instead got %q", method, want, got)
}
if method == http.MethodHead {
// for head requests only check the status code
if resp.StatusCode != http.StatusMethodNotAllowed {
return fmt.Errorf("expected 405 status code for HEAD request, instead got %v", resp.StatusCode)
}
continue
}
if err := checkHTTPResponseApiErr(resp, expected); err != nil {
return fmt.Errorf("%s: %w", method, err)
}
}
return nil
})
}
func RouterInvalidURLEscape(s *S3Conf) error {
testName := "RouterInvalidURLEscape"
return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error {
req, err := http.NewRequest(http.MethodGet, s.endpoint+"/bucket/object", nil)
if err != nil {
return err
}
// http.NewRequest refuses a malformed escape, so the raw path
// is set on the url directly
req.URL.Opaque = "/bucket/(*&^%&())"
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
return checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidURI))
})
}
// CORS middleware tests
func CORSMiddleware_invalid_method(s *S3Conf) error {
testName := "CORSMiddleware_invalid_method"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
err := putBucketCors(s3client, &s3.PutBucketCorsInput{
Bucket: &bucket,
CORSConfiguration: &types.CORSConfiguration{
CORSRules: []types.CORSRule{
{
AllowedOrigins: []string{"http://www.example.com"},
AllowedMethods: []string{http.MethodPut},
},
},
},
})
if err != nil {
return err
}
// create a PutObject signed request
req, err := createSignedReq(http.MethodPut, s.endpoint, bucket+"/my-obj", s.awsID, s.awsSecret, "s3", s.awsRegion, "", nil, time.Now(), map[string]string{
"Origin": "http://www.example.com",
"Access-Control-Request-Method": "invalid_method",
})
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
result, err := extractCORSHeaders(resp)
if err != nil {
return err
}
return checkApiErr(result.err, s3err.GetInvalidCORSMethodErr("invalid_method"))
})
}
func CORSMiddleware_invalid_headers(s *S3Conf) error {
testName := "CORSMiddleware_invalid_headers"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
err := putBucketCors(s3client, &s3.PutBucketCorsInput{
Bucket: &bucket,
CORSConfiguration: &types.CORSConfiguration{
CORSRules: []types.CORSRule{
{
AllowedOrigins: []string{"http://www.example.com"},
AllowedMethods: []string{http.MethodPut},
},
},
},
})
if err != nil {
return err
}
// create a PutObject signed request
req, err := createSignedReq(http.MethodPut, s.endpoint, bucket+"/my-obj", s.awsID, s.awsSecret, "s3", s.awsRegion, "", nil, time.Now(), map[string]string{
"Origin": "http://www.example.com",
"Access-Control-Request-Headers": "invalid header",
})
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
result, err := extractCORSHeaders(resp)
if err != nil {
return err
}
return checkApiErr(result.err, s3err.GetInvalidCORSRequestHeaderErr("invalid header"))
})
}
func CORSMiddleware_access_forbidden(s *S3Conf) error {
testName := "CORSMiddleware_access_forbidden"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
err := putBucketCors(s3client, &s3.PutBucketCorsInput{
Bucket: &bucket,
CORSConfiguration: &types.CORSConfiguration{
CORSRules: []types.CORSRule{
{
AllowedOrigins: []string{"http://example.com", "https://example.com"},
AllowedMethods: []string{http.MethodGet},
AllowedHeaders: []string{"X-Amz-Date", "X-Amz-Content-Sha256"},
},
{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{http.MethodHead},
},
},
},
})
if err != nil {
return err
}
for _, test := range []struct {
origin string
method string
headers string
}{
// origin deson't match
{"http://non-matching-origin.net", http.MethodGet, "X-Amz-Date"},
// method doesn't match
{"http://example.com", http.MethodPut, "X-Amz-Content-Sha256"},
// header doesn't match
{"http://example.com", http.MethodGet, "X-Amz-Expected-Bucket-Owner"},
// extra header
{"http://example.com", http.MethodGet, "X-Amz-Date,X-Amz-Content-Sha256,Extra-Header"},
// extra header (2nd rule)
{"https://any-origin.com", http.MethodHead, "X-Amz-Extra-Header"},
// origin match, method not (2nd rule)
{"https://any-origin.com", http.MethodPost, ""},
} {
req, err := createSignedReq(http.MethodPut, s.endpoint, bucket+"/my-obj", s.awsID, s.awsSecret, "s3", s.awsRegion, "", nil, time.Now(), map[string]string{
"Origin": test.origin,
"Access-Control-Request-Headers": test.headers,
"Access-Control-Request-Method": test.method,
})
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
res, err := extractCORSHeaders(resp)
if err != nil {
return err
}
// no error expected, all the headers should be empty
if err := comparePreflightResult(&PreflightResult{}, res); err != nil {
return err
}
}
return nil
})
}
func CORSMiddleware_access_granted(s *S3Conf) error {
testName := "CORSMiddleware_access_granted"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
err := putBucketCors(s3client, &s3.PutBucketCorsInput{
Bucket: &bucket,
CORSConfiguration: &types.CORSConfiguration{
CORSRules: []types.CORSRule{
{
AllowedOrigins: []string{"http://example.com", "https://example.com"},
AllowedMethods: []string{http.MethodGet, http.MethodHead},
AllowedHeaders: []string{"X-Amz-Date", "X-Amz-Content-Sha256"},
ExposeHeaders: []string{"Content-Type", "Content-Length"},
MaxAgeSeconds: getPtr(int32(100)),
},
{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{http.MethodHead},
AllowedHeaders: []string{"X-Amz-Meta-Something"},
},
{
AllowedOrigins: []string{"something.net"},
AllowedMethods: []string{http.MethodPost, http.MethodPut},
AllowedHeaders: []string{"Authorization"},
ExposeHeaders: []string{"Content-Disposition", "Content-Encoding"},
MaxAgeSeconds: getPtr(int32(3000)),
ID: getPtr("unique_id"),
},
},
},
})
if err != nil {
return err
}
varyHdr := "Origin, Access-Control-Request-Headers, Access-Control-Request-Method"
for _, test := range []struct {
origin string
method string
headers string
result PreflightResult
}{
// first rule matches
{"http://example.com", http.MethodGet, "X-Amz-Date", PreflightResult{"http://example.com", "GET, HEAD", "x-amz-date, x-amz-content-sha256", "Content-Type, Content-Length", "100", "true", varyHdr, nil}},
{"http://example.com", http.MethodGet, "X-Amz-Content-Sha256", PreflightResult{"http://example.com", "GET, HEAD", "x-amz-date, x-amz-content-sha256", "Content-Type, Content-Length", "100", "true", varyHdr, nil}},
{"http://example.com", http.MethodHead, "", PreflightResult{"http://example.com", "GET, HEAD", "", "Content-Type, Content-Length", "100", "true", varyHdr, nil}},
{"https://example.com", http.MethodGet, "X-Amz-Date,X-Amz-Content-Sha256", PreflightResult{"https://example.com", "GET, HEAD", "x-amz-date, x-amz-content-sha256", "Content-Type, Content-Length", "100", "true", varyHdr, nil}},
// second rule matches
{"http://anything.com", http.MethodHead, "X-Amz-Meta-Something", PreflightResult{"*", "HEAD", "x-amz-meta-something", "", "", "false", varyHdr, nil}},
{"hello.com", http.MethodHead, "", PreflightResult{"*", "HEAD", "", "", "", "false", varyHdr, nil}},
// third rule matches
{"something.net", http.MethodPut, "Authorization", PreflightResult{"something.net", "POST, PUT", "authorization", "Content-Disposition, Content-Encoding", "3000", "true", varyHdr, nil}},
{"something.net", http.MethodPost, "", PreflightResult{"something.net", "POST, PUT", "", "Content-Disposition, Content-Encoding", "3000", "true", varyHdr, nil}},
} {
req, err := createSignedReq(http.MethodPut, s.endpoint, bucket+"/my-obj", s.awsID, s.awsSecret, "s3", s.awsRegion, "", nil, time.Now(), map[string]string{
"Origin": test.origin,
"Access-Control-Request-Headers": test.headers,
"Access-Control-Request-Method": test.method,
})
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
res, err := extractCORSHeaders(resp)
if err != nil {
return err
}
if err := comparePreflightResult(&test.result, res); err != nil {
return err
}
}
return nil
})
}