Upstream pterodactyl/wings state of the touched files before 19512e69fc

router/downloader/downloader.go

@@ -0,0 +1,364 @@
+package downloader
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"emperror.dev/errors"
+	"github.com/apex/log"
+	"github.com/google/uuid"
+
+	"github.com/pterodactyl/wings/config"
+	wnet "github.com/pterodactyl/wings/internal/network"
+	"github.com/pterodactyl/wings/server"
+)
+
+var client *http.Client
+
+func init() {
+	dialer := &net.Dialer{
+		LocalAddr: nil,
+		Timeout:   time.Second * 30,
+		// Check every address immediately before connecting to it, since the
+		// destination is resolved again when dialing.
+		ControlContext: checkConnection,
+	}
+
+	trnspt := http.DefaultTransport.(*http.Transport).Clone()
+	// Always connect directly, so that every destination is checked.
+	trnspt.Proxy = nil
+	// Store the file exactly as it is served. Asking for a compressed response
+	// would also hide the length of the file, which is needed to write it.
+	trnspt.DisableCompression = true
+	trnspt.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, _, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		// Refuse the destination if any of its addresses are internal, or if it
+		// cannot be resolved.
+		ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
+		if err != nil || len(ips) == 0 {
+			log.WithField("host", host).WithField("error", err).Debug("downloader: failed to resolve remote file host")
+			return nil, errors.WithStack(ErrInvalidIPAddress)
+		}
+		for _, ip := range ips {
+			if isBlocked(host, ip) {
+				return nil, errors.WithStack(ErrInternalResolution)
+			}
+		}
+		return dialer.DialContext(context.WithValue(ctx, dialHostKey{}, host), network, addr)
+	}
+
+	client = &http.Client{
+		Timeout:   time.Hour * 2,
+		Transport: trnspt,
+		// Disallow any redirect on an HTTP call. This is a security requirement: do not modify
+		// this logic without first ensuring that the new target location IS NOT within the current
+		// instance's local network.
+		//
+		// This specific error response just causes the client to not follow the redirect and
+		// returns the actual redirect response to the caller. Not perfect, but simple and most
+		// people won't be using URLs that redirect anyways hopefully?
+		//
+		// We'll re-evaluate this down the road if needed.
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+}
+
+// dialHostKey is the context key for the host name a connection is being made
+// to, which is checked against the allowlist when the connection is made.
+type dialHostKey struct{}
+
+// checkConnection is called with the address a connection is about to be made
+// to, and refuses to connect to any internal address.
+func checkConnection(ctx context.Context, _ string, address string, _ syscall.RawConn) error {
+	ipStr, _, err := net.SplitHostPort(address)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	ip, err := netip.ParseAddr(ipStr)
+	if err != nil {
+		return errors.WithStack(ErrInternalResolution)
+	}
+	host, _ := ctx.Value(dialHostKey{}).(string)
+	if isBlocked(host, ip) {
+		return errors.WithStack(ErrInternalResolution)
+	}
+	return nil
+}
+
+// isBlocked reports whether connecting to the address, resolved from the host,
+// is refused because it is internal and not allowed by the operator.
+func isBlocked(host string, ip netip.Addr) bool {
+	return wnet.IsInternal(ip) && !wnet.IsAllowed(config.Get().Api.RemoteDownloadAllowlist, host, ip)
+}
+
+var instance = &Downloader{
+	// Tracks all the active downloads.
+	downloadCache: make(map[string]*Download),
+	// Tracks all the downloads active for a given server instance. This is
+	// primarily used to make things quicker and keep the code a little more
+	// legible throughout here.
+	serverCache: make(map[string][]string),
+}
+
+const (
+	ErrInternalResolution = errors.Sentinel("downloader: destination resolves to internal network location")
+	ErrInvalidIPAddress   = errors.Sentinel("downloader: invalid IP address")
+	ErrDownloadFailed     = errors.Sentinel("downloader: download request failed")
+)
+
+type Counter struct {
+	total   int
+	onWrite func(total int)
+}
+
+func (c *Counter) Write(p []byte) (int, error) {
+	n := len(p)
+	c.total += n
+	c.onWrite(c.total)
+	return n, nil
+}
+
+type DownloadRequest struct {
+	Directory string
+	URL       *url.URL
+	FileName  string
+	UseHeader bool
+}
+
+type Download struct {
+	Identifier string
+	path       string
+	mu         sync.RWMutex
+	req        DownloadRequest
+	server     *server.Server
+	progress   float64
+	cancelFunc *context.CancelFunc
+}
+
+// New starts a new tracked download which allows for cancellation later on by calling
+// the Downloader.Cancel function.
+func New(s *server.Server, r DownloadRequest) *Download {
+	dl := Download{
+		Identifier: uuid.Must(uuid.NewRandom()).String(),
+		req:        r,
+		server:     s,
+	}
+	instance.track(&dl)
+	return &dl
+}
+
+// ByServer returns all the tracked downloads for a given server instance.
+func ByServer(sid string) []*Download {
+	instance.mu.Lock()
+	defer instance.mu.Unlock()
+	var downloads []*Download
+	if v, ok := instance.serverCache[sid]; ok {
+		for _, id := range v {
+			if dl, ok := instance.downloadCache[id]; ok {
+				downloads = append(downloads, dl)
+			}
+		}
+	}
+	return downloads
+}
+
+// ByID returns a single Download matching a given identifier. If no download is found
+// the second argument in the response will be false.
+func ByID(dlid string) *Download {
+	return instance.find(dlid)
+}
+
+func (dl *Download) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		Identifier string
+		Progress   float64
+	}{
+		Identifier: dl.Identifier,
+		Progress:   dl.Progress(),
+	})
+}
+
+// Execute executes a given download for the server and begins writing the file to the disk. Once
+// completed the download will be removed from the cache.
+func (dl *Download) Execute() error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour*12)
+	dl.cancelFunc = &cancel
+	defer dl.Cancel()
+
+	// At this point we have verified the destination is not within the local network, so we can
+	// now make a request to that URL and pull down the file, saving it to the server's data
+	// directory.
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, dl.req.URL.String(), nil)
+	if err != nil {
+		return errors.WrapIf(err, "downloader: failed to create request")
+	}
+
+	req.Header.Set("User-Agent", "Pterodactyl Panel (https://pterodactyl.io)")
+	res, err := client.Do(req)
+	if err != nil {
+		if IsDownloadError(err) {
+			return err
+		}
+		return errors.WithStack(fmt.Errorf("%w: %w", ErrDownloadFailed, err))
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return errors.New("downloader: got bad response status from endpoint: " + res.Status)
+	}
+
+	if res.ContentLength < 1 {
+		return errors.New("downloader: request is missing ContentLength")
+	}
+
+	if dl.req.UseHeader {
+		if contentDisposition := res.Header.Get("Content-Disposition"); contentDisposition != "" {
+			_, params, err := mime.ParseMediaType(contentDisposition)
+			if err != nil {
+				return errors.WrapIf(err, "downloader: invalid \"Content-Disposition\" header")
+			}
+
+			// Only use the name of the file, so that it is saved in the
+			// directory that was asked for.
+			if v, ok := params["filename"]; ok {
+				if name := filepath.Base(filepath.Clean("/" + v)); name != "/" && name != "." {
+					dl.path = name
+				}
+			}
+		}
+	}
+	if dl.path == "" {
+		if dl.req.FileName != "" {
+			dl.path = dl.req.FileName
+		} else {
+			parts := strings.Split(dl.req.URL.Path, "/")
+			dl.path = parts[len(parts)-1]
+		}
+	}
+
+	p := dl.Path()
+	if err := dl.server.Filesystem().IsIgnored(p); err != nil {
+		return err
+	}
+	dl.server.Log().WithField("path", p).Debug("writing remote file to disk")
+
+	// Write the file while tracking the progress, Write will check that the
+	// size of the file won't exceed the disk limit.
+	r := io.TeeReader(res.Body, dl.counter(res.ContentLength))
+	if err := dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644); err != nil {
+		return errors.WrapIf(err, "downloader: failed to write file to server directory")
+	}
+	return nil
+}
+
+// Cancel cancels a running download and frees up the associated resources. If a file is being
+// written a partial file will remain present on the disk.
+func (dl *Download) Cancel() {
+	if dl.cancelFunc != nil {
+		(*dl.cancelFunc)()
+	}
+	instance.remove(dl.Identifier)
+}
+
+// BelongsTo checks if the given download belongs to the provided server.
+func (dl *Download) BelongsTo(s *server.Server) bool {
+	return dl.server.ID() == s.ID()
+}
+
+// Progress returns the current progress of the download as a float value between 0 and 1 where
+// 1 indicates that the download is completed.
+func (dl *Download) Progress() float64 {
+	dl.mu.RLock()
+	defer dl.mu.RUnlock()
+	return dl.progress
+}
+
+func (dl *Download) Path() string {
+	return filepath.Join(dl.req.Directory, dl.path)
+}
+
+// Handles a write event by updating the progress completed percentage and firing off
+// events to the server websocket as needed.
+func (dl *Download) counter(contentLength int64) *Counter {
+	onWrite := func(t int) {
+		dl.mu.Lock()
+		defer dl.mu.Unlock()
+		dl.progress = float64(t) / float64(contentLength)
+	}
+	return &Counter{
+		onWrite: onWrite,
+	}
+}
+
+// Downloader represents a global downloader that keeps track of all currently processing downloads
+// for the machine.
+type Downloader struct {
+	mu            sync.RWMutex
+	downloadCache map[string]*Download
+	serverCache   map[string][]string
+}
+
+// track tracks a download in the internal cache for this instance.
+func (d *Downloader) track(dl *Download) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	sid := dl.server.ID()
+	if _, ok := d.downloadCache[dl.Identifier]; !ok {
+		d.downloadCache[dl.Identifier] = dl
+		if _, ok := d.serverCache[sid]; !ok {
+			d.serverCache[sid] = []string{}
+		}
+		d.serverCache[sid] = append(d.serverCache[sid], dl.Identifier)
+	}
+}
+
+// find finds a given download entry using the provided ID and returns it.
+func (d *Downloader) find(dlid string) *Download {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+	if entry, ok := d.downloadCache[dlid]; ok {
+		return entry
+	}
+	return nil
+}
+
+// remove removes the given download reference from the cache storing them. This also updates
+// the slice of active downloads for a given server to not include this download.
+func (d *Downloader) remove(dlID string) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if _, ok := d.downloadCache[dlID]; !ok {
+		return
+	}
+	sID := d.downloadCache[dlID].server.ID()
+	delete(d.downloadCache, dlID)
+	if tracked, ok := d.serverCache[sID]; ok {
+		var out []string
+		for _, k := range tracked {
+			if k != dlID {
+				out = append(out, k)
+			}
+		}
+		d.serverCache[sID] = out
+	}
+}
+
+func IsDownloadError(err error) bool {
+	return errors.Is(err, ErrDownloadFailed) || errors.Is(err, ErrInvalidIPAddress) || errors.Is(err, ErrInternalResolution)
+}

router/downloader/downloader_pull_test.go

@@ -0,0 +1,129 @@
+package downloader
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/pterodactyl/wings/config"
+	"github.com/pterodactyl/wings/remote"
+	"github.com/pterodactyl/wings/server"
+)
+
+const pullTestServerUuid = "ffffffff-ffff-4fff-8fff-ffffffffffff"
+
+// newPullTestServer returns a server whose files are stored in a temporary data
+// directory. Loopback is allowed so that the tests can use a local origin.
+func newPullTestServer(t *testing.T) *server.Server {
+	t.Helper()
+	previous := config.Get()
+	t.Cleanup(func() { config.Set(previous) })
+	next := *previous
+	next.System.Data = t.TempDir()
+	next.Api.RemoteDownloadAllowlist = []string{"127.0.0.1"}
+	config.Set(&next)
+
+	settings, err := json.Marshal(map[string]string{"uuid": pullTestServerUuid})
+	require.NoError(t, err)
+	s, err := server.NewEmptyManager(nil).InitServer(pullTestServerUuid, remote.ServerConfigurationResponse{
+		Settings:             settings,
+		ProcessConfiguration: &remote.ProcessConfiguration{},
+	})
+	require.NoError(t, err)
+	require.NoError(t, os.MkdirAll(s.Filesystem().Path(), 0o755))
+	return s
+}
+
+// A file from a server that compresses responses for clients that accept gzip
+// can be pulled.
+func TestDownloadAcceptsGzipEncodedResponses(t *testing.T) {
+	s := newPullTestServer(t)
+
+	payload := []byte(strings.Repeat("pull test payload\n", 4096))
+	var compressed bytes.Buffer
+	zw := gzip.NewWriter(&compressed)
+	_, err := zw.Write(payload)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body := payload
+		if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+			w.Header().Set("Content-Encoding", "gzip")
+			body = compressed.Bytes()
+		}
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.Header().Set("Content-Length", strconv.Itoa(len(body)))
+		_, _ = w.Write(body)
+	}))
+	t.Cleanup(origin.Close)
+
+	u, err := url.Parse(origin.URL + "/payload.bin")
+	require.NoError(t, err)
+	require.NoError(t, New(s, DownloadRequest{Directory: "/", URL: u, FileName: "payload.bin"}).Execute())
+
+	got, err := os.ReadFile(filepath.Join(s.Filesystem().Path(), "payload.bin"))
+	require.NoError(t, err)
+	require.Equal(t, payload, got)
+}
+
+// Only the name of a file from Content-Disposition is used, so the file is saved
+// in the directory that was asked for.
+func TestDownloadFilenameFromHeaderIsSavedInRequestedDirectory(t *testing.T) {
+	tests := []struct {
+		directory string
+		want      string
+	}{
+		{directory: "/chosen/dir", want: "chosen/dir/named.txt"},
+		{directory: "chosen/dir", want: "chosen/dir/named.txt"},
+		{directory: "", want: "named.txt"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.directory, func(t *testing.T) {
+			s := newPullTestServer(t)
+
+			origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Disposition", `attachment; filename="../../named.txt"`)
+				w.Header().Set("Content-Length", "5")
+				_, _ = w.Write([]byte("hello"))
+			}))
+			t.Cleanup(origin.Close)
+
+			u, err := url.Parse(origin.URL + "/file")
+			require.NoError(t, err)
+			require.NoError(t, New(s, DownloadRequest{Directory: tt.directory, URL: u, FileName: "file", UseHeader: true}).Execute())
+
+			root := s.Filesystem().Path()
+			require.FileExists(t, filepath.Join(root, tt.want))
+			require.NoFileExists(t, filepath.Join(filepath.Dir(root), "named.txt"))
+		})
+	}
+}
+
+// A download that fails to connect is reported as a failed download.
+func TestDownloadConnectionFailureIsDownloadError(t *testing.T) {
+	s := newPullTestServer(t)
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	addr := l.Addr().String()
+	require.NoError(t, l.Close())
+
+	u, err := url.Parse("http://" + addr + "/file")
+	require.NoError(t, err)
+	err = New(s, DownloadRequest{Directory: "/", URL: u, FileName: "file"}).Execute()
+	require.Error(t, err)
+	require.True(t, IsDownloadError(err), "got %v", err)
+	require.ErrorIs(t, err, ErrDownloadFailed)
+}