From fbbf479795f9b0f3431a82ebdda24729e9f9fc94 Mon Sep 17 00:00:00 2001 From: upstream Date: Tue, 13 Oct 2026 22:50:57 +0200 Subject: [PATCH] Upstream pterodactyl/wings state of the touched files before 19512e69fc --- router/downloader/downloader.go | 364 ++++++++++++++++++++++ router/downloader/downloader_pull_test.go | 129 ++++++++ 2 files changed, 493 insertions(+) create mode 100644 router/downloader/downloader.go create mode 100644 router/downloader/downloader_pull_test.go diff --git a/router/downloader/downloader.go b/router/downloader/downloader.go new file mode 100644 index 0000000..1662625 --- /dev/null +++ b/router/downloader/downloader.go @@ -0,0 +1,364 @@ +package downloader + +import ( + "context" + "encoding/json" + "fmt" + "io" + "mime" + "net" + "net/http" + "net/netip" + "net/url" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "emperror.dev/errors" + "github.com/apex/log" + "github.com/google/uuid" + + "github.com/pterodactyl/wings/config" + wnet "github.com/pterodactyl/wings/internal/network" + "github.com/pterodactyl/wings/server" +) + +var client *http.Client + +func init() { + dialer := &net.Dialer{ + LocalAddr: nil, + Timeout: time.Second * 30, + // Check every address immediately before connecting to it, since the + // destination is resolved again when dialing. + ControlContext: checkConnection, + } + + trnspt := http.DefaultTransport.(*http.Transport).Clone() + // Always connect directly, so that every destination is checked. + trnspt.Proxy = nil + // Store the file exactly as it is served. Asking for a compressed response + // would also hide the length of the file, which is needed to write it. + trnspt.DisableCompression = true + trnspt.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, errors.WithStack(err) + } + // Refuse the destination if any of its addresses are internal, or if it + // cannot be resolved. + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) + if err != nil || len(ips) == 0 { + log.WithField("host", host).WithField("error", err).Debug("downloader: failed to resolve remote file host") + return nil, errors.WithStack(ErrInvalidIPAddress) + } + for _, ip := range ips { + if isBlocked(host, ip) { + return nil, errors.WithStack(ErrInternalResolution) + } + } + return dialer.DialContext(context.WithValue(ctx, dialHostKey{}, host), network, addr) + } + + client = &http.Client{ + Timeout: time.Hour * 2, + Transport: trnspt, + // Disallow any redirect on an HTTP call. This is a security requirement: do not modify + // this logic without first ensuring that the new target location IS NOT within the current + // instance's local network. + // + // This specific error response just causes the client to not follow the redirect and + // returns the actual redirect response to the caller. Not perfect, but simple and most + // people won't be using URLs that redirect anyways hopefully? + // + // We'll re-evaluate this down the road if needed. + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +// dialHostKey is the context key for the host name a connection is being made +// to, which is checked against the allowlist when the connection is made. +type dialHostKey struct{} + +// checkConnection is called with the address a connection is about to be made +// to, and refuses to connect to any internal address. +func checkConnection(ctx context.Context, _ string, address string, _ syscall.RawConn) error { + ipStr, _, err := net.SplitHostPort(address) + if err != nil { + return errors.WithStack(err) + } + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return errors.WithStack(ErrInternalResolution) + } + host, _ := ctx.Value(dialHostKey{}).(string) + if isBlocked(host, ip) { + return errors.WithStack(ErrInternalResolution) + } + return nil +} + +// isBlocked reports whether connecting to the address, resolved from the host, +// is refused because it is internal and not allowed by the operator. +func isBlocked(host string, ip netip.Addr) bool { + return wnet.IsInternal(ip) && !wnet.IsAllowed(config.Get().Api.RemoteDownloadAllowlist, host, ip) +} + +var instance = &Downloader{ + // Tracks all the active downloads. + downloadCache: make(map[string]*Download), + // Tracks all the downloads active for a given server instance. This is + // primarily used to make things quicker and keep the code a little more + // legible throughout here. + serverCache: make(map[string][]string), +} + +const ( + ErrInternalResolution = errors.Sentinel("downloader: destination resolves to internal network location") + ErrInvalidIPAddress = errors.Sentinel("downloader: invalid IP address") + ErrDownloadFailed = errors.Sentinel("downloader: download request failed") +) + +type Counter struct { + total int + onWrite func(total int) +} + +func (c *Counter) Write(p []byte) (int, error) { + n := len(p) + c.total += n + c.onWrite(c.total) + return n, nil +} + +type DownloadRequest struct { + Directory string + URL *url.URL + FileName string + UseHeader bool +} + +type Download struct { + Identifier string + path string + mu sync.RWMutex + req DownloadRequest + server *server.Server + progress float64 + cancelFunc *context.CancelFunc +} + +// New starts a new tracked download which allows for cancellation later on by calling +// the Downloader.Cancel function. +func New(s *server.Server, r DownloadRequest) *Download { + dl := Download{ + Identifier: uuid.Must(uuid.NewRandom()).String(), + req: r, + server: s, + } + instance.track(&dl) + return &dl +} + +// ByServer returns all the tracked downloads for a given server instance. +func ByServer(sid string) []*Download { + instance.mu.Lock() + defer instance.mu.Unlock() + var downloads []*Download + if v, ok := instance.serverCache[sid]; ok { + for _, id := range v { + if dl, ok := instance.downloadCache[id]; ok { + downloads = append(downloads, dl) + } + } + } + return downloads +} + +// ByID returns a single Download matching a given identifier. If no download is found +// the second argument in the response will be false. +func ByID(dlid string) *Download { + return instance.find(dlid) +} + +func (dl *Download) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Identifier string + Progress float64 + }{ + Identifier: dl.Identifier, + Progress: dl.Progress(), + }) +} + +// Execute executes a given download for the server and begins writing the file to the disk. Once +// completed the download will be removed from the cache. +func (dl *Download) Execute() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Hour*12) + dl.cancelFunc = &cancel + defer dl.Cancel() + + // At this point we have verified the destination is not within the local network, so we can + // now make a request to that URL and pull down the file, saving it to the server's data + // directory. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, dl.req.URL.String(), nil) + if err != nil { + return errors.WrapIf(err, "downloader: failed to create request") + } + + req.Header.Set("User-Agent", "Pterodactyl Panel (https://pterodactyl.io)") + res, err := client.Do(req) + if err != nil { + if IsDownloadError(err) { + return err + } + return errors.WithStack(fmt.Errorf("%w: %w", ErrDownloadFailed, err)) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return errors.New("downloader: got bad response status from endpoint: " + res.Status) + } + + if res.ContentLength < 1 { + return errors.New("downloader: request is missing ContentLength") + } + + if dl.req.UseHeader { + if contentDisposition := res.Header.Get("Content-Disposition"); contentDisposition != "" { + _, params, err := mime.ParseMediaType(contentDisposition) + if err != nil { + return errors.WrapIf(err, "downloader: invalid \"Content-Disposition\" header") + } + + // Only use the name of the file, so that it is saved in the + // directory that was asked for. + if v, ok := params["filename"]; ok { + if name := filepath.Base(filepath.Clean("/" + v)); name != "/" && name != "." { + dl.path = name + } + } + } + } + if dl.path == "" { + if dl.req.FileName != "" { + dl.path = dl.req.FileName + } else { + parts := strings.Split(dl.req.URL.Path, "/") + dl.path = parts[len(parts)-1] + } + } + + p := dl.Path() + if err := dl.server.Filesystem().IsIgnored(p); err != nil { + return err + } + dl.server.Log().WithField("path", p).Debug("writing remote file to disk") + + // Write the file while tracking the progress, Write will check that the + // size of the file won't exceed the disk limit. + r := io.TeeReader(res.Body, dl.counter(res.ContentLength)) + if err := dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644); err != nil { + return errors.WrapIf(err, "downloader: failed to write file to server directory") + } + return nil +} + +// Cancel cancels a running download and frees up the associated resources. If a file is being +// written a partial file will remain present on the disk. +func (dl *Download) Cancel() { + if dl.cancelFunc != nil { + (*dl.cancelFunc)() + } + instance.remove(dl.Identifier) +} + +// BelongsTo checks if the given download belongs to the provided server. +func (dl *Download) BelongsTo(s *server.Server) bool { + return dl.server.ID() == s.ID() +} + +// Progress returns the current progress of the download as a float value between 0 and 1 where +// 1 indicates that the download is completed. +func (dl *Download) Progress() float64 { + dl.mu.RLock() + defer dl.mu.RUnlock() + return dl.progress +} + +func (dl *Download) Path() string { + return filepath.Join(dl.req.Directory, dl.path) +} + +// Handles a write event by updating the progress completed percentage and firing off +// events to the server websocket as needed. +func (dl *Download) counter(contentLength int64) *Counter { + onWrite := func(t int) { + dl.mu.Lock() + defer dl.mu.Unlock() + dl.progress = float64(t) / float64(contentLength) + } + return &Counter{ + onWrite: onWrite, + } +} + +// Downloader represents a global downloader that keeps track of all currently processing downloads +// for the machine. +type Downloader struct { + mu sync.RWMutex + downloadCache map[string]*Download + serverCache map[string][]string +} + +// track tracks a download in the internal cache for this instance. +func (d *Downloader) track(dl *Download) { + d.mu.Lock() + defer d.mu.Unlock() + sid := dl.server.ID() + if _, ok := d.downloadCache[dl.Identifier]; !ok { + d.downloadCache[dl.Identifier] = dl + if _, ok := d.serverCache[sid]; !ok { + d.serverCache[sid] = []string{} + } + d.serverCache[sid] = append(d.serverCache[sid], dl.Identifier) + } +} + +// find finds a given download entry using the provided ID and returns it. +func (d *Downloader) find(dlid string) *Download { + d.mu.RLock() + defer d.mu.RUnlock() + if entry, ok := d.downloadCache[dlid]; ok { + return entry + } + return nil +} + +// remove removes the given download reference from the cache storing them. This also updates +// the slice of active downloads for a given server to not include this download. +func (d *Downloader) remove(dlID string) { + d.mu.Lock() + defer d.mu.Unlock() + if _, ok := d.downloadCache[dlID]; !ok { + return + } + sID := d.downloadCache[dlID].server.ID() + delete(d.downloadCache, dlID) + if tracked, ok := d.serverCache[sID]; ok { + var out []string + for _, k := range tracked { + if k != dlID { + out = append(out, k) + } + } + d.serverCache[sID] = out + } +} + +func IsDownloadError(err error) bool { + return errors.Is(err, ErrDownloadFailed) || errors.Is(err, ErrInvalidIPAddress) || errors.Is(err, ErrInternalResolution) +} diff --git a/router/downloader/downloader_pull_test.go b/router/downloader/downloader_pull_test.go new file mode 100644 index 0000000..5de7296 --- /dev/null +++ b/router/downloader/downloader_pull_test.go @@ -0,0 +1,129 @@ +package downloader + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/remote" + "github.com/pterodactyl/wings/server" +) + +const pullTestServerUuid = "ffffffff-ffff-4fff-8fff-ffffffffffff" + +// newPullTestServer returns a server whose files are stored in a temporary data +// directory. Loopback is allowed so that the tests can use a local origin. +func newPullTestServer(t *testing.T) *server.Server { + t.Helper() + previous := config.Get() + t.Cleanup(func() { config.Set(previous) }) + next := *previous + next.System.Data = t.TempDir() + next.Api.RemoteDownloadAllowlist = []string{"127.0.0.1"} + config.Set(&next) + + settings, err := json.Marshal(map[string]string{"uuid": pullTestServerUuid}) + require.NoError(t, err) + s, err := server.NewEmptyManager(nil).InitServer(pullTestServerUuid, remote.ServerConfigurationResponse{ + Settings: settings, + ProcessConfiguration: &remote.ProcessConfiguration{}, + }) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(s.Filesystem().Path(), 0o755)) + return s +} + +// A file from a server that compresses responses for clients that accept gzip +// can be pulled. +func TestDownloadAcceptsGzipEncodedResponses(t *testing.T) { + s := newPullTestServer(t) + + payload := []byte(strings.Repeat("pull test payload\n", 4096)) + var compressed bytes.Buffer + zw := gzip.NewWriter(&compressed) + _, err := zw.Write(payload) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := payload + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + w.Header().Set("Content-Encoding", "gzip") + body = compressed.Bytes() + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + _, _ = w.Write(body) + })) + t.Cleanup(origin.Close) + + u, err := url.Parse(origin.URL + "/payload.bin") + require.NoError(t, err) + require.NoError(t, New(s, DownloadRequest{Directory: "/", URL: u, FileName: "payload.bin"}).Execute()) + + got, err := os.ReadFile(filepath.Join(s.Filesystem().Path(), "payload.bin")) + require.NoError(t, err) + require.Equal(t, payload, got) +} + +// Only the name of a file from Content-Disposition is used, so the file is saved +// in the directory that was asked for. +func TestDownloadFilenameFromHeaderIsSavedInRequestedDirectory(t *testing.T) { + tests := []struct { + directory string + want string + }{ + {directory: "/chosen/dir", want: "chosen/dir/named.txt"}, + {directory: "chosen/dir", want: "chosen/dir/named.txt"}, + {directory: "", want: "named.txt"}, + } + for _, tt := range tests { + t.Run(tt.directory, func(t *testing.T) { + s := newPullTestServer(t) + + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Disposition", `attachment; filename="../../named.txt"`) + w.Header().Set("Content-Length", "5") + _, _ = w.Write([]byte("hello")) + })) + t.Cleanup(origin.Close) + + u, err := url.Parse(origin.URL + "/file") + require.NoError(t, err) + require.NoError(t, New(s, DownloadRequest{Directory: tt.directory, URL: u, FileName: "file", UseHeader: true}).Execute()) + + root := s.Filesystem().Path() + require.FileExists(t, filepath.Join(root, tt.want)) + require.NoFileExists(t, filepath.Join(filepath.Dir(root), "named.txt")) + }) + } +} + +// A download that fails to connect is reported as a failed download. +func TestDownloadConnectionFailureIsDownloadError(t *testing.T) { + s := newPullTestServer(t) + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := l.Addr().String() + require.NoError(t, l.Close()) + + u, err := url.Parse("http://" + addr + "/file") + require.NoError(t, err) + err = New(s, DownloadRequest{Directory: "/", URL: u, FileName: "file"}).Execute() + require.Error(t, err) + require.True(t, IsDownloadError(err), "got %v", err) + require.ErrorIs(t, err, ErrDownloadFailed) +}