fix(downloader): allow remote downloads without a Content-Length

Pulling a file from a URL failed with "request is missing ContentLength"
whenever the response had no known length. That happens for chunked
responses, and also for any gzip encoded response, because Go's HTTP
client decompresses it transparently and reports the length as -1.
raw.githubusercontent.com and pastebin both answer that way.

Instead of rejecting these responses, Filesystem.Write now accepts a
negative size meaning "unknown". The data is written to a hidden
temporary file next to the target through the existing quota file, so
the disk limit is enforced on every write, and it is only renamed over
the target once the copy has finished. If the download does not fit or
fails for any other reason, the temporary file is removed and an
existing file at the target is left untouched.

Downloads with a known length behave as before. Progress is only
tracked when the total size is known.

Fixes #133

server/filesystem/filesystem.go

@@ -3,6 +3,7 @@ package filesystem
 import (
 	"fmt"
 	"io"
+	"math/rand/v2"
 	"os"
 	"path/filepath"
 	"slices"
@@ -16,6 +17,7 @@ import (
 	"github.com/apex/log"
 	"github.com/gabriel-vasile/mimetype"
 	ignore "github.com/sabhiram/go-gitignore"
+	"golang.org/x/sys/unix"
 
 	"github.com/pelican/wings/config"
 	"github.com/pelican/wings/internal/ufs"
@@ -152,6 +154,12 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error {
 	return err
 }
 
+// Write writes the contents of r to the file at p, creating or truncating it
+// as needed. newSize is the number of bytes that will be read from r and is
+// checked against the disk limit before anything is written. A negative
+// newSize means the size is not known ahead of time, in which case the disk
+// limit is enforced while the data is being written instead, and an existing
+// file at p is only replaced once all of the data has been written.
 func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode ufs.FileMode) error {
 	var currentSize int64
 	st, err := fs.unixFS.Stat(p)
@@ -180,6 +188,10 @@ func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode ufs.FileM
 		return err
 	}
 
+	if newSize < 0 {
+		return fs.writeUnknownSize(p, r, mode)
+	}
+
 	// Touch the file and return the handle to it at this point. This will
 	// create or truncate the file, and create any necessary parent directories
 	// if they are missing.
@@ -210,6 +222,75 @@ func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode ufs.FileM
 	return err
 }
 
+// writeUnknownSize writes r to p when the final size of the data is not known
+// up front. The data goes to a hidden temporary file next to p, with every
+// write checked against the disk limit, and is only renamed over p once all of
+// it has been written. If anything goes wrong, including the data not fitting
+// in the disk limit, the temporary file is removed and any existing file at p
+// is left untouched.
+func (fs *Filesystem) writeUnknownSize(p string, r io.Reader, mode ufs.FileMode) error {
+	dirfd, name, closeFd, err := fs.unixFS.SafePath(p)
+	defer closeFd()
+	if err != nil {
+		return err
+	}
+
+	tmpName := "." + name + "." + strconv.FormatUint(rand.Uint64(), 36) + ".tmp"
+	file, err := fs.unixFS.OpenFileat(dirfd, tmpName, ufs.O_RDWR|ufs.O_CREATE|ufs.O_EXCL, mode)
+	if err != nil {
+		return err
+	}
+
+	dst := newQuotaFile(fs, file, 0)
+	_, err = io.Copy(dst, r)
+	if closeErr := dst.Close(); err == nil {
+		err = closeErr
+	}
+	if err == nil && !fs.isTest {
+		err = fs.unixFS.Lchownat(dirfd, tmpName, config.Get().System.User.Uid, config.Get().System.User.Gid)
+	}
+	if err == nil {
+		err = fs.replaceFile(dirfd, tmpName, name)
+	}
+	if err != nil {
+		fs.removeTempFile(dirfd, tmpName)
+		return err
+	}
+	return nil
+}
+
+// replaceFile atomically renames tmpName over name within dirfd, releasing the
+// disk space used by the file that gets replaced.
+func (fs *Filesystem) replaceFile(dirfd int, tmpName, name string) error {
+	var replacedSize int64
+	st, err := fs.unixFS.Lstatat(dirfd, name)
+	if err != nil && !errors.Is(err, ufs.ErrNotExist) {
+		return err
+	} else if err == nil && st.Mode().IsRegular() {
+		replacedSize = st.Size()
+	}
+
+	if err := unix.Renameat(dirfd, tmpName, dirfd, name); err != nil {
+		return &ufs.LinkError{Op: "rename", Old: tmpName, New: name, Err: err}
+	}
+	fs.unixFS.Add(-replacedSize)
+	return nil
+}
+
+// removeTempFile removes a temporary file created by writeUnknownSize and
+// releases the disk space that was reserved while writing it.
+func (fs *Filesystem) removeTempFile(dirfd int, tmpName string) {
+	st, err := fs.unixFS.Lstatat(dirfd, tmpName)
+	if err != nil {
+		return
+	}
+	if err := unix.Unlinkat(dirfd, tmpName, 0); err != nil {
+		log.WithField("file", tmpName).WithField("error", err).Warn("failed to remove temporary file")
+		return
+	}
+	fs.unixFS.Add(-st.Size())
+}
+
 // CreateDirectory creates a new directory (name) at a specified path (p) for
 // the server.
 func (fs *Filesystem) CreateDirectory(name string, p string) error {

server/filesystem/filesystem_test.go

@@ -297,6 +297,62 @@ func TestFilesystem_Writefile(t *testing.T) {
 			g.Assert(getFileContent(f)).Equal("new data")
 		})
 
+		g.It("can replace a file with contents of an unknown size", func() {
+			fs.SetDiskLimit(1024)
+
+			r := bytes.NewReader([]byte("original data"))
+			err := fs.Write("test.txt", r, r.Size(), 0o644)
+			g.Assert(err).IsNil()
+			g.Assert(fs.CachedUsage()).Equal(int64(13))
+
+			err = fs.Write("test.txt", bytes.NewBufferString("new data"), -1, 0o644)
+			g.Assert(err).IsNil()
+
+			f, _, err := fs.File("test.txt")
+			g.Assert(err).IsNil()
+			defer f.Close()
+			g.Assert(getFileContent(f)).Equal("new data")
+			g.Assert(fs.CachedUsage()).Equal(int64(8))
+
+			entries, err := fs.ReadDir("/")
+			g.Assert(err).IsNil()
+			g.Assert(len(entries)).Equal(1)
+		})
+
+		g.It("cannot write contents of an unknown size that exceed the disk limits", func() {
+			fs.SetDiskLimit(1024)
+
+			err := fs.Write("test.txt", bytes.NewReader(make([]byte, 2048)), -1, 0o644)
+			g.Assert(err).IsNotNil()
+			g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue()
+
+			_, err = rfs.StatServerFile("test.txt")
+			g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue("err is not os.ErrNotExist")
+			g.Assert(fs.CachedUsage()).Equal(int64(0))
+		})
+
+		g.It("keeps the existing file when contents of an unknown size do not fit", func() {
+			fs.SetDiskLimit(1024)
+
+			r := bytes.NewReader([]byte("original data"))
+			err := fs.Write("test.txt", r, r.Size(), 0o644)
+			g.Assert(err).IsNil()
+
+			err = fs.Write("test.txt", bytes.NewReader(make([]byte, 1020)), -1, 0o644)
+			g.Assert(err).IsNotNil()
+			g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue()
+
+			f, _, err := fs.File("test.txt")
+			g.Assert(err).IsNil()
+			defer f.Close()
+			g.Assert(getFileContent(f)).Equal("original data")
+			g.Assert(fs.CachedUsage()).Equal(int64(13))
+
+			entries, err := fs.ReadDir("/")
+			g.Assert(err).IsNil()
+			g.Assert(len(entries)).Equal(1)
+		})
+
 		g.AfterEach(func() {
 			buf.Truncate(0)
 			_ = fs.TruncateRootDirectory()