fix(downloader): allow remote downloads without a Content-Length

Pulling a file from a URL failed with "request is missing ContentLength"
whenever the response had no known length. That happens for chunked
responses, and also for any gzip encoded response, because Go's HTTP
client decompresses it transparently and reports the length as -1.
raw.githubusercontent.com and pastebin both answer that way.

Instead of rejecting these responses, Filesystem.Write now accepts a
negative size meaning "unknown". The data is written to a hidden
temporary file next to the target through the existing quota file, so
the disk limit is enforced on every write, and it is only renamed over
the target once the copy has finished. If the download does not fit or
fails for any other reason, the temporary file is removed and an
existing file at the target is left untouched.

Downloads with a known length behave as before. Progress is only
tracked when the total size is known.

Fixes #133
This commit is contained in:
dualfroz committed 2026-09-24 18:45:26 +02:00
1 parent 7ffd9d8166
commit 6b628f71d7
4 files changed
+410 -7

No files matched your search

+56
View File
@@ -297,6 +297,62 @@ func TestFilesystem_Writefile(t *testing.T) {
g.Assert(getFileContent(f)).Equal("new data")
})
g.It("can replace a file with contents of an unknown size", func() {
fs.SetDiskLimit(1024)
r := bytes.NewReader([]byte("original data"))
err := fs.Write("test.txt", r, r.Size(), 0o644)
g.Assert(err).IsNil()
g.Assert(fs.CachedUsage()).Equal(int64(13))
err = fs.Write("test.txt", bytes.NewBufferString("new data"), -1, 0o644)
g.Assert(err).IsNil()
f, _, err := fs.File("test.txt")
g.Assert(err).IsNil()
defer f.Close()
g.Assert(getFileContent(f)).Equal("new data")
g.Assert(fs.CachedUsage()).Equal(int64(8))
entries, err := fs.ReadDir("/")
g.Assert(err).IsNil()
g.Assert(len(entries)).Equal(1)
})
g.It("cannot write contents of an unknown size that exceed the disk limits", func() {
fs.SetDiskLimit(1024)
err := fs.Write("test.txt", bytes.NewReader(make([]byte, 2048)), -1, 0o644)
g.Assert(err).IsNotNil()
g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue()
_, err = rfs.StatServerFile("test.txt")
g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue("err is not os.ErrNotExist")
g.Assert(fs.CachedUsage()).Equal(int64(0))
})
g.It("keeps the existing file when contents of an unknown size do not fit", func() {
fs.SetDiskLimit(1024)
r := bytes.NewReader([]byte("original data"))
err := fs.Write("test.txt", r, r.Size(), 0o644)
g.Assert(err).IsNil()
err = fs.Write("test.txt", bytes.NewReader(make([]byte, 1020)), -1, 0o644)
g.Assert(err).IsNotNil()
g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue()
f, _, err := fs.File("test.txt")
g.Assert(err).IsNil()
defer f.Close()
g.Assert(getFileContent(f)).Equal("original data")
g.Assert(fs.CachedUsage()).Equal(int64(13))
entries, err := fs.ReadDir("/")
g.Assert(err).IsNil()
g.Assert(len(entries)).Equal(1)
})
g.AfterEach(func() {
buf.Truncate(0)
_ = fs.TruncateRootDirectory()