fix(downloader): allow remote downloads without a Content-Length
Pulling a file from a URL failed with "request is missing ContentLength" whenever the response had no known length. That happens for chunked responses, and also for any gzip encoded response, because Go's HTTP client decompresses it transparently and reports the length as -1. raw.githubusercontent.com and pastebin both answer that way. Instead of rejecting these responses, Filesystem.Write now accepts a negative size meaning "unknown". The data is written to a hidden temporary file next to the target through the existing quota file, so the disk limit is enforced on every write, and it is only renamed over the target once the copy has finished. If the download does not fit or fails for any other reason, the temporary file is removed and an existing file at the target is left untouched. Downloads with a known length behave as before. Progress is only tracked when the total size is known. Fixes #133
This commit is contained in:
1 parent
7ffd9d8166
commit
6b628f71d7
4 files changed
+410
-7
No files matched your search
@@ -297,6 +297,62 @@ func TestFilesystem_Writefile(t *testing.T) {
|
||||
g.Assert(getFileContent(f)).Equal("new data")
|
||||
})
|
||||
|
||||
g.It("can replace a file with contents of an unknown size", func() {
|
||||
fs.SetDiskLimit(1024)
|
||||
|
||||
r := bytes.NewReader([]byte("original data"))
|
||||
err := fs.Write("test.txt", r, r.Size(), 0o644)
|
||||
g.Assert(err).IsNil()
|
||||
g.Assert(fs.CachedUsage()).Equal(int64(13))
|
||||
|
||||
err = fs.Write("test.txt", bytes.NewBufferString("new data"), -1, 0o644)
|
||||
g.Assert(err).IsNil()
|
||||
|
||||
f, _, err := fs.File("test.txt")
|
||||
g.Assert(err).IsNil()
|
||||
defer f.Close()
|
||||
g.Assert(getFileContent(f)).Equal("new data")
|
||||
g.Assert(fs.CachedUsage()).Equal(int64(8))
|
||||
|
||||
entries, err := fs.ReadDir("/")
|
||||
g.Assert(err).IsNil()
|
||||
g.Assert(len(entries)).Equal(1)
|
||||
})
|
||||
|
||||
g.It("cannot write contents of an unknown size that exceed the disk limits", func() {
|
||||
fs.SetDiskLimit(1024)
|
||||
|
||||
err := fs.Write("test.txt", bytes.NewReader(make([]byte, 2048)), -1, 0o644)
|
||||
g.Assert(err).IsNotNil()
|
||||
g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue()
|
||||
|
||||
_, err = rfs.StatServerFile("test.txt")
|
||||
g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue("err is not os.ErrNotExist")
|
||||
g.Assert(fs.CachedUsage()).Equal(int64(0))
|
||||
})
|
||||
|
||||
g.It("keeps the existing file when contents of an unknown size do not fit", func() {
|
||||
fs.SetDiskLimit(1024)
|
||||
|
||||
r := bytes.NewReader([]byte("original data"))
|
||||
err := fs.Write("test.txt", r, r.Size(), 0o644)
|
||||
g.Assert(err).IsNil()
|
||||
|
||||
err = fs.Write("test.txt", bytes.NewReader(make([]byte, 1020)), -1, 0o644)
|
||||
g.Assert(err).IsNotNil()
|
||||
g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue()
|
||||
|
||||
f, _, err := fs.File("test.txt")
|
||||
g.Assert(err).IsNil()
|
||||
defer f.Close()
|
||||
g.Assert(getFileContent(f)).Equal("original data")
|
||||
g.Assert(fs.CachedUsage()).Equal(int64(13))
|
||||
|
||||
entries, err := fs.ReadDir("/")
|
||||
g.Assert(err).IsNil()
|
||||
g.Assert(len(entries)).Equal(1)
|
||||
})
|
||||
|
||||
g.AfterEach(func() {
|
||||
buf.Truncate(0)
|
||||
_ = fs.TruncateRootDirectory()
|
||||
|
||||
Reference in new issue
Block a user