fix(downloader): allow remote downloads without a Content-Length

Pulling a file from a URL failed with "request is missing ContentLength"
whenever the response had no known length. That happens for chunked
responses, and also for any gzip encoded response, because Go's HTTP
client decompresses it transparently and reports the length as -1.
raw.githubusercontent.com and pastebin both answer that way.

Instead of rejecting these responses, Filesystem.Write now accepts a
negative size meaning "unknown". The data is written to a hidden
temporary file next to the target through the existing quota file, so
the disk limit is enforced on every write, and it is only renamed over
the target once the copy has finished. If the download does not fit or
fails for any other reason, the temporary file is removed and an
existing file at the target is left untouched.

Downloads with a known length behave as before. Progress is only
tracked when the total size is known.

Fixes #133

router/downloader/downloader.go

@@ -252,10 +252,6 @@ func (dl *Download) Execute() error {
 		return errors.New("downloader: got bad response status from endpoint: " + res.Status)
 	}
 
-	if res.ContentLength < 1 {
-		return errors.New("downloader: request is missing ContentLength")
-	}
-
 	if dl.req.UseHeader {
 		if contentDisposition := res.Header.Get("Content-Disposition"); contentDisposition != "" {
 			_, params, err := mime.ParseMediaType(contentDisposition)
@@ -285,8 +281,14 @@ func (dl *Download) Execute() error {
 	dl.server.Log().WithField("path", p).Debug("writing remote file to disk")
 
 	// Write the file while tracking the progress, Write will check that the
-	// size of the file won't exceed the disk limit.
-	r := io.TeeReader(res.Body, dl.counter(res.ContentLength))
+	// size of the file won't exceed the disk limit. The length is unknown (-1)
+	// for chunked responses and for responses that the HTTP client transparently
+	// decompressed, in which case Write enforces the limit as the data comes in
+	// and the progress stays at zero since there is nothing to measure it against.
+	var r io.Reader = res.Body
+	if res.ContentLength > 0 {
+		r = io.TeeReader(res.Body, dl.counter(res.ContentLength))
+	}
 	if err := dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644); err != nil {
 		return errors.WrapIf(err, "downloader: failed to write file to server directory")
 	}
@@ -308,7 +310,8 @@ func (dl *Download) BelongsTo(s *server.Server) bool {
 }
 
 // Progress returns the current progress of the download as a float value between 0 and 1 where
-// 1 indicates that the download is completed.
+// 1 indicates that the download is completed. If the remote server did not report the size of
+// the file, the progress remains at 0 until the download finishes.
 func (dl *Download) Progress() float64 {
 	dl.mu.RLock()
 	defer dl.mu.RUnlock()