From 7ffd9d81665d96011faa56b7cc5ea268c9cc516c Mon Sep 17 00:00:00 2001 From: upstream Date: Thu, 24 Sep 2026 18:35:19 +0200 Subject: [PATCH] Upstream pelican/wings state of the touched files before 04487e1e6b --- router/downloader/downloader.go | 404 +++++++++++++++ server/filesystem/filesystem.go | 605 +++++++++++++++++++++++ server/filesystem/filesystem_test.go | 711 +++++++++++++++++++++++++++ 3 files changed, 1720 insertions(+) create mode 100644 router/downloader/downloader.go create mode 100644 server/filesystem/filesystem.go create mode 100644 server/filesystem/filesystem_test.go diff --git a/router/downloader/downloader.go b/router/downloader/downloader.go new file mode 100644 index 0000000..69b81cc --- /dev/null +++ b/router/downloader/downloader.go @@ -0,0 +1,404 @@ +package downloader + +import ( + "context" + "fmt" + "io" + "mime" + "net" + "net/http" + "net/url" + "path/filepath" + "strings" + "sync" + "time" + + "emperror.dev/errors" + "github.com/goccy/go-json" + "github.com/google/uuid" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/server" +) + +var client *http.Client + +func init() { + dialer := &net.Dialer{ + LocalAddr: nil, + } + + trnspt := http.DefaultTransport.(*http.Transport).Clone() + trnspt.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + c, err := dialer.DialContext(ctx, network, addr) + if err != nil { + return nil, errors.WithStack(err) + } + + ipStr, _, err := net.SplitHostPort(c.RemoteAddr().String()) + if err != nil { + return c, errors.WithStack(err) + } + ip := net.ParseIP(ipStr) + if ip == nil { + return c, errors.WithStack(ErrInvalidIPAddress) + } + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() { + return c, errors.WithStack(ErrInternalResolution) + } + for _, block := range internalRanges { + if !block.Contains(ip) { + continue + } + return c, errors.WithStack(ErrInternalResolution) + } + return c, nil + } + + client = &http.Client{ + Timeout: time.Hour * 12, + + Transport: trnspt, + + // Disallow any redirect on an HTTP call. This is a security requirement: do not modify + // this logic without first ensuring that the new target location IS NOT within the current + // instance's local network. + // + // This specific error response just causes the client to not follow the redirect and + // returns the actual redirect response to the caller. Not perfect, but simple and most + // people won't be using URLs that redirect anyways hopefully? + // + // We'll re-evaluate this down the road if needed. + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +var instance = &Downloader{ + // Tracks all the active downloads. + downloadCache: make(map[string]*Download), + // Tracks all the downloads active for a given server instance. This is + // primarily used to make things quicker and keep the code a little more + // legible throughout here. + serverCache: make(map[string][]string), +} + +// Internal IP ranges that should be blocked if the resource requested resolves within. +var internalRanges = []*net.IPNet{ + mustParseCIDR("127.0.0.1/8"), + mustParseCIDR("10.0.0.0/8"), + mustParseCIDR("172.16.0.0/12"), + mustParseCIDR("192.168.0.0/16"), + mustParseCIDR("169.254.0.0/16"), + mustParseCIDR("::1/128"), + mustParseCIDR("fe80::/10"), + mustParseCIDR("fc00::/7"), +} + +const ( + ErrInternalResolution = errors.Sentinel("downloader: destination resolves to internal network location") + ErrInvalidIPAddress = errors.Sentinel("downloader: invalid IP address") + ErrDownloadFailed = errors.Sentinel("downloader: download request failed") +) + +const defaultMaxRedirects = 10 + +type Counter struct { + total int + onWrite func(total int) +} + +func (c *Counter) Write(p []byte) (int, error) { + n := len(p) + c.total += n + c.onWrite(c.total) + return n, nil +} + +type DownloadRequest struct { + Directory string + URL *url.URL + FileName string + UseHeader bool +} + +type Download struct { + Identifier string + path string + mu sync.RWMutex + req DownloadRequest + server *server.Server + progress float64 + cancelFunc *context.CancelFunc +} + +// New starts a new tracked download which allows for cancellation later on by calling +// the Downloader.Cancel function. +func New(s *server.Server, r DownloadRequest) *Download { + dl := Download{ + Identifier: uuid.Must(uuid.NewRandom()).String(), + req: r, + server: s, + } + instance.track(&dl) + return &dl +} + +// ByServer returns all the tracked downloads for a given server instance. +func ByServer(sid string) []*Download { + instance.mu.Lock() + defer instance.mu.Unlock() + var downloads []*Download + if v, ok := instance.serverCache[sid]; ok { + for _, id := range v { + if dl, ok := instance.downloadCache[id]; ok { + downloads = append(downloads, dl) + } + } + } + return downloads +} + +// ByID returns a single Download matching a given identifier. If no download is found +// the second argument in the response will be false. +func ByID(dlid string) *Download { + return instance.find(dlid) +} + +//goland:noinspection GoVetCopyLock +func (dl Download) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Identifier string + Progress float64 + }{ + Identifier: dl.Identifier, + Progress: dl.Progress(), + }) +} + +// Execute executes a given download for the server and begins writing the file to the disk. Once +// completed the download will be removed from the cache. +func (dl *Download) Execute() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Hour*12) + dl.cancelFunc = &cancel + defer dl.Cancel() + + // At this point we have verified the destination is not within the local network, so we can + // now make a request to that URL and pull down the file, saving it to the server's data + // directory. + currentURL := dl.req.URL + if currentURL == nil { + return errors.New("downloader: download request url is nil") + } + + visited := make(map[string]struct{}) + var res *http.Response + var finalURL *url.URL + + maxRedirects := maxRedirectAttempts() + for redirects := 0; redirects < maxRedirects; redirects++ { + urlStr := currentURL.String() + if _, seen := visited[urlStr]; seen { + return errors.New("downloader: detected redirect loop") + } + visited[urlStr] = struct{}{} + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil) + if err != nil { + return errors.WrapIf(err, "downloader: failed to create request") + } + + req.Header.Set("User-Agent", "Pelican Panel (https://pelican.dev)") + res, err = client.Do(req) + if err != nil { + return errors.WrapIf(err, "downloader: failed to perform request") + } + + if res.StatusCode >= http.StatusMultipleChoices && res.StatusCode < http.StatusBadRequest { + location := res.Header.Get("Location") + res.Body.Close() + if location == "" { + return errors.New("downloader: redirect response missing location header") + } + + nextURL, err := currentURL.Parse(location) + if err != nil { + return errors.WrapIf(err, "downloader: invalid redirect location") + } + if nextURL.Scheme != "http" && nextURL.Scheme != "https" { + return errors.New("downloader: redirect to unsupported scheme") + } + + currentURL = nextURL + finalURL = nextURL + continue + } + + finalURL = currentURL + break + } + + if res == nil { + return errors.New("downloader: exceeded maximum redirect attempts") + } + defer res.Body.Close() + + if res.StatusCode >= http.StatusMultipleChoices && res.StatusCode < http.StatusBadRequest { + return errors.New("downloader: exceeded maximum redirect attempts") + } + + if res.StatusCode != http.StatusOK { + return errors.New("downloader: got bad response status from endpoint: " + res.Status) + } + + if res.ContentLength < 1 { + return errors.New("downloader: request is missing ContentLength") + } + + if dl.req.UseHeader { + if contentDisposition := res.Header.Get("Content-Disposition"); contentDisposition != "" { + _, params, err := mime.ParseMediaType(contentDisposition) + if err != nil { + return errors.WrapIf(err, "downloader: invalid \"Content-Disposition\" header") + } + + if v, ok := params["filename"]; ok { + dl.path = v + } + } + } + if dl.path == "" { + if dl.req.FileName != "" { + dl.path = dl.req.FileName + } else { + pathSource := dl.req.URL + if finalURL != nil { + pathSource = finalURL + } + parts := strings.Split(pathSource.Path, "/") + dl.path = parts[len(parts)-1] + } + } + + p := dl.Path() + dl.server.Log().WithField("path", p).Debug("writing remote file to disk") + + // Write the file while tracking the progress, Write will check that the + // size of the file won't exceed the disk limit. + r := io.TeeReader(res.Body, dl.counter(res.ContentLength)) + if err := dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644); err != nil { + return errors.WrapIf(err, "downloader: failed to write file to server directory") + } + return nil +} + +// Cancel cancels a running download and frees up the associated resources. If a file is being +// written a partial file will remain present on the disk. +func (dl *Download) Cancel() { + if dl.cancelFunc != nil { + (*dl.cancelFunc)() + } + instance.remove(dl.Identifier) +} + +// BelongsTo checks if the given download belongs to the provided server. +func (dl *Download) BelongsTo(s *server.Server) bool { + return dl.server.ID() == s.ID() +} + +// Progress returns the current progress of the download as a float value between 0 and 1 where +// 1 indicates that the download is completed. +func (dl *Download) Progress() float64 { + dl.mu.RLock() + defer dl.mu.RUnlock() + return dl.progress +} + +func (dl *Download) Path() string { + return filepath.Join(dl.req.Directory, dl.path) +} + +// Handles a write event by updating the progress completed percentage and firing off +// events to the server websocket as needed. +func (dl *Download) counter(contentLength int64) *Counter { + onWrite := func(t int) { + dl.mu.Lock() + defer dl.mu.Unlock() + dl.progress = float64(t) / float64(contentLength) + } + return &Counter{ + onWrite: onWrite, + } +} + +// Downloader represents a global downloader that keeps track of all currently processing downloads +// for the machine. +type Downloader struct { + mu sync.RWMutex + downloadCache map[string]*Download + serverCache map[string][]string +} + +// track tracks a download in the internal cache for this instance. +func (d *Downloader) track(dl *Download) { + d.mu.Lock() + defer d.mu.Unlock() + sid := dl.server.ID() + if _, ok := d.downloadCache[dl.Identifier]; !ok { + d.downloadCache[dl.Identifier] = dl + if _, ok := d.serverCache[sid]; !ok { + d.serverCache[sid] = []string{} + } + d.serverCache[sid] = append(d.serverCache[sid], dl.Identifier) + } +} + +// find finds a given download entry using the provided ID and returns it. +func (d *Downloader) find(dlid string) *Download { + d.mu.RLock() + defer d.mu.RUnlock() + if entry, ok := d.downloadCache[dlid]; ok { + return entry + } + return nil +} + +// remove removes the given download reference from the cache storing them. This also updates +// the slice of active downloads for a given server to not include this download. +func (d *Downloader) remove(dlID string) { + d.mu.Lock() + defer d.mu.Unlock() + if _, ok := d.downloadCache[dlID]; !ok { + return + } + sID := d.downloadCache[dlID].server.ID() + delete(d.downloadCache, dlID) + if tracked, ok := d.serverCache[sID]; ok { + var out []string + for _, k := range tracked { + if k != dlID { + out = append(out, k) + } + } + d.serverCache[sID] = out + } +} + +func mustParseCIDR(ip string) *net.IPNet { + _, block, err := net.ParseCIDR(ip) + if err != nil { + panic(fmt.Errorf("downloader: failed to parse CIDR: %s", err)) + } + return block +} + +func maxRedirectAttempts() int { + cfg := config.Get() + if cfg != nil { + if v := cfg.Api.RemoteDownload.MaxRedirects; v > 0 { + return v + } + } + return defaultMaxRedirects +} diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go new file mode 100644 index 0000000..5430e30 --- /dev/null +++ b/server/filesystem/filesystem.go @@ -0,0 +1,605 @@ +package filesystem + +import ( + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "emperror.dev/errors" + "github.com/apex/log" + "github.com/gabriel-vasile/mimetype" + ignore "github.com/sabhiram/go-gitignore" + + "github.com/pelican/wings/config" + "github.com/pelican/wings/internal/ufs" +) + +type Filesystem struct { + unixFS *ufs.Quota + + mu sync.RWMutex + lastLookupTime *usageLookupTime + lookupInProgress atomic.Bool + diskCheckInterval time.Duration + denylist *ignore.GitIgnore + + isTest bool +} + +// New creates a new Filesystem instance for a given server. +func New(root string, size int64, denylist []string) (*Filesystem, error) { + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, err + } + unixFS, err := ufs.NewUnixFS(root, config.UseOpenat2()) + if err != nil { + return nil, err + } + quota := ufs.NewQuota(unixFS, size) + + return &Filesystem{ + unixFS: quota, + + diskCheckInterval: time.Duration(config.Get().System.DiskCheckInterval), + lastLookupTime: &usageLookupTime{}, + denylist: ignore.CompileIgnoreLines(denylist...), + }, nil +} + +// Path returns the root path for the Filesystem instance. +func (fs *Filesystem) Path() string { + return fs.unixFS.BasePath() +} + +// ReadDir reads directory entries. +func (fs *Filesystem) ReadDir(path string) ([]ufs.DirEntry, error) { + return fs.unixFS.ReadDir(path) +} + +// ReadDirStat is like ReadDir except that it returns FileInfo for each entry +// instead of just a DirEntry. +func (fs *Filesystem) ReadDirStat(path string) ([]ufs.FileInfo, error) { + return ufs.ReadDirMap(fs.unixFS.UnixFS, path, func(e ufs.DirEntry) (ufs.FileInfo, error) { + return e.Info() + }) +} + +// File returns a reader for a file instance as well as the stat information. +func (fs *Filesystem) File(p string) (ufs.File, Stat, error) { + f, err := fs.unixFS.Open(p) + if err != nil { + return nil, Stat{}, err + } + st, err := statFromFile(f) + if err != nil { + _ = f.Close() + return nil, Stat{}, err + } + if st.IsDir() { + return f, Stat{}, errors.New("filesystem: is a directory") + } + return f, st, nil +} + +func (fs *Filesystem) UnixFS() *ufs.UnixFS { + return fs.unixFS.UnixFS +} + +// Touch acts by creating the given file and path on the disk if it is not present +// already. If it is present, the file is opened using the defaults which will truncate +// the contents. The opened file is then returned to the caller. +func (fs *Filesystem) Touch(p string, flag int) (ufs.File, error) { + var currentSize int64 + st, err := fs.unixFS.Stat(p) + if err != nil && !errors.Is(err, ufs.ErrNotExist) { + return nil, err + } else if err == nil && !st.IsDir() { + currentSize = st.Size() + } + + file, err := fs.unixFS.Touch(p, flag, 0o644) + if err != nil { + return nil, err + } + return newQuotaFile(fs, file, currentSize), nil +} + +// Writefile writes a file to the system. If the file does not already exist one +// will be created. This will also properly recalculate the disk space used by +// the server when writing new files or modifying existing ones. +// +// DEPRECATED: use `Write` instead. +func (fs *Filesystem) Writefile(p string, r io.Reader) error { + var currentSize int64 + st, err := fs.unixFS.Stat(p) + if err != nil && !errors.Is(err, ufs.ErrNotExist) { + return errors.Wrap(err, "server/filesystem: writefile: failed to stat file") + } else if err == nil { + if st.IsDir() { + // TODO: resolved + return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) + } + currentSize = st.Size() + } + + // Touch the file and return the handle to it at this point. This will + // create or truncate the file, and create any necessary parent directories + // if they are missing. + file, err := fs.unixFS.Touch(p, ufs.O_RDWR|ufs.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("error touching file: %w", err) + } + defer file.Close() + + // Do not use CopyBuffer here, it is wasteful as the file implements + // io.ReaderFrom, which causes it to not use the buffer anyways. + n, err := io.Copy(file, r) + + // Adjust the disk usage to account for the old size and the new size of the file. + fs.unixFS.Add(n - currentSize) + + if err := fs.chownFile(p); err != nil { + return fmt.Errorf("error chowning file: %w", err) + } + // Return the error from io.Copy. + return err +} + +func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode ufs.FileMode) error { + var currentSize int64 + st, err := fs.unixFS.Stat(p) + if err != nil && !errors.Is(err, ufs.ErrNotExist) { + return errors.Wrap(err, "server/filesystem: writefile: failed to stat file") + } else if err == nil { + if st.IsDir() { + // TODO: resolved + return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) + } + currentSize = st.Size() + } + + // Check that the new size we're writing to the disk can fit. If there is currently + // a file we'll subtract that current file size from the size of the buffer to determine + // the amount of new data we're writing (or amount we're removing if smaller). + if err := fs.HasSpaceFor(newSize - currentSize); err != nil { + return err + } + + // Ensure the parent directories exist and are owned by the server user + // before creating the file. Touch would create any missing parents + // implicitly, but as the user Wings runs as; creating them here lets us + // chown the ones we add. + if err := fs.mkdirAll(filepath.Dir(p), 0o755); err != nil { + return err + } + + // Touch the file and return the handle to it at this point. This will + // create or truncate the file, and create any necessary parent directories + // if they are missing. + file, err := fs.unixFS.Touch(p, ufs.O_RDWR|ufs.O_TRUNC, mode) + if err != nil { + return err + } + defer file.Close() + + if newSize == 0 { + // Subtract the previous size of the file if the new size is 0. + fs.unixFS.Add(-currentSize) + } else { + // Do not use CopyBuffer here, it is wasteful as the file implements + // io.ReaderFrom, which causes it to not use the buffer anyways. + var n int64 + n, err = io.Copy(file, io.LimitReader(r, newSize)) + + // Adjust the disk usage to account for the old size and the new size of the file. + fs.unixFS.Add(n - currentSize) + } + + if err := fs.chownFile(p); err != nil { + return err + } + + // Return any remaining error. + return err +} + +// CreateDirectory creates a new directory (name) at a specified path (p) for +// the server. +func (fs *Filesystem) CreateDirectory(name string, p string) error { + return fs.mkdirAll(filepath.Join(p, name), 0o755) +} + +func (fs *Filesystem) Rename(oldpath, newpath string) error { + return fs.unixFS.Rename(oldpath, newpath) +} + +func (fs *Filesystem) Symlink(oldpath, newpath string) error { + return fs.unixFS.Symlink(oldpath, newpath) +} + +func (fs *Filesystem) chownFile(name string) error { + if fs.isTest { + return nil + } + + uid := config.Get().System.User.Uid + gid := config.Get().System.User.Gid + return fs.unixFS.Lchown(name, uid, gid) +} + +// mkdirAll creates the directory p along with any missing parents, chowning +// every directory it creates to the server user so they are not left owned by +// the user Wings runs as. +func (fs *Filesystem) mkdirAll(p string, mode ufs.FileMode) error { + created, err := fs.unixFS.MkdirAll(p, mode) + if err != nil { + return err + } + for _, dir := range created { + if err := fs.chownFile(dir); err != nil { + return err + } + } + return nil +} + +// Chown recursively iterates over a file or directory and sets the permissions on all of the +// underlying files. Iterate over all of the files and directories. If it is a file just +// go ahead and perform the chown operation. Otherwise dig deeper into the directory until +// we've run out of directories to dig into. +func (fs *Filesystem) Chown(p string) error { + if fs.isTest { + return nil + } + + uid := config.Get().System.User.Uid + gid := config.Get().System.User.Gid + + dirfd, name, closeFd, err := fs.unixFS.SafePath(p) + defer closeFd() + if err != nil { + return err + } + + // Start by just chowning the initial path that we received. + if err := fs.unixFS.Lchownat(dirfd, name, uid, gid); err != nil { + return errors.Wrap(err, "server/filesystem: chown: failed to chown path") + } + + // If this is not a directory we can now return from the function, there is nothing + // left that we need to do. + if st, err := fs.unixFS.Lstatat(dirfd, name); err != nil || !st.IsDir() { + return nil + } + + // This walker is probably some of the most efficient code in Wings. It has + // an internally re-used buffer for listing directory entries and doesn't + // need to check if every individual path it touches is safe as the code + // doesn't traverse symlinks, is immune to symlink timing attacks, and + // gives us a dirfd and file name to make a direct syscall with. + if err := fs.unixFS.WalkDirat(dirfd, name, func(dirfd int, name, _ string, info ufs.DirEntry, err error) error { + if err != nil { + return err + } + if err := fs.unixFS.Lchownat(dirfd, name, uid, gid); err != nil { + return err + } + return nil + }); err != nil { + return fmt.Errorf("server/filesystem: chown: failed to chown during walk function: %w", err) + } + return nil +} + +func (fs *Filesystem) Chmod(path string, mode ufs.FileMode) error { + return fs.unixFS.Chmod(path, mode) +} + +// Begin looping up to 50 times to try and create a unique copy file name. This will take +// an input of "file.txt" and generate "file copy.txt". If that name is already taken, it will +// then try to write "file copy 2.txt" and so on, until reaching 50 loops. At that point we +// won't waste anymore time, just use the current timestamp and make that copy. +// +// Could probably make this more efficient by checking if there are any files matching the copy +// pattern, and trying to find the highest number and then incrementing it by one rather than +// looping endlessly. +func (fs *Filesystem) findCopySuffix(dirfd int, name, extension string) (string, error) { + var i int + suffix := "" + + for i = 0; i < 51; i++ { + if i == 1 { + suffix = " copy" + } else if i == 50 { + suffix = " copy." + time.Now().Format(time.RFC3339) + } else if i > 1 { + suffix = " copy " + strconv.Itoa(i-1) + } + + n := name + suffix + extension + // If we stat the file and it does not exist that means we're good to create the copy. If it + // does exist, we'll just continue to the next loop and try again. + if _, err := fs.unixFS.Lstatat(dirfd, n); err != nil { + if !errors.Is(err, ufs.ErrNotExist) { + return "", err + } + break + } + } + + return name + suffix + extension, nil +} + +// Copy copies a given file to the same location and appends a suffix to the +// file to indicate that it has been copied. +func (fs *Filesystem) Copy(p string) error { + dirfd, name, closeFd, err := fs.unixFS.SafePath(p) + defer closeFd() + if err != nil { + return err + } + source, err := fs.unixFS.OpenFileat(dirfd, name, ufs.O_RDONLY, 0) + if err != nil { + return err + } + defer source.Close() + info, err := source.Stat() + if err != nil { + return err + } + if info.IsDir() || !info.Mode().IsRegular() { + // If this is a directory or not a regular file, just throw a not-exist error + // since anything calling this function should understand what that means. + return ufs.ErrNotExist + } + currentSize := info.Size() + + // Check that copying this file wouldn't put the server over its limit. + if err := fs.HasSpaceFor(currentSize); err != nil { + return err + } + + base := info.Name() + extension := fs.Ext(base) + baseName := strings.TrimSuffix(base, extension) + + newName, err := fs.findCopySuffix(dirfd, baseName, extension) + if err != nil { + return err + } + dst, err := fs.unixFS.OpenFileat(dirfd, newName, ufs.O_WRONLY|ufs.O_CREATE, info.Mode()) + if err != nil { + return err + } + defer dst.Close() + + // Do not use CopyBuffer here, it is wasteful as the file implements + // io.ReaderFrom, which causes it to not use the buffer anyways. + n, err := io.Copy(dst, io.LimitReader(source, currentSize)) + fs.unixFS.Add(n) + + if !fs.isTest { + if err := fs.unixFS.Lchownat(dirfd, newName, config.Get().System.User.Uid, config.Get().System.User.Gid); err != nil { + return err + } + } + // Return the error from io.Copy. + return err +} + +func (fs *Filesystem) Ext(n string) string { + + extension := filepath.Ext(n) + baseName := strings.TrimSuffix(n, extension) + + // Ensure that ".tar" is also counted as apart of the file extension. + // There might be a better way to handle this for other double file extensions, + // but this is a good workaround for now. + if strings.HasSuffix(baseName, ".tar") { + extension = ".tar" + extension + baseName = strings.TrimSuffix(baseName, ".tar") + } + return extension +} + +// TruncateRootDirectory removes _all_ files and directories from a server's +// data directory and resets the used disk space to zero. +func (fs *Filesystem) TruncateRootDirectory() error { + if err := os.RemoveAll(fs.Path()); err != nil { + return err + } + if err := os.Mkdir(fs.Path(), 0o755); err != nil { + return err + } + _ = fs.unixFS.Close() + unixFS, err := ufs.NewUnixFS(fs.Path(), config.UseOpenat2()) + if err != nil { + return err + } + var limit int64 + if fs.isTest { + limit = 0 + } else { + limit = fs.unixFS.Limit() + } + fs.unixFS = ufs.NewQuota(unixFS, limit) + return nil +} + +// Delete removes a file or folder from the system. Prevents the user from +// accidentally (or maliciously) removing their root server data directory. +func (fs *Filesystem) Delete(p string) error { + return fs.unixFS.RemoveAll(p) +} + +// SafeDeleteRecursively deletes a file or directory while respecting the denylist. +// For files, deletion is skipped if the file matches the denylist. For directories, +// it recursively deletes all non-denylisted files and subdirectories. Empty directories +// are removed automatically, but directories containing denylisted files are preserved. +func (fs *Filesystem) SafeDeleteRecursively(p string) error { + info, err := fs.unixFS.Lstat(p) + if err != nil { + return err + } + + // Check if this path (file or directory) is denylisted + if err := fs.IsIgnored(p); err != nil { + // Skip denylisted file or directory + return nil + } + + if !info.IsDir() { + return fs.Delete(p) + } + + entries, err := fs.ReadDir(p) + if err != nil { + return err + } + + hasRemainingFiles := false + + for _, e := range entries { + child := filepath.Join(p, e.Name()) + + if e.IsDir() { + if err := fs.SafeDeleteRecursively(child); err != nil { + return err + } + // Check if the directory still exists after recursive deletion + if _, statErr := fs.unixFS.Lstat(child); statErr == nil { + hasRemainingFiles = true + } + continue + } + + // File handling - check if ignored + if err := fs.IsIgnored(child); err != nil { + hasRemainingFiles = true + continue // skip denylisted + } + + if err := fs.Delete(child); err != nil { + return err + } + } + + // Only remove directory if no files remain + if !hasRemainingFiles { + return fs.unixFS.Remove(p) + } + + return nil +} + +//type fileOpener struct { +// fs *Filesystem +// busy uint +//} +// +//// Attempts to open a given file up to "attempts" number of times, using a backoff. If the file +//// cannot be opened because of a "text file busy" error, we will attempt until the number of attempts +//// has been exhaused, at which point we will abort with an error. +//func (fo *fileOpener) open(path string, flags int, perm ufs.FileMode) (ufs.File, error) { +// for { +// f, err := fo.fs.unixFS.OpenFile(path, flags, perm) +// +// // If there is an error because the text file is busy, go ahead and sleep for a few +// // hundred milliseconds and then try again up to three times before just returning the +// // error back to the caller. +// // +// // Based on code from: https://github.com/golang/go/issues/22220#issuecomment-336458122 +// if err != nil && fo.busy < 3 && strings.Contains(err.Error(), "text file busy") { +// time.Sleep(100 * time.Millisecond << fo.busy) +// fo.busy++ +// continue +// } +// +// return f, err +// } +//} + +// ListDirectory lists the contents of a given directory and returns stat +// information about each file and folder within it. +func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { + // Read entries from the path on the filesystem, using the mapped reader, so + // we can map the DirEntry slice into a Stat slice with mimetype information. + out, err := ufs.ReadDirMap(fs.unixFS.UnixFS, p, func(e ufs.DirEntry) (Stat, error) { + info, err := e.Info() + if err != nil { + return Stat{}, err + } + + var d string + if e.Type().IsDir() { + d = "inode/directory" + } else { + d = "application/octet-stream" + } + var m *mimetype.MIME + if e.Type().IsRegular() { + // TODO: I should probably find a better way to do this. + eO := e.(interface { + Open() (ufs.File, error) + }) + f, err := eO.Open() + if err != nil { + return Stat{}, err + } + m, err = mimetype.DetectReader(f) + if err != nil { + log.Error(err.Error()) + } + _ = f.Close() + } + + st := Stat{FileInfo: info, Mimetype: d} + if m != nil { + st.Mimetype = m.String() + } + return st, nil + }) + if err != nil { + return nil, err + } + + // Sort entries alphabetically. + slices.SortStableFunc(out, func(a, b Stat) int { + switch { + case a.Name() == b.Name(): + return 0 + case a.Name() > b.Name(): + return 1 + default: + return -1 + } + }) + + // Sort folders before other file types. + slices.SortStableFunc(out, func(a, b Stat) int { + switch { + case a.IsDir() && b.IsDir(): + return 0 + case a.IsDir(): + return -1 + default: + return 1 + } + }) + + return out, nil +} + +func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { + if fs.isTest { + return nil + } + return fs.unixFS.Chtimes(path, atime, mtime) +} diff --git a/server/filesystem/filesystem_test.go b/server/filesystem/filesystem_test.go new file mode 100644 index 0000000..f1ed073 --- /dev/null +++ b/server/filesystem/filesystem_test.go @@ -0,0 +1,711 @@ +package filesystem + +import ( + "bufio" + "bytes" + "errors" + "math" + "math/rand" + "os" + "path/filepath" + "testing" + "unicode/utf8" + + . "github.com/franela/goblin" + + "github.com/pelican/wings/internal/ufs" + + "github.com/pelican/wings/config" +) + +func NewFs() (*Filesystem, *rootFs) { + config.Set(&config.Configuration{ + AuthenticationToken: "abc", + System: config.SystemConfiguration{ + RootDirectory: "/server", + DiskCheckInterval: 150, + }, + }) + + tmpDir, err := os.MkdirTemp(os.TempDir(), "pelican") + if err != nil { + panic(err) + return nil, nil + } + // Resolve symlinks in tmpDir so tests work on macOS where /var -> /private/var. + if resolved, err := filepath.EvalSymlinks(tmpDir); err == nil { + tmpDir = resolved + } + + rfs := rootFs{root: tmpDir} + + p := filepath.Join(tmpDir, "server") + if err := os.Mkdir(p, 0o755); err != nil { + panic(err) + return nil, nil + } + + fs, _ := New(p, 0, []string{}) + fs.isTest = true + if err := fs.TruncateRootDirectory(); err != nil { + panic(err) + return nil, nil + } + + return fs, &rfs +} + +type rootFs struct { + root string +} + +func getFileContent(file ufs.File) string { + var w bytes.Buffer + if _, err := bufio.NewReader(file).WriteTo(&w); err != nil { + panic(err) + } + return w.String() +} + +func (rfs *rootFs) CreateServerFile(p string, c []byte) error { + f, err := os.Create(filepath.Join(rfs.root, "server", p)) + + if err == nil { + _, _ = f.Write(c) + _ = f.Close() + } + + return err +} + +func (rfs *rootFs) CreateServerFileFromString(p string, c string) error { + return rfs.CreateServerFile(p, []byte(c)) +} + +func (rfs *rootFs) StatServerFile(p string) (os.FileInfo, error) { + return os.Stat(filepath.Join(rfs.root, "server", p)) +} + +func TestFilesystem_Openfile(t *testing.T) { + g := Goblin(t) + fs, rfs := NewFs() + + g.Describe("File", func() { + g.It("returns custom error when file does not exist", func() { + _, _, err := fs.File("foo/bar.txt") + + g.Assert(err).IsNotNil() + // TODO + //g.Assert(IsErrorCode(err, ErrNotExist)).IsTrue() + }) + + g.It("returns file stat information", func() { + _ = rfs.CreateServerFile("foo.txt", []byte("hello world")) + + f, st, err := fs.File("foo.txt") + g.Assert(err).IsNil() + + g.Assert(st.Name()).Equal("foo.txt") + g.Assert(f).IsNotNil() + _ = f.Close() + }) + + g.AfterEach(func() { + _ = fs.TruncateRootDirectory() + }) + }) +} + +func TestFilesystem_Touch(t *testing.T) { + g := Goblin(t) + fs, _ := NewFs() + + g.Describe("Touch", func() { + g.It("enforces disk limits while writing to the returned handle", func() { + fs.SetDiskLimit(10) + + f, err := fs.Touch("quota.txt", ufs.O_RDWR|ufs.O_TRUNC) + g.Assert(err).IsNil() + defer f.Close() + + n, err := f.WriteAt([]byte("1234567890"), 0) + g.Assert(err).IsNil() + g.Assert(n).Equal(10) + g.Assert(fs.CachedUsage()).Equal(int64(10)) + + n, err = f.WriteAt([]byte("1"), 10) + g.Assert(err).IsNotNil() + g.Assert(n).Equal(0) + g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue() + g.Assert(fs.CachedUsage()).Equal(int64(10)) + }) + + g.It("enforces disk limits while sequentially writing to the returned handle", func() { + fs.SetDiskLimit(10) + + f, err := fs.Touch("quota.txt", ufs.O_RDWR|ufs.O_TRUNC) + g.Assert(err).IsNil() + defer f.Close() + + n, err := f.Write([]byte("1234567890")) + g.Assert(err).IsNil() + g.Assert(n).Equal(10) + + n, err = f.Write([]byte("1")) + g.Assert(err).IsNotNil() + g.Assert(n).Equal(0) + g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue() + g.Assert(fs.CachedUsage()).Equal(int64(10)) + }) + + g.It("updates disk usage when a truncated file is closed smaller", func() { + r := bytes.NewReader([]byte("1234567890")) + err := fs.Write("quota.txt", r, r.Size(), 0o644) + g.Assert(err).IsNil() + g.Assert(fs.CachedUsage()).Equal(int64(10)) + + f, err := fs.Touch("quota.txt", ufs.O_RDWR|ufs.O_TRUNC) + g.Assert(err).IsNil() + + n, err := f.WriteAt([]byte("1234"), 0) + g.Assert(err).IsNil() + g.Assert(n).Equal(4) + + err = f.Close() + g.Assert(err).IsNil() + g.Assert(fs.CachedUsage()).Equal(int64(4)) + }) + + g.It("does not reset disk usage after a failed huge-offset write", func() { + const usage = int64(5 * 1024 * 1024) + fs.SetDiskLimit(10 * 1024 * 1024) + fs.unixFS.SetUsage(usage) + + f, err := fs.Touch("quota.txt", ufs.O_RDWR|ufs.O_TRUNC) + g.Assert(err).IsNil() + + n, err := f.WriteAt([]byte("x"), math.MaxInt64) + g.Assert(err).IsNotNil() + g.Assert(n).Equal(0) + + err = f.Close() + g.Assert(err).IsNil() + g.Assert(fs.CachedUsage()).Equal(usage) + }) + + g.AfterEach(func() { + _ = fs.TruncateRootDirectory() + }) + }) +} + +func TestFilesystem_Writefile(t *testing.T) { + g := Goblin(t) + fs, rfs := NewFs() + + g.Describe("Open and WriteFile", func() { + buf := &bytes.Buffer{} + + // Test that a file can be written to the disk and that the disk space used as a result + // is updated correctly in the end. + g.It("can create a new file", func() { + r := bytes.NewReader([]byte("test file content")) + + g.Assert(fs.CachedUsage()).Equal(int64(0)) + + err := fs.Write("test.txt", r, r.Size(), 0o644) + g.Assert(err).IsNil() + + f, _, err := fs.File("test.txt") + g.Assert(err).IsNil() + defer f.Close() + g.Assert(getFileContent(f)).Equal("test file content") + g.Assert(fs.CachedUsage()).Equal(r.Size()) + }) + + g.It("can create a new file inside a nested directory with leading slash", func() { + r := bytes.NewReader([]byte("test file content")) + + err := fs.Write("/some/nested/test.txt", r, r.Size(), 0o644) + g.Assert(err).IsNil() + + f, _, err := fs.File("/some/nested/test.txt") + g.Assert(err).IsNil() + defer f.Close() + g.Assert(getFileContent(f)).Equal("test file content") + }) + + g.It("can create a new file inside a nested directory without a trailing slash", func() { + r := bytes.NewReader([]byte("test file content")) + + err := fs.Write("some/../foo/bar/test.txt", r, r.Size(), 0o644) + g.Assert(err).IsNil() + + f, _, err := fs.File("foo/bar/test.txt") + g.Assert(err).IsNil() + defer f.Close() + g.Assert(getFileContent(f)).Equal("test file content") + }) + + g.It("cannot create a file outside the root directory", func() { + r := bytes.NewReader([]byte("test file content")) + + err := fs.Write("/some/../foo/../../test.txt", r, r.Size(), 0o644) + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("cannot write a file that exceeds the disk limits", func() { + fs.SetDiskLimit(1024) + + b := make([]byte, 1025) + _, err := rand.Read(b) + g.Assert(err).IsNil() + g.Assert(len(b)).Equal(1025) + + r := bytes.NewReader(b) + err = fs.Write("test.txt", r, int64(len(b)), 0o644) + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue() + }) + + g.It("cannot write a file whose claimed size overflows the quota check", func() { + fs.SetDiskLimit(1024) + fs.unixFS.SetUsage(1) + + r := bytes.NewReader([]byte("small body")) + err := fs.Write("overflow.txt", r, math.MaxInt64, 0o644) + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue() + + _, err = rfs.StatServerFile("overflow.txt") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue("err is not os.ErrNotExist") + }) + + g.It("truncates the file when writing new contents", func() { + r := bytes.NewReader([]byte("original data")) + err := fs.Write("test.txt", r, r.Size(), 0o644) + g.Assert(err).IsNil() + + r = bytes.NewReader([]byte("new data")) + err = fs.Write("test.txt", r, r.Size(), 0o644) + g.Assert(err).IsNil() + + f, _, err := fs.File("test.txt") + g.Assert(err).IsNil() + defer f.Close() + g.Assert(getFileContent(f)).Equal("new data") + }) + + g.AfterEach(func() { + buf.Truncate(0) + _ = fs.TruncateRootDirectory() + }) + }) +} + +func TestFilesystem_CreateDirectory(t *testing.T) { + g := Goblin(t) + fs, rfs := NewFs() + + g.Describe("CreateDirectory", func() { + g.It("should create missing directories automatically", func() { + err := fs.CreateDirectory("test", "foo/bar/baz") + g.Assert(err).IsNil() + + st, err := rfs.StatServerFile("foo/bar/baz/test") + g.Assert(err).IsNil() + g.Assert(st.IsDir()).IsTrue() + g.Assert(st.Name()).Equal("test") + }) + + g.It("should work with leading and trailing slashes", func() { + err := fs.CreateDirectory("test", "/foozie/barzie/bazzy/") + g.Assert(err).IsNil() + + st, err := rfs.StatServerFile("foozie/barzie/bazzy/test") + g.Assert(err).IsNil() + g.Assert(st.IsDir()).IsTrue() + g.Assert(st.Name()).Equal("test") + }) + + g.It("should not allow the creation of directories outside the root", func() { + err := fs.CreateDirectory("test", "e/../../something") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("should not increment the disk usage", func() { + err := fs.CreateDirectory("test", "/") + g.Assert(err).IsNil() + g.Assert(fs.CachedUsage()).Equal(int64(0)) + }) + + g.AfterEach(func() { + _ = fs.TruncateRootDirectory() + }) + }) +} + +func TestFilesystem_Rename(t *testing.T) { + g := Goblin(t) + fs, rfs := NewFs() + + g.Describe("Rename", func() { + g.BeforeEach(func() { + if err := rfs.CreateServerFileFromString("source.txt", "text content"); err != nil { + panic(err) + } + }) + + g.It("returns an error if the target already exists", func() { + err := rfs.CreateServerFileFromString("target.txt", "taget content") + g.Assert(err).IsNil() + + err = fs.Rename("source.txt", "target.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrExist)).IsTrue("err is not ErrExist") + }) + + g.It("returns an error if the final destination is the root directory", func() { + err := fs.Rename("source.txt", "/") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("returns an error if the source destination is the root directory", func() { + err := fs.Rename("/", "target.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("does not allow renaming to a location outside the root", func() { + err := fs.Rename("source.txt", "../target.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("does not allow renaming from a location outside the root", func() { + err := rfs.CreateServerFileFromString("/../ext-source.txt", "taget content") + + err = fs.Rename("/../ext-source.txt", "target.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("allows a file to be renamed", func() { + err := fs.Rename("source.txt", "target.txt") + g.Assert(err).IsNil() + + _, err = rfs.StatServerFile("source.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + + st, err := rfs.StatServerFile("target.txt") + g.Assert(err).IsNil() + g.Assert(st.Name()).Equal("target.txt") + g.Assert(st.Size()).IsNotZero() + }) + + g.It("allows a folder to be renamed", func() { + err := os.Mkdir(filepath.Join(rfs.root, "/server/source_dir"), 0o755) + g.Assert(err).IsNil() + + err = fs.Rename("source_dir", "target_dir") + g.Assert(err).IsNil() + + _, err = rfs.StatServerFile("source_dir") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + + st, err := rfs.StatServerFile("target_dir") + g.Assert(err).IsNil() + g.Assert(st.IsDir()).IsTrue() + }) + + g.It("returns an error if the source does not exist", func() { + err := fs.Rename("missing.txt", "target.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + }) + + g.It("creates directories if they are missing", func() { + err := fs.Rename("source.txt", "nested/folder/target.txt") + g.Assert(err).IsNil() + + st, err := rfs.StatServerFile("nested/folder/target.txt") + g.Assert(err).IsNil() + g.Assert(st.Name()).Equal("target.txt") + }) + + g.AfterEach(func() { + _ = fs.TruncateRootDirectory() + }) + }) +} + +func TestFilesystem_Copy(t *testing.T) { + g := Goblin(t) + fs, rfs := NewFs() + + g.Describe("Copy", func() { + g.BeforeEach(func() { + if err := rfs.CreateServerFileFromString("source.txt", "text content"); err != nil { + panic(err) + } + + fs.unixFS.SetUsage(int64(utf8.RuneCountInString("test content"))) + }) + + g.It("should return an error if the source does not exist", func() { + err := fs.Copy("foo.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + }) + + g.It("should return an error if the source is outside the root", func() { + err := rfs.CreateServerFileFromString("/../ext-source.txt", "text content") + + err = fs.Copy("../ext-source.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("should return an error if the source directory is outside the root", func() { + err := os.MkdirAll(filepath.Join(rfs.root, "nested/in/dir"), 0o755) + g.Assert(err).IsNil() + + err = rfs.CreateServerFileFromString("/../nested/in/dir/ext-source.txt", "external content") + g.Assert(err).IsNil() + + err = fs.Copy("../nested/in/dir/ext-source.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + + err = fs.Copy("nested/in/../../../nested/in/dir/ext-source.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("should return an error if the source is a directory", func() { + err := os.Mkdir(filepath.Join(rfs.root, "server/dir"), 0o755) + g.Assert(err).IsNil() + + err = fs.Copy("dir") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + }) + + g.It("should return an error if there is not space to copy the file", func() { + fs.SetDiskLimit(2) + + err := fs.Copy("source.txt") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue("err is not ErrCodeDiskSpace") + }) + + g.It("should create a copy of the file and increment the disk used", func() { + err := fs.Copy("source.txt") + g.Assert(err).IsNil() + + _, err = rfs.StatServerFile("source.txt") + g.Assert(err).IsNil() + + _, err = rfs.StatServerFile("source copy.txt") + g.Assert(err).IsNil() + }) + + g.It("should create a copy of the file with a suffix if a copy already exists", func() { + err := fs.Copy("source.txt") + g.Assert(err).IsNil() + + err = fs.Copy("source.txt") + g.Assert(err).IsNil() + + r := []string{"source.txt", "source copy.txt", "source copy 1.txt"} + + for _, name := range r { + _, err = rfs.StatServerFile(name) + g.Assert(err).IsNil() + } + + g.Assert(fs.CachedUsage()).Equal(int64(utf8.RuneCountInString("test content")) * 3) + }) + + g.It("should create a copy inside of a directory", func() { + err := os.MkdirAll(filepath.Join(rfs.root, "/server/nested/in/dir"), 0o755) + g.Assert(err).IsNil() + + err = rfs.CreateServerFileFromString("nested/in/dir/source.txt", "test content") + g.Assert(err).IsNil() + + err = fs.Copy("nested/in/dir/source.txt") + g.Assert(err).IsNil() + + _, err = rfs.StatServerFile("nested/in/dir/source.txt") + g.Assert(err).IsNil() + + _, err = rfs.StatServerFile("nested/in/dir/source copy.txt") + g.Assert(err).IsNil() + }) + + g.AfterEach(func() { + _ = fs.TruncateRootDirectory() + }) + }) +} + +func TestFilesystem_Delete(t *testing.T) { + g := Goblin(t) + fs, rfs := NewFs() + + g.Describe("Delete", func() { + g.BeforeEach(func() { + if err := rfs.CreateServerFileFromString("source.txt", "test content"); err != nil { + panic(err) + } + + fs.unixFS.SetUsage(int64(utf8.RuneCountInString("test content"))) + }) + + g.It("does not delete files outside the root directory", func() { + err := rfs.CreateServerFileFromString("/../ext-source.txt", "external content") + + err = fs.Delete("../ext-source.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("does not allow the deletion of the root directory", func() { + err := fs.Delete("/") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.It("does not return an error if the target does not exist", func() { + err := fs.Delete("missing.txt") + g.Assert(err).IsNil() + + st, err := rfs.StatServerFile("source.txt") + g.Assert(err).IsNil() + g.Assert(st.Name()).Equal("source.txt") + }) + + g.It("deletes files and subtracts their size from the disk usage", func() { + err := fs.Delete("source.txt") + g.Assert(err).IsNil() + + _, err = rfs.StatServerFile("source.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + + g.Assert(fs.CachedUsage()).Equal(int64(0)) + }) + + g.It("deletes all items inside a directory if the directory is deleted", func() { + sources := []string{ + "foo/source.txt", + "foo/bar/source.txt", + "foo/bar/baz/source.txt", + } + + err := os.MkdirAll(filepath.Join(rfs.root, "/server/foo/bar/baz"), 0o755) + g.Assert(err).IsNil() + + for _, s := range sources { + err = rfs.CreateServerFileFromString(s, "test content") + g.Assert(err).IsNil() + } + + fs.unixFS.SetUsage(int64(utf8.RuneCountInString("test content") * 3)) + + err = fs.Delete("foo") + g.Assert(err).IsNil() + g.Assert(fs.unixFS.Usage()).Equal(int64(0)) + + for _, s := range sources { + _, err = rfs.StatServerFile(s) + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + } + }) + + g.It("deletes a symlink but not it's target within the root directory", func() { + // Symlink to a file inside the root directory. + err := os.Symlink(filepath.Join(rfs.root, "server/source.txt"), filepath.Join(rfs.root, "server/symlink.txt")) + g.Assert(err).IsNil() + + // Delete the symlink itself. + err = fs.Delete("symlink.txt") + g.Assert(err).IsNil() + + // Ensure the symlink was deleted. + _, err = os.Lstat(filepath.Join(rfs.root, "server/symlink.txt")) + g.Assert(err).IsNotNil() + + // Ensure the symlink target still exists. + _, err = os.Lstat(filepath.Join(rfs.root, "server/source.txt")) + g.Assert(err).IsNil() + }) + + g.It("does not delete files symlinked outside of the root directory", func() { + // Create a file outside the root directory. + err := rfs.CreateServerFileFromString("/../source.txt", "test content") + g.Assert(err).IsNil() + + // Create a symlink to the file outside the root directory. + err = os.Symlink(filepath.Join(rfs.root, "source.txt"), filepath.Join(rfs.root, "/server/symlink.txt")) + g.Assert(err).IsNil() + + // Delete the symlink. (This should pass as we will delete the symlink itself, not it's target) + err = fs.Delete("symlink.txt") + g.Assert(err).IsNil() + + // Ensure the file outside the root directory still exists. + _, err = os.Lstat(filepath.Join(rfs.root, "source.txt")) + g.Assert(err).IsNil() + }) + + g.It("does not delete files symlinked through a directory outside of the root directory", func() { + // Create a directory outside the root directory. + err := os.Mkdir(filepath.Join(rfs.root, "foo"), 0o755) + g.Assert(err).IsNil() + + // Create a file inside the directory that is outside the root. + err = rfs.CreateServerFileFromString("/../foo/source.txt", "test content") + g.Assert(err).IsNil() + + // Symlink the directory that is outside the root to a file inside the root. + err = os.Symlink(filepath.Join(rfs.root, "foo"), filepath.Join(rfs.root, "server/symlink")) + g.Assert(err).IsNil() + + // Delete a file inside the symlinked directory. + err = fs.Delete("symlink/source.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + + // Ensure the file outside the root directory still exists. + _, err = os.Lstat(filepath.Join(rfs.root, "foo/source.txt")) + g.Assert(err).IsNil() + }) + + g.It("returns an error when trying to delete a non-existent file symlinked through a directory outside of the root directory", func() { + // Create a directory outside the root directory. + err := os.Mkdir(filepath.Join(rfs.root, "foo2"), 0o755) + g.Assert(err).IsNil() + + // Symlink the directory that is outside the root to a file inside the root. + err = os.Symlink(filepath.Join(rfs.root, "foo2"), filepath.Join(rfs.root, "server/symlink")) + g.Assert(err).IsNil() + + // Delete a file inside the symlinked directory. + err = fs.Delete("symlink/source.txt") + g.Assert(err).IsNotNil() + g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + }) + + g.AfterEach(func() { + _ = fs.TruncateRootDirectory() + }) + }) +}