commit e1659ae47bfe0f67709e6d4d8811ef9c55b3d573 Author: upstream Date: Tue Sep 29 11:29:15 2026 +0200 Upstream rclone/rclone state of the touched files before a6b519a87e diff --git a/fs/walk/walk.go b/fs/walk/walk.go new file mode 100644 index 0000000..4cbef90 --- /dev/null +++ b/fs/walk/walk.go @@ -0,0 +1,653 @@ +// Package walk walks directories +package walk + +import ( + "context" + "errors" + "fmt" + "path" + "sort" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/dirtree" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/list" +) + +// ErrorSkipDir is used as a return value from Walk to indicate that the +// directory named in the call is to be skipped. It is not returned as +// an error by any function. +var ErrorSkipDir = errors.New("skip this directory") + +// ErrorCantListR is returned by WalkR if the underlying Fs isn't +// capable of doing a recursive listing. +var ErrorCantListR = errors.New("recursive directory listing not available") + +// Func is the type of the function called for directory +// visited by Walk. The path argument contains remote path to the directory. +// +// If there was a problem walking to directory named by path, the +// incoming error will describe the problem and the function can +// decide how to handle that error (and Walk will not descend into +// that directory). If an error is returned, processing stops. The +// sole exception is when the function returns the special value +// ErrorSkipDir. If the function returns ErrorSkipDir, Walk skips the +// directory's contents entirely. +type Func func(path string, entries fs.DirEntries, err error) error + +// Walk lists the directory. +// +// If includeAll is not set it will use the filters defined. +// +// If maxLevel is < 0 then it will recurse indefinitely, else it will +// only do maxLevel levels. +// +// It calls fn for each tranche of DirEntries read. +// +// Note that fn will not be called concurrently whereas the directory +// listing will proceed concurrently. +// +// Parent directories are always listed before their children. +// +// This is implemented by WalkR if Config.UseListR is true +// and f supports it and level > 1, or WalkN otherwise. +// +// If --files-from and --no-traverse is set then a DirTree will be +// constructed with just those files in and then walked with WalkR +// +// Note: this will flag filter-aware backends! +// +// NB (f, path) to be replaced by fs.Dir at some point +func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error { + ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) + ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List + if ci.NoTraverse && fi.HaveFilesFrom() { + return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject)) + } + // FIXME should this just be maxLevel < 0 - why the maxLevel > 1 + if (maxLevel < 0 || maxLevel > 1) && ci.UseListR && f.Features().ListR != nil { + return walkListR(ctx, f, path, includeAll, maxLevel, fn) + } + return walkListDirSorted(ctx, f, path, includeAll, maxLevel, fn) +} + +// ListType is uses to choose which combination of files or directories is requires +type ListType byte + +// Types of listing for ListR +const ( + ListObjects ListType = 1 << iota // list objects only + ListDirs // list dirs only + ListAll = ListObjects | ListDirs // list files and dirs +) + +// Objects returns true if the list type specifies objects +func (l ListType) Objects() bool { + return (l & ListObjects) != 0 +} + +// Dirs returns true if the list type specifies dirs +func (l ListType) Dirs() bool { + return (l & ListDirs) != 0 +} + +// Filter in (inplace) to only contain the type of list entry required +func (l ListType) Filter(in *fs.DirEntries) { + if l == ListAll { + return + } + out := (*in)[:0] + for _, entry := range *in { + switch entry.(type) { + case fs.Object: + if l.Objects() { + out = append(out, entry) + } + case fs.Directory: + if l.Dirs() { + out = append(out, entry) + } + default: + fs.Errorf(nil, "Unknown object type %T", entry) + } + } + *in = out +} + +// ListR lists the directory recursively. +// +// If includeAll is not set it will use the filters defined. +// +// If maxLevel is < 0 then it will recurse indefinitely, else it will +// only do maxLevel levels. +// +// If synthesizeDirs is set then for bucket-based remotes it will +// synthesize directories from the file structure. This uses extra +// memory so don't set this if you don't need directories, likewise do +// set this if you are interested in directories. +// +// It calls fn for each tranche of DirEntries read. Note that these +// don't necessarily represent a directory +// +// Note that fn will not be called concurrently whereas the directory +// listing will proceed concurrently. +// +// Directories are not listed in any particular order so you can't +// rely on parents coming before children or alphabetical ordering +// +// This is implemented by using ListR on the backend if possible and +// efficient, otherwise by Walk. +// +// Note: this will flag filter-aware backends +// +// NB (f, path) to be replaced by fs.Dir at some point +func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error { + fi := filter.GetConfig(ctx) + // FIXME disable this with --no-fast-list ??? `--disable ListR` will do it... + doListR := f.Features().ListR + + // Can't use ListR if... + if doListR == nil || // ...no ListR + fi.HaveFilesFrom() || // ...using --files-from + maxLevel >= 0 || // ...using bounded recursion + len(fi.Opt.ExcludeFile) > 0 || // ...using --exclude-file + fi.UsesDirectoryFilters() { // ...using any directory filters + return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn) + } + ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List + return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased) +} + +// listRwalk walks the file tree for ListR using Walk +// Note: this will flag filter-aware backends (via Walk) +func listRwalk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error { + var listErr error + walkErr := Walk(ctx, f, path, includeAll, maxLevel, func(path string, entries fs.DirEntries, err error) error { + // Carry on listing but return the error at the end + if err != nil { + listErr = err + err = fs.CountError(ctx, err) + fs.Errorf(path, "error listing: %v", err) + return nil + } + listType.Filter(&entries) + return fn(entries) + }) + if listErr != nil { + return listErr + } + return walkErr +} + +// dirMap keeps track of directories made for bucket-based remotes. +// true => directory has been sent +// false => directory has been seen but not sent +type dirMap struct { + mu sync.Mutex + m map[string]bool + root string +} + +// make a new dirMap +func newDirMap(root string) *dirMap { + return &dirMap{ + m: make(map[string]bool), + root: root, + } +} + +// add adds a directory and parents with sent +func (dm *dirMap) add(dir string, sent bool) { + for { + if dir == dm.root || dir == "" { + return + } + currentSent, found := dm.m[dir] + if found { + // If it has been sent already then nothing more to do + if currentSent { + return + } + // If not sent already don't override + if !sent { + return + } + // currentSent == false && sent == true so needs overriding + } + dm.m[dir] = sent + // Add parents in as unsent + dir = parentDir(dir) + sent = false + } +} + +// parentDir finds the parent directory of path +func parentDir(entryPath string) string { + dirPath := path.Dir(entryPath) + if dirPath == "." { + dirPath = "" + } + return dirPath +} + +// add all the directories in entries and their parents to the dirMap +func (dm *dirMap) addEntries(entries fs.DirEntries) error { + dm.mu.Lock() + defer dm.mu.Unlock() + for _, entry := range entries { + switch x := entry.(type) { + case fs.Object: + dm.add(parentDir(x.Remote()), false) + case fs.Directory: + dm.add(x.Remote(), true) + default: + return fmt.Errorf("unknown object type %T", entry) + } + } + return nil +} + +// send any missing parents to fn +func (dm *dirMap) sendEntries(fn fs.ListRCallback) (err error) { + // Count the strings first so we allocate the minimum memory + n := 0 + for _, sent := range dm.m { + if !sent { + n++ + } + } + if n == 0 { + return nil + } + dirs := make([]string, 0, n) + // Fill the dirs up then sort it + for dir, sent := range dm.m { + if !sent { + dirs = append(dirs, dir) + } + } + sort.Strings(dirs) + // Now convert to bulkier Dir in batches and send + now := time.Now() + list := list.NewHelper(fn) + for _, dir := range dirs { + err = list.Add(fs.NewDir(dir, now)) + if err != nil { + return err + } + } + return list.Flush() +} + +// listR walks the file tree using ListR +func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListRCallback, doListR fs.ListRFn, synthesizeDirs bool) error { + fi := filter.GetConfig(ctx) + includeDirectory := fi.IncludeDirectory(ctx, f) + if !includeAll { + includeAll = fi.InActive() + } + var dm *dirMap + if synthesizeDirs { + dm = newDirMap(path) + } + var mu sync.Mutex + err := doListR(ctx, path, func(entries fs.DirEntries) (err error) { + accounting.Stats(ctx).Listed(int64(len(entries))) + if synthesizeDirs { + err = dm.addEntries(entries) + if err != nil { + return err + } + } + listType.Filter(&entries) + entries = list.RemoveEscaping(entries) + if !includeAll { + filteredEntries := entries[:0] + for _, entry := range entries { + var include bool + switch x := entry.(type) { + case fs.Object: + include = fi.IncludeObject(ctx, x) + case fs.Directory: + include, err = includeDirectory(x.Remote()) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown object type %T", entry) + } + if include { + filteredEntries = append(filteredEntries, entry) + } + } + entries = filteredEntries + } + mu.Lock() + defer mu.Unlock() + return fn(entries) + }) + if err != nil { + return err + } + if synthesizeDirs { + err = dm.sendEntries(fn) + if err != nil { + return err + } + } + return nil +} + +// walkListDirSorted lists the directory. +// +// It implements Walk using non recursive directory listing. +func walkListDirSorted(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error { + return walk(ctx, f, path, includeAll, maxLevel, fn, list.DirSorted) +} + +// walkListR lists the directory. +// +// It implements Walk using recursive directory listing if +// available, or returns ErrorCantListR if not. +func walkListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error { + listR := f.Features().ListR + if listR == nil { + return ErrorCantListR + } + return walkR(ctx, f, path, includeAll, maxLevel, fn, listR) +} + +type listDirFunc func(ctx context.Context, fs fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) + +func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listDir listDirFunc) error { + var ( + wg sync.WaitGroup // sync closing of go routines + traversing sync.WaitGroup // running directory traversals + doClose sync.Once // close the channel once + mu sync.Mutex // stop fn being called concurrently + ci = fs.GetConfig(ctx) // current config + ) + // listJob describe a directory listing that needs to be done + type listJob struct { + remote string + depth int + } + + in := make(chan listJob, ci.Checkers) + errs := make(chan error, 1) + quit := make(chan struct{}) + closeQuit := func() { + doClose.Do(func() { + close(quit) + go func() { + for range in { + traversing.Done() + } + }() + }) + } + for range ci.Checkers { + wg.Go(func() { + for { + select { + case <-ctx.Done(): + closeQuit() + select { + case errs <- ctx.Err(): + default: + } + return + case job, ok := <-in: + if !ok { + return + } + entries, err := listDir(ctx, f, includeAll, job.remote) + var jobs []listJob + if err == nil && job.depth != 0 { + entries.ForDir(func(dir fs.Directory) { + // Recurse for the directory + jobs = append(jobs, listJob{ + remote: dir.Remote(), + depth: job.depth - 1, + }) + }) + } + mu.Lock() + err = fn(job.remote, entries, err) + mu.Unlock() + // NB once we have passed entries to fn we mustn't touch it again + if err != nil && err != ErrorSkipDir { + traversing.Done() + err = fs.CountError(ctx, err) + fs.Errorf(job.remote, "error listing: %v", err) + closeQuit() + // Send error to error channel if space + select { + case errs <- err: + default: + } + continue + } + if err == nil && len(jobs) > 0 { + traversing.Add(len(jobs)) + go func() { + // Now we have traversed this directory, send these + // jobs off for traversal in the background + for _, newJob := range jobs { + in <- newJob + } + }() + } + traversing.Done() + case <-quit: + return + } + } + }) + } + // Start the process + traversing.Add(1) + in <- listJob{ + remote: path, + depth: maxLevel - 1, + } + traversing.Wait() + close(in) + wg.Wait() + close(errs) + // return the first error returned or nil + return <-errs +} + +func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll bool, maxLevel int, listR fs.ListRFn) (dirtree.DirTree, error) { + fi := filter.GetConfig(ctx) + dirs := dirtree.New() + // Entries can come in arbitrary order. We use toPrune to keep + // all directories to exclude later. + toPrune := make(map[string]bool) + includeDirectory := fi.IncludeDirectory(ctx, f) + var mu sync.Mutex + err := listR(ctx, startPath, func(entries fs.DirEntries) error { + accounting.Stats(ctx).Listed(int64(len(entries))) + entries = list.RemoveEscaping(entries) + mu.Lock() + defer mu.Unlock() + for _, entry := range entries { + slashes := strings.Count(entry.Remote(), "/") + excluded := true + switch x := entry.(type) { + case fs.Object: + // Make sure we don't delete excluded files if not required + if includeAll || fi.IncludeObject(ctx, x) { + if maxLevel < 0 || slashes <= maxLevel-1 { + dirs.Add(x) + excluded = false + } + } + // Make sure we include any parent directories of excluded objects + if excluded { + dirPath := parentDir(x.Remote()) + slashes-- + if maxLevel >= 0 { + for ; slashes > maxLevel-1; slashes-- { + dirPath = parentDir(dirPath) + } + } + inc, err := includeDirectory(dirPath) + if err != nil { + return err + } + if inc || includeAll { + // If the directory doesn't exist already, create it + _, obj := dirs.Find(dirPath) + if obj == nil { + dirs.AddDir(fs.NewDir(dirPath, time.Now())) + } + } + } + // Check if we need to prune a directory later. + if !includeAll && len(fi.Opt.ExcludeFile) > 0 { + basename := path.Base(x.Remote()) + for _, excludeFile := range fi.Opt.ExcludeFile { + if basename == excludeFile { + excludeDir := parentDir(x.Remote()) + toPrune[excludeDir] = true + } + } + } + case fs.Directory: + inc, err := includeDirectory(x.Remote()) + if err != nil { + return err + } + if includeAll || inc { + if maxLevel < 0 || slashes <= maxLevel-1 { + if slashes == maxLevel-1 { + // Just add the object if at maxLevel + dirs.Add(x) + } else { + dirs.AddDir(x) + } + } + } + default: + return fmt.Errorf("unknown object type %T", entry) + } + } + return nil + }) + if err != nil { + return nil, err + } + dirs.CheckParents(startPath) + if len(dirs) == 0 { + dirs[startPath] = nil + } + err = dirs.Prune(toPrune) + if err != nil { + return nil, err + } + dirs.Sort() + return dirs, nil +} + +// Create a DirTree using List +func walkNDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listDir listDirFunc) (dirtree.DirTree, error) { + dirs := make(dirtree.DirTree) + fn := func(dirPath string, entries fs.DirEntries, err error) error { + if err == nil { + dirs[dirPath] = entries + } + return err + } + err := walk(ctx, f, path, includeAll, maxLevel, fn, listDir) + if err != nil { + return nil, err + } + return dirs, nil +} + +// NewDirTree returns a DirTree filled with the directory listing +// using the parameters supplied. +// +// If includeAll is not set it will use the filters defined. +// +// If maxLevel is < 0 then it will recurse indefinitely, else it will +// only do maxLevel levels. +// +// This is implemented by WalkR if f supports ListR and level > 1, or +// WalkN otherwise. +// +// If --files-from and --no-traverse is set then a DirTree will be +// constructed with just those files in. +// +// NB (f, path) to be replaced by fs.Dir at some point +func NewDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int) (dirtree.DirTree, error) { + ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) + // if --no-traverse and --files-from build DirTree just from files + if ci.NoTraverse && fi.HaveFilesFrom() { + return walkRDirTree(ctx, f, path, includeAll, maxLevel, fi.MakeListR(ctx, f.NewObject)) + } + // if have ListR; and recursing; and not using --files-from; then build a DirTree with ListR + if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && ListR != nil && !fi.HaveFilesFrom() { + return walkRDirTree(ctx, f, path, includeAll, maxLevel, ListR) + } + // otherwise just use List + return walkNDirTree(ctx, f, path, includeAll, maxLevel, list.DirSorted) +} + +func walkR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listR fs.ListRFn) error { + dirs, err := walkRDirTree(ctx, f, path, includeAll, maxLevel, listR) + if err != nil { + return err + } + skipping := false + skipPrefix := "" + emptyDir := fs.DirEntries{} + for _, dirPath := range dirs.Dirs() { + if skipping { + // Skip over directories as required + if strings.HasPrefix(dirPath, skipPrefix) { + continue + } + skipping = false + } + entries := dirs[dirPath] + if entries == nil { + entries = emptyDir + } + err = fn(dirPath, entries, nil) + if err == ErrorSkipDir { + skipping = true + skipPrefix = dirPath + if skipPrefix != "" { + skipPrefix += "/" + } + } else if err != nil { + return err + } + } + return nil +} + +// GetAll runs ListR getting all the results +func GetAll(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int) (objs []fs.Object, dirs []fs.Directory, err error) { + err = ListR(ctx, f, path, includeAll, maxLevel, ListAll, func(entries fs.DirEntries) error { + for _, entry := range entries { + switch x := entry.(type) { + case fs.Object: + objs = append(objs, x) + case fs.Directory: + dirs = append(dirs, x) + } + } + return nil + }) + return +} diff --git a/fs/walk/walk_test.go b/fs/walk/walk_test.go new file mode 100644 index 0000000..1906d16 --- /dev/null +++ b/fs/walk/walk_test.go @@ -0,0 +1,1081 @@ +package walk + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "sync" + "testing" + + "github.com/rclone/rclone/fs" + _ "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fstest/mockdir" + "github.com/rclone/rclone/fstest/mockfs" + "github.com/rclone/rclone/fstest/mockobject" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var errDirNotFound, errorBoom error + +func init() { + errDirNotFound = fserrors.FsError(fs.ErrorDirNotFound) + fserrors.Count(errDirNotFound) + errorBoom = fserrors.FsError(errors.New("boom")) + fserrors.Count(errorBoom) +} + +type ( + listResult struct { + entries fs.DirEntries + err error + } + + listResults map[string]listResult + + errorMap map[string]error + + listDirs struct { + mu sync.Mutex + t *testing.T + fs fs.Fs + includeAll bool + results listResults + walkResults listResults + walkErrors errorMap + finalError error + checkMaps bool + maxLevel int + } +) + +func newListDirs(t *testing.T, f fs.Fs, includeAll bool, results listResults, walkErrors errorMap, finalError error) *listDirs { + return &listDirs{ + t: t, + fs: f, + includeAll: includeAll, + results: results, + walkErrors: walkErrors, + walkResults: listResults{}, + finalError: finalError, + checkMaps: true, + maxLevel: -1, + } +} + +// NoCheckMaps marks the maps as to be ignored at the end +func (ls *listDirs) NoCheckMaps() *listDirs { + ls.checkMaps = false + return ls +} + +// SetLevel(1) turns off recursion +func (ls *listDirs) SetLevel(maxLevel int) *listDirs { + ls.maxLevel = maxLevel + return ls +} + +// ListDir returns the expected listing for the directory +func (ls *listDirs) ListDir(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) { + ls.mu.Lock() + defer ls.mu.Unlock() + assert.Equal(ls.t, ls.fs, f) + assert.Equal(ls.t, ls.includeAll, includeAll) + + // Fetch results for this path + result, ok := ls.results[dir] + if !ok { + ls.t.Errorf("Unexpected list of %q", dir) + return nil, errors.New("unexpected list") + } + delete(ls.results, dir) + + // Put expected results for call of WalkFn + ls.walkResults[dir] = result + + return result.entries, result.err +} + +// ListR returns the expected listing for the directory using ListR +func (ls *listDirs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { + ls.mu.Lock() + defer ls.mu.Unlock() + + var errorReturn error + for dirPath, result := range ls.results { + // Put expected results for call of WalkFn + // Note that we don't call the function at all if we got an error + if result.err != nil { + errorReturn = result.err + } + if errorReturn == nil { + err = callback(result.entries) + require.NoError(ls.t, err) + ls.walkResults[dirPath] = result + } + } + ls.results = listResults{} + return errorReturn +} + +// IsFinished checks everything expected was used up +func (ls *listDirs) IsFinished() { + if ls.checkMaps { + assert.Equal(ls.t, errorMap{}, ls.walkErrors) + assert.Equal(ls.t, listResults{}, ls.results) + assert.Equal(ls.t, listResults{}, ls.walkResults) + } +} + +// WalkFn is called by the walk to test the expectations +func (ls *listDirs) WalkFn(dir string, entries fs.DirEntries, err error) error { + ls.mu.Lock() + defer ls.mu.Unlock() + // ls.t.Logf("WalkFn(%q, %v, %q)", dir, entries, err) + + // Fetch expected entries and err + result, ok := ls.walkResults[dir] + if !ok { + ls.t.Errorf("Unexpected walk of %q (result not found)", dir) + return errors.New("result not found") + } + delete(ls.walkResults, dir) + + // Check arguments are as expected + assert.Equal(ls.t, result.entries, entries) + assert.Equal(ls.t, result.err, err) + + // Fetch return value + returnErr, ok := ls.walkErrors[dir] + if !ok { + ls.t.Errorf("Unexpected walk of %q (error not found)", dir) + return errors.New("error not found") + } + delete(ls.walkErrors, dir) + + return returnErr +} + +// Walk does the walk and tests the expectations +func (ls *listDirs) Walk() { + err := walk(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListDir) + assert.True(ls.t, errors.Is(ls.finalError, err)) + ls.IsFinished() +} + +// WalkR does the walkR and tests the expectations +func (ls *listDirs) WalkR() { + err := walkR(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR) + assert.Equal(ls.t, ls.finalError, err) + if ls.finalError == nil { + ls.IsFinished() + } +} + +func testWalkEmpty(t *testing.T) *listDirs { + return newListDirs(t, nil, false, + listResults{ + "": {entries: fs.DirEntries{}, err: nil}, + }, + errorMap{ + "": nil, + }, + nil, + ) +} +func TestWalkEmpty(t *testing.T) { testWalkEmpty(t).Walk() } +func TestWalkREmpty(t *testing.T) { testWalkEmpty(t).WalkR() } + +func testWalkEmptySkip(t *testing.T) *listDirs { + return newListDirs(t, nil, true, + listResults{ + "": {entries: fs.DirEntries{}, err: nil}, + }, + errorMap{ + "": ErrorSkipDir, + }, + nil, + ) +} +func TestWalkEmptySkip(t *testing.T) { testWalkEmptySkip(t).Walk() } +func TestWalkREmptySkip(t *testing.T) { testWalkEmptySkip(t).WalkR() } + +func testWalkNotFound(t *testing.T) *listDirs { + return newListDirs(t, nil, true, + listResults{ + "": {err: errDirNotFound}, + }, + errorMap{ + "": errDirNotFound, + }, + errDirNotFound, + ) +} +func TestWalkNotFound(t *testing.T) { testWalkNotFound(t).Walk() } +func TestWalkRNotFound(t *testing.T) { testWalkNotFound(t).WalkR() } + +func TestWalkNotFoundMaskError(t *testing.T) { + // this doesn't work for WalkR + newListDirs(t, nil, true, + listResults{ + "": {err: errDirNotFound}, + }, + errorMap{ + "": nil, + }, + nil, + ).Walk() +} + +func TestWalkNotFoundSkipError(t *testing.T) { + // this doesn't work for WalkR + newListDirs(t, nil, true, + listResults{ + "": {err: errDirNotFound}, + }, + errorMap{ + "": ErrorSkipDir, + }, + nil, + ).Walk() +} + +func testWalkLevels(t *testing.T, maxLevel int) *listDirs { + da := mockdir.New("a") + oA := mockobject.Object("A") + db := mockdir.New("a/b") + oB := mockobject.Object("a/B") + dc := mockdir.New("a/b/c") + oC := mockobject.Object("a/b/C") + dd := mockdir.New("a/b/c/d") + oD := mockobject.Object("a/b/c/D") + return newListDirs(t, nil, false, + listResults{ + "": {entries: fs.DirEntries{oA, da}, err: nil}, + "a": {entries: fs.DirEntries{oB, db}, err: nil}, + "a/b": {entries: fs.DirEntries{oC, dc}, err: nil}, + "a/b/c": {entries: fs.DirEntries{oD, dd}, err: nil}, + "a/b/c/d": {entries: fs.DirEntries{}, err: nil}, + }, + errorMap{ + "": nil, + "a": nil, + "a/b": nil, + "a/b/c": nil, + "a/b/c/d": nil, + }, + nil, + ).SetLevel(maxLevel) +} +func TestWalkLevels(t *testing.T) { testWalkLevels(t, -1).Walk() } +func TestWalkRLevels(t *testing.T) { testWalkLevels(t, -1).WalkR() } +func TestWalkLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).Walk() } +func TestWalkRLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).WalkR() } + +func TestWalkNDirTree(t *testing.T) { + ls := testWalkLevels(t, -1) + entries, err := walkNDirTree(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.ListDir) + require.NoError(t, err) + assert.Equal(t, `/ + A + a/ +a/ + B + b/ +a/b/ + C + c/ +a/b/c/ + D + d/ +a/b/c/d/ +`, entries.String()) +} + +func testWalkLevelsNoRecursive(t *testing.T) *listDirs { + da := mockdir.New("a") + oA := mockobject.Object("A") + return newListDirs(t, nil, false, + listResults{ + "": {entries: fs.DirEntries{oA, da}, err: nil}, + }, + errorMap{ + "": nil, + }, + nil, + ).SetLevel(1) +} +func TestWalkLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).Walk() } +func TestWalkRLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).WalkR() } + +func testWalkLevels2(t *testing.T) *listDirs { + da := mockdir.New("a") + oA := mockobject.Object("A") + db := mockdir.New("a/b") + oB := mockobject.Object("a/B") + return newListDirs(t, nil, false, + listResults{ + "": {entries: fs.DirEntries{oA, da}, err: nil}, + "a": {entries: fs.DirEntries{oB, db}, err: nil}, + }, + errorMap{ + "": nil, + "a": nil, + }, + nil, + ).SetLevel(2) +} +func TestWalkLevels2(t *testing.T) { testWalkLevels2(t).Walk() } +func TestWalkRLevels2(t *testing.T) { testWalkLevels2(t).WalkR() } + +func testWalkSkip(t *testing.T) *listDirs { + da := mockdir.New("a") + db := mockdir.New("a/b") + dc := mockdir.New("a/b/c") + return newListDirs(t, nil, false, + listResults{ + "": {entries: fs.DirEntries{da}, err: nil}, + "a": {entries: fs.DirEntries{db}, err: nil}, + "a/b": {entries: fs.DirEntries{dc}, err: nil}, + }, + errorMap{ + "": nil, + "a": nil, + "a/b": ErrorSkipDir, + }, + nil, + ) +} +func TestWalkSkip(t *testing.T) { testWalkSkip(t).Walk() } +func TestWalkRSkip(t *testing.T) { testWalkSkip(t).WalkR() } + +func walkErrors(t *testing.T, expectedErr error) *listDirs { + lr := listResults{} + em := errorMap{} + de := make(fs.DirEntries, 10) + for i := range de { + path := string('0' + rune(i)) + de[i] = mockdir.New(path) + lr[path] = listResult{entries: nil, err: fs.ErrorDirNotFound} + em[path] = fs.ErrorDirNotFound + } + lr[""] = listResult{entries: de, err: nil} + em[""] = nil + return newListDirs(t, nil, true, + lr, + em, + expectedErr, + ).NoCheckMaps() +} + +func testWalkErrors(t *testing.T) *listDirs { + return walkErrors(t, errDirNotFound) +} + +func testWalkRErrors(t *testing.T) *listDirs { + return walkErrors(t, fs.ErrorDirNotFound) +} + +func TestWalkErrors(t *testing.T) { testWalkErrors(t).Walk() } +func TestWalkRErrors(t *testing.T) { testWalkRErrors(t).WalkR() } + +func makeTree(level int, terminalErrors bool) (listResults, errorMap) { + lr := listResults{} + em := errorMap{} + var fill func(path string, level int) + fill = func(path string, level int) { + de := fs.DirEntries{} + if level > 0 { + for _, a := range "0123456789" { + subPath := string(a) + if path != "" { + subPath = path + "/" + subPath + } + de = append(de, mockdir.New(subPath)) + fill(subPath, level-1) + } + } + lr[path] = listResult{entries: de, err: nil} + em[path] = nil + if level == 0 && terminalErrors { + em[path] = errorBoom + } + } + fill("", level) + return lr, em +} + +func testWalkMulti(t *testing.T) *listDirs { + lr, em := makeTree(3, false) + return newListDirs(t, nil, true, + lr, + em, + nil, + ) +} +func TestWalkMulti(t *testing.T) { testWalkMulti(t).Walk() } +func TestWalkRMulti(t *testing.T) { testWalkMulti(t).WalkR() } + +func testWalkMultiErrors(t *testing.T) *listDirs { + lr, em := makeTree(3, true) + return newListDirs(t, nil, true, + lr, + em, + errorBoom, + ).NoCheckMaps() +} +func TestWalkMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() } +func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() } + +// a very simple listRcallback function +func makeListRCallback(entries fs.DirEntries, err error) fs.ListRFn { + return func(ctx context.Context, dir string, callback fs.ListRCallback) error { + if err == nil { + err = callback(entries) + } + return err + } +} + +func TestWalkRDirTree(t *testing.T) { + for _, test := range []struct { + entries fs.DirEntries + want string + err error + root string + level int + exclude string + }{ + { + entries: fs.DirEntries{}, + want: "/\n", + level: -1, + }, + { + entries: fs.DirEntries{mockobject.Object("a")}, + want: `/ + a +`, + level: -1, + }, + { + entries: fs.DirEntries{mockobject.Object("a/b")}, + want: `/ + a/ +a/ + b +`, + level: -1, + }, + { + entries: fs.DirEntries{mockobject.Object("a/b/c/d")}, + want: `/ + a/ +a/ + b/ +a/b/ + c/ +a/b/c/ + d +`, + level: -1, + }, + { + entries: fs.DirEntries{mockobject.Object("a")}, + err: errorBoom, + level: -1, + }, + { + entries: fs.DirEntries{ + mockobject.Object("0/1/2/3"), + mockobject.Object("4/5/6/7"), + mockobject.Object("8/9/a/b"), + mockobject.Object("c/d/e/f"), + mockobject.Object("g/h/i/j"), + mockobject.Object("k/l/m/n"), + mockobject.Object("o/p/q/r"), + mockobject.Object("s/t/u/v"), + mockobject.Object("w/x/y/z"), + }, + want: `/ + 0/ + 4/ + 8/ + c/ + g/ + k/ + o/ + s/ + w/ +0/ + 1/ +0/1/ + 2/ +0/1/2/ + 3 +4/ + 5/ +4/5/ + 6/ +4/5/6/ + 7 +8/ + 9/ +8/9/ + a/ +8/9/a/ + b +c/ + d/ +c/d/ + e/ +c/d/e/ + f +g/ + h/ +g/h/ + i/ +g/h/i/ + j +k/ + l/ +k/l/ + m/ +k/l/m/ + n +o/ + p/ +o/p/ + q/ +o/p/q/ + r +s/ + t/ +s/t/ + u/ +s/t/u/ + v +w/ + x/ +w/x/ + y/ +w/x/y/ + z +`, + level: -1, + }, + { + entries: fs.DirEntries{ + mockobject.Object("a/b/c/d/e/f1"), + mockobject.Object("a/b/c/d/e/f2"), + mockobject.Object("a/b/c/d/e/f3"), + }, + want: `a/b/c/ + d/ +a/b/c/d/ + e/ +a/b/c/d/e/ + f1 + f2 + f3 +`, + root: "a/b/c", + level: -1, + }, + { + entries: fs.DirEntries{ + mockobject.Object("A"), + mockobject.Object("a/B"), + mockobject.Object("a/b/C"), + mockobject.Object("a/b/c/D"), + mockobject.Object("a/b/c/d/E"), + }, + want: `/ + A + a/ +a/ + B + b/ +a/b/ +`, + level: 2, + }, + { + entries: fs.DirEntries{ + mockobject.Object("a/b/c"), + mockobject.Object("a/b/c/d/e"), + }, + want: `/ + a/ +a/ + b/ +a/b/ +`, + level: 2, + }, + { + entries: fs.DirEntries{ + mockobject.Object("a/.bzEmpty"), + mockobject.Object("a/b1/.bzEmpty"), + mockobject.Object("a/b2/.bzEmpty"), + }, + want: `/ + a/ +a/ + .bzEmpty + b1/ + b2/ +a/b1/ + .bzEmpty +a/b2/ + .bzEmpty +`, + level: -1, + exclude: ""}, + { + entries: fs.DirEntries{ + mockobject.Object("a/.bzEmpty"), + mockobject.Object("a/b1/.bzEmpty"), + mockobject.Object("a/b2/.bzEmpty"), + }, + want: `/ + a/ +a/ + b1/ + b2/ +a/b1/ +a/b2/ +`, + level: -1, + exclude: ".bzEmpty", + }, + } { + ctx := context.Background() + if test.exclude != "" { + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + require.NoError(t, fi.Add(false, test.exclude)) + // Change the active filter + ctx = filter.ReplaceConfig(ctx, fi) + + } + r, err := walkRDirTree(ctx, nil, test.root, test.exclude == "", test.level, makeListRCallback(test.entries, test.err)) + what := fmt.Sprintf("%+v", test) + assert.Equal(t, test.err, err, what) + assert.Equal(t, test.want, r.String(), what) + } +} + +func TestWalkRDirTreeExclude(t *testing.T) { + ctx := context.Background() + fi := filter.GetConfig(ctx) + for _, test := range []struct { + entries fs.DirEntries + want string + err error + root string + level int + excludeFile string + includeAll bool + }{ + {fs.DirEntries{mockobject.Object("a"), mockobject.Object("ignore")}, "", nil, "", -1, "ignore", false}, + {fs.DirEntries{mockobject.Object("a")}, `/ + a +`, nil, "", -1, "ignore", false}, + {fs.DirEntries{ + mockobject.Object("a"), + mockobject.Object("b/b"), + mockobject.Object("b/.ignore"), + }, `/ + a +`, nil, "", -1, ".ignore", false}, + {fs.DirEntries{ + mockobject.Object("a"), + mockobject.Object("b/.ignore"), + mockobject.Object("b/b"), + }, `/ + a + b/ +b/ + .ignore + b +`, nil, "", -1, ".ignore", true}, + {fs.DirEntries{ + mockobject.Object("a"), + mockobject.Object("b/b"), + mockobject.Object("b/c/d/e"), + mockobject.Object("b/c/ign"), + mockobject.Object("b/c/x"), + }, `/ + a + b/ +b/ + b +`, nil, "", -1, "ign", false}, + {fs.DirEntries{ + mockobject.Object("a"), + mockobject.Object("b/b"), + mockobject.Object("b/c/d/e"), + mockobject.Object("b/c/ign"), + mockobject.Object("b/c/x"), + }, `/ + a + b/ +b/ + b + c/ +b/c/ + d/ + ign + x +b/c/d/ + e +`, nil, "", -1, "ign", true}, + } { + fi.Opt.ExcludeFile = []string{test.excludeFile} + r, err := walkRDirTree(context.Background(), nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err)) + assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test)) + assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test)) + } + // Set to default value, to avoid side effects + fi.Opt.ExcludeFile = nil +} + +func TestListType(t *testing.T) { + assert.Equal(t, true, ListObjects.Objects()) + assert.Equal(t, false, ListObjects.Dirs()) + assert.Equal(t, false, ListDirs.Objects()) + assert.Equal(t, true, ListDirs.Dirs()) + assert.Equal(t, true, ListAll.Objects()) + assert.Equal(t, true, ListAll.Dirs()) + + var ( + a = mockobject.Object("a") + b = mockobject.Object("b") + dir = mockdir.New("dir") + adir = mockobject.Object("dir/a") + dir2 = mockdir.New("dir2") + origEntries = fs.DirEntries{ + a, b, dir, adir, dir2, + } + dirEntries = fs.DirEntries{ + dir, dir2, + } + objEntries = fs.DirEntries{ + a, b, adir, + } + ) + copyOrigEntries := func() (out fs.DirEntries) { + out = make(fs.DirEntries, len(origEntries)) + copy(out, origEntries) + return out + } + + got := copyOrigEntries() + ListAll.Filter(&got) + assert.Equal(t, origEntries, got) + + got = copyOrigEntries() + ListObjects.Filter(&got) + assert.Equal(t, objEntries, got) + + got = copyOrigEntries() + ListDirs.Filter(&got) + assert.Equal(t, dirEntries, got) +} + +func TestListRConfinement(t *testing.T) { + ctx := context.Background() + f, err := mockfs.NewFs(ctx, "mock", "/", nil) + require.NoError(t, err) + objects := fs.DirEntries{ + mockobject.Object("ok"), + mockobject.Object(".."), + mockobject.Object("../escape"), + mockdir.New("../evildir"), + // leading-slash climbers: filterDir doesn't guard the ListR path, so + // these reach RemoveEscaping directly and must still be dropped. + mockobject.Object("/../slashescape"), + mockobject.Object("//../../slashescape2"), + mockobject.Object("dir/deep"), + } + var got []string + callback := func(entries fs.DirEntries) error { + for _, entry := range entries { + got = append(got, entry.Remote()) + } + return nil + } + doListR := func(ctx context.Context, dir string, callback fs.ListRCallback) error { + return callback(objects) + } + // includeAll = true exercises the unconditional confinement - the + // include/exclude filter block is skipped in this mode, so an escaping + // entry would otherwise pass straight through. + err = listR(ctx, f, "", true, ListAll, callback, doListR, false) + require.NoError(t, err) + require.Equal(t, []string{"ok", "dir/deep"}, got) +} + +func TestListR(t *testing.T) { + ctx := context.Background() + objects := fs.DirEntries{ + mockobject.Object("a"), + mockobject.Object("b"), + mockdir.New("dir"), + mockobject.Object("dir/a"), + mockobject.Object("dir/b"), + mockobject.Object("dir/c"), + } + f, err := mockfs.NewFs(ctx, "mock", "/", nil) + require.NoError(t, err) + var got []string + clearCallback := func() { + got = nil + } + callback := func(entries fs.DirEntries) error { + for _, entry := range entries { + got = append(got, entry.Remote()) + } + return nil + } + doListR := func(ctx context.Context, dir string, callback fs.ListRCallback) error { + var os fs.DirEntries + for _, o := range objects { + if dir == "" || strings.HasPrefix(o.Remote(), dir+"/") { + os = append(os, o) + } + } + return callback(os) + } + + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + require.NoError(t, fi.AddRule("+ b")) + require.NoError(t, fi.AddRule("- *")) + + // Change the active filter + ctx = filter.ReplaceConfig(ctx, fi) + + // Base case + clearCallback() + err = listR(ctx, f, "", true, ListAll, callback, doListR, false) + require.NoError(t, err) + require.Equal(t, []string{"a", "b", "dir", "dir/a", "dir/b", "dir/c"}, got) + + // Base case - with Objects + clearCallback() + err = listR(ctx, f, "", true, ListObjects, callback, doListR, false) + require.NoError(t, err) + require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/c"}, got) + + // Base case - with Dirs + clearCallback() + err = listR(ctx, f, "", true, ListDirs, callback, doListR, false) + require.NoError(t, err) + require.Equal(t, []string{"dir"}, got) + + // With filter + clearCallback() + err = listR(ctx, f, "", false, ListAll, callback, doListR, false) + require.NoError(t, err) + require.Equal(t, []string{"b", "dir", "dir/b"}, got) + + // With filter - with Objects + clearCallback() + err = listR(ctx, f, "", false, ListObjects, callback, doListR, false) + require.NoError(t, err) + require.Equal(t, []string{"b", "dir/b"}, got) + + // With filter - with Dir + clearCallback() + err = listR(ctx, f, "", false, ListDirs, callback, doListR, false) + require.NoError(t, err) + require.Equal(t, []string{"dir"}, got) + + // With filter and subdir + clearCallback() + err = listR(ctx, f, "dir", false, ListAll, callback, doListR, false) + require.NoError(t, err) + require.Equal(t, []string{"dir/b"}, got) + + // Now bucket-based + objects = fs.DirEntries{ + mockobject.Object("a"), + mockobject.Object("b"), + mockobject.Object("dir/a"), + mockobject.Object("dir/b"), + mockobject.Object("dir/subdir/c"), + mockdir.New("dir/subdir"), + } + + // Base case + clearCallback() + err = listR(ctx, f, "", true, ListAll, callback, doListR, true) + require.NoError(t, err) + require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/subdir/c", "dir/subdir", "dir"}, got) + + // With filter + clearCallback() + err = listR(ctx, f, "", false, ListAll, callback, doListR, true) + require.NoError(t, err) + require.Equal(t, []string{"b", "dir/b", "dir/subdir", "dir"}, got) + + // With filter and subdir + clearCallback() + err = listR(ctx, f, "dir", false, ListAll, callback, doListR, true) + require.NoError(t, err) + require.Equal(t, []string{"dir/b", "dir/subdir"}, got) + + // With filter and subdir - with Objects + clearCallback() + err = listR(ctx, f, "dir", false, ListObjects, callback, doListR, true) + require.NoError(t, err) + require.Equal(t, []string{"dir/b"}, got) + + // With filter and subdir - with Dirs + clearCallback() + err = listR(ctx, f, "dir", false, ListDirs, callback, doListR, true) + require.NoError(t, err) + require.Equal(t, []string{"dir/subdir"}, got) +} + +func TestDirMapAdd(t *testing.T) { + type add struct { + dir string + sent bool + } + for i, test := range []struct { + root string + in []add + want map[string]bool + }{ + { + root: "", + in: []add{ + {"", true}, + }, + want: map[string]bool{}, + }, + { + root: "", + in: []add{ + {"a/b/c", true}, + }, + want: map[string]bool{ + "a/b/c": true, + "a/b": false, + "a": false, + }, + }, + { + root: "", + in: []add{ + {"a/b/c", true}, + {"a/b", true}, + }, + want: map[string]bool{ + "a/b/c": true, + "a/b": true, + "a": false, + }, + }, + { + root: "", + in: []add{ + {"a/b", true}, + {"a/b/c", false}, + }, + want: map[string]bool{ + "a/b/c": false, + "a/b": true, + "a": false, + }, + }, + { + root: "root", + in: []add{ + {"root/a/b", true}, + {"root/a/b/c", false}, + }, + want: map[string]bool{ + "root/a/b/c": false, + "root/a/b": true, + "root/a": false, + }, + }, + } { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + dm := newDirMap(test.root) + for _, item := range test.in { + dm.add(item.dir, item.sent) + } + assert.Equal(t, test.want, dm.m) + }) + } +} + +func TestDirMapAddEntries(t *testing.T) { + dm := newDirMap("") + entries := fs.DirEntries{ + mockobject.Object("dir/a"), + mockobject.Object("dir/b"), + mockdir.New("dir"), + mockobject.Object("dir2/a"), + mockobject.Object("dir2/b"), + } + require.NoError(t, dm.addEntries(entries)) + assert.Equal(t, map[string]bool{"dir": true, "dir2": false}, dm.m) +} + +func TestDirMapSendEntries(t *testing.T) { + var got []string + clearCallback := func() { + got = nil + } + callback := func(entries fs.DirEntries) error { + for _, entry := range entries { + got = append(got, entry.Remote()) + } + return nil + } + + // general test + dm := newDirMap("") + entries := fs.DirEntries{ + mockobject.Object("dir/a"), + mockobject.Object("dir/b"), + mockdir.New("dir"), + mockobject.Object("dir2/a"), + mockobject.Object("dir2/b"), + mockobject.Object("dir1/a"), + mockobject.Object("dir3/b"), + } + require.NoError(t, dm.addEntries(entries)) + clearCallback() + err := dm.sendEntries(callback) + require.NoError(t, err) + assert.Equal(t, []string{ + "dir1", + "dir2", + "dir3", + }, got) + + // return error from callback + callback2 := func(entries fs.DirEntries) error { + return io.EOF + } + err = dm.sendEntries(callback2) + require.Equal(t, io.EOF, err) + + // empty + dm = newDirMap("") + clearCallback() + err = dm.sendEntries(callback) + require.NoError(t, err) + assert.Equal(t, []string(nil), got) +}