rclone-rclone/fs/walk/walk_test.go

package walk

import (
	"context"
	"errors"
	"fmt"
	"io"
	"strings"
	"sync"
	"testing"

	"github.com/rclone/rclone/fs"
	_ "github.com/rclone/rclone/fs/accounting"
	"github.com/rclone/rclone/fs/filter"
	"github.com/rclone/rclone/fs/fserrors"
	"github.com/rclone/rclone/fstest/mockdir"
	"github.com/rclone/rclone/fstest/mockfs"
	"github.com/rclone/rclone/fstest/mockobject"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var errDirNotFound, errorBoom error

func init() {
	errDirNotFound = fserrors.FsError(fs.ErrorDirNotFound)
	fserrors.Count(errDirNotFound)
	errorBoom = fserrors.FsError(errors.New("boom"))
	fserrors.Count(errorBoom)
}

type (
	listResult struct {
		entries fs.DirEntries
		err     error
	}

	listResults map[string]listResult

	errorMap map[string]error

	listDirs struct {
		mu          sync.Mutex
		t           *testing.T
		fs          fs.Fs
		includeAll  bool
		results     listResults
		walkResults listResults
		walkErrors  errorMap
		finalError  error
		checkMaps   bool
		maxLevel    int
	}
)

func newListDirs(t *testing.T, f fs.Fs, includeAll bool, results listResults, walkErrors errorMap, finalError error) *listDirs {
	return &listDirs{
		t:           t,
		fs:          f,
		includeAll:  includeAll,
		results:     results,
		walkErrors:  walkErrors,
		walkResults: listResults{},
		finalError:  finalError,
		checkMaps:   true,
		maxLevel:    -1,
	}
}

// NoCheckMaps marks the maps as to be ignored at the end
func (ls *listDirs) NoCheckMaps() *listDirs {
	ls.checkMaps = false
	return ls
}

// SetLevel(1) turns off recursion
func (ls *listDirs) SetLevel(maxLevel int) *listDirs {
	ls.maxLevel = maxLevel
	return ls
}

// ListDir returns the expected listing for the directory
func (ls *listDirs) ListDir(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
	ls.mu.Lock()
	defer ls.mu.Unlock()
	assert.Equal(ls.t, ls.fs, f)
	assert.Equal(ls.t, ls.includeAll, includeAll)

	// Fetch results for this path
	result, ok := ls.results[dir]
	if !ok {
		ls.t.Errorf("Unexpected list of %q", dir)
		return nil, errors.New("unexpected list")
	}
	delete(ls.results, dir)

	// Put expected results for call of WalkFn
	ls.walkResults[dir] = result

	return result.entries, result.err
}

// ListR returns the expected listing for the directory using ListR
func (ls *listDirs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
	ls.mu.Lock()
	defer ls.mu.Unlock()

	var errorReturn error
	for dirPath, result := range ls.results {
		// Put expected results for call of WalkFn
		// Note that we don't call the function at all if we got an error
		if result.err != nil {
			errorReturn = result.err
		}
		if errorReturn == nil {
			err = callback(result.entries)
			require.NoError(ls.t, err)
			ls.walkResults[dirPath] = result
		}
	}
	ls.results = listResults{}
	return errorReturn
}

// IsFinished checks everything expected was used up
func (ls *listDirs) IsFinished() {
	if ls.checkMaps {
		assert.Equal(ls.t, errorMap{}, ls.walkErrors)
		assert.Equal(ls.t, listResults{}, ls.results)
		assert.Equal(ls.t, listResults{}, ls.walkResults)
	}
}

// WalkFn is called by the walk to test the expectations
func (ls *listDirs) WalkFn(dir string, entries fs.DirEntries, err error) error {
	ls.mu.Lock()
	defer ls.mu.Unlock()
	// ls.t.Logf("WalkFn(%q, %v, %q)", dir, entries, err)

	// Fetch expected entries and err
	result, ok := ls.walkResults[dir]
	if !ok {
		ls.t.Errorf("Unexpected walk of %q (result not found)", dir)
		return errors.New("result not found")
	}
	delete(ls.walkResults, dir)

	// Check arguments are as expected
	assert.Equal(ls.t, result.entries, entries)
	assert.Equal(ls.t, result.err, err)

	// Fetch return value
	returnErr, ok := ls.walkErrors[dir]
	if !ok {
		ls.t.Errorf("Unexpected walk of %q (error not found)", dir)
		return errors.New("error not found")
	}
	delete(ls.walkErrors, dir)

	return returnErr
}

// Walk does the walk and tests the expectations
func (ls *listDirs) Walk() {
	err := walk(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListDir)
	assert.True(ls.t, errors.Is(ls.finalError, err))
	ls.IsFinished()
}

// WalkR does the walkR and tests the expectations
func (ls *listDirs) WalkR() {
	err := walkR(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR)
	assert.Equal(ls.t, ls.finalError, err)
	if ls.finalError == nil {
		ls.IsFinished()
	}
}

func testWalkEmpty(t *testing.T) *listDirs {
	return newListDirs(t, nil, false,
		listResults{
			"": {entries: fs.DirEntries{}, err: nil},
		},
		errorMap{
			"": nil,
		},
		nil,
	)
}
func TestWalkEmpty(t *testing.T)  { testWalkEmpty(t).Walk() }
func TestWalkREmpty(t *testing.T) { testWalkEmpty(t).WalkR() }

func testWalkEmptySkip(t *testing.T) *listDirs {
	return newListDirs(t, nil, true,
		listResults{
			"": {entries: fs.DirEntries{}, err: nil},
		},
		errorMap{
			"": ErrorSkipDir,
		},
		nil,
	)
}
func TestWalkEmptySkip(t *testing.T)  { testWalkEmptySkip(t).Walk() }
func TestWalkREmptySkip(t *testing.T) { testWalkEmptySkip(t).WalkR() }

func testWalkNotFound(t *testing.T) *listDirs {
	return newListDirs(t, nil, true,
		listResults{
			"": {err: errDirNotFound},
		},
		errorMap{
			"": errDirNotFound,
		},
		errDirNotFound,
	)
}
func TestWalkNotFound(t *testing.T)  { testWalkNotFound(t).Walk() }
func TestWalkRNotFound(t *testing.T) { testWalkNotFound(t).WalkR() }

func TestWalkNotFoundMaskError(t *testing.T) {
	// this doesn't work for WalkR
	newListDirs(t, nil, true,
		listResults{
			"": {err: errDirNotFound},
		},
		errorMap{
			"": nil,
		},
		nil,
	).Walk()
}

func TestWalkNotFoundSkipError(t *testing.T) {
	// this doesn't work for WalkR
	newListDirs(t, nil, true,
		listResults{
			"": {err: errDirNotFound},
		},
		errorMap{
			"": ErrorSkipDir,
		},
		nil,
	).Walk()
}

func testWalkLevels(t *testing.T, maxLevel int) *listDirs {
	da := mockdir.New("a")
	oA := mockobject.Object("A")
	db := mockdir.New("a/b")
	oB := mockobject.Object("a/B")
	dc := mockdir.New("a/b/c")
	oC := mockobject.Object("a/b/C")
	dd := mockdir.New("a/b/c/d")
	oD := mockobject.Object("a/b/c/D")
	return newListDirs(t, nil, false,
		listResults{
			"":        {entries: fs.DirEntries{oA, da}, err: nil},
			"a":       {entries: fs.DirEntries{oB, db}, err: nil},
			"a/b":     {entries: fs.DirEntries{oC, dc}, err: nil},
			"a/b/c":   {entries: fs.DirEntries{oD, dd}, err: nil},
			"a/b/c/d": {entries: fs.DirEntries{}, err: nil},
		},
		errorMap{
			"":        nil,
			"a":       nil,
			"a/b":     nil,
			"a/b/c":   nil,
			"a/b/c/d": nil,
		},
		nil,
	).SetLevel(maxLevel)
}
func TestWalkLevels(t *testing.T)               { testWalkLevels(t, -1).Walk() }
func TestWalkRLevels(t *testing.T)              { testWalkLevels(t, -1).WalkR() }
func TestWalkLevelsNoRecursive10(t *testing.T)  { testWalkLevels(t, 10).Walk() }
func TestWalkRLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).WalkR() }

func TestWalkNDirTree(t *testing.T) {
	ls := testWalkLevels(t, -1)
	entries, err := walkNDirTree(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.ListDir)
	require.NoError(t, err)
	assert.Equal(t, `/
  A
  a/
a/
  B
  b/
a/b/
  C
  c/
a/b/c/
  D
  d/
a/b/c/d/
`, entries.String())
}

func testWalkLevelsNoRecursive(t *testing.T) *listDirs {
	da := mockdir.New("a")
	oA := mockobject.Object("A")
	return newListDirs(t, nil, false,
		listResults{
			"": {entries: fs.DirEntries{oA, da}, err: nil},
		},
		errorMap{
			"": nil,
		},
		nil,
	).SetLevel(1)
}
func TestWalkLevelsNoRecursive(t *testing.T)  { testWalkLevelsNoRecursive(t).Walk() }
func TestWalkRLevelsNoRecursive(t *testing.T) { testWalkLevelsNoRecursive(t).WalkR() }

func testWalkLevels2(t *testing.T) *listDirs {
	da := mockdir.New("a")
	oA := mockobject.Object("A")
	db := mockdir.New("a/b")
	oB := mockobject.Object("a/B")
	return newListDirs(t, nil, false,
		listResults{
			"":  {entries: fs.DirEntries{oA, da}, err: nil},
			"a": {entries: fs.DirEntries{oB, db}, err: nil},
		},
		errorMap{
			"":  nil,
			"a": nil,
		},
		nil,
	).SetLevel(2)
}
func TestWalkLevels2(t *testing.T)  { testWalkLevels2(t).Walk() }
func TestWalkRLevels2(t *testing.T) { testWalkLevels2(t).WalkR() }

func testWalkSkip(t *testing.T) *listDirs {
	da := mockdir.New("a")
	db := mockdir.New("a/b")
	dc := mockdir.New("a/b/c")
	return newListDirs(t, nil, false,
		listResults{
			"":    {entries: fs.DirEntries{da}, err: nil},
			"a":   {entries: fs.DirEntries{db}, err: nil},
			"a/b": {entries: fs.DirEntries{dc}, err: nil},
		},
		errorMap{
			"":    nil,
			"a":   nil,
			"a/b": ErrorSkipDir,
		},
		nil,
	)
}
func TestWalkSkip(t *testing.T)  { testWalkSkip(t).Walk() }
func TestWalkRSkip(t *testing.T) { testWalkSkip(t).WalkR() }

func walkErrors(t *testing.T, expectedErr error) *listDirs {
	lr := listResults{}
	em := errorMap{}
	de := make(fs.DirEntries, 10)
	for i := range de {
		path := string('0' + rune(i))
		de[i] = mockdir.New(path)
		lr[path] = listResult{entries: nil, err: fs.ErrorDirNotFound}
		em[path] = fs.ErrorDirNotFound
	}
	lr[""] = listResult{entries: de, err: nil}
	em[""] = nil
	return newListDirs(t, nil, true,
		lr,
		em,
		expectedErr,
	).NoCheckMaps()
}

func testWalkErrors(t *testing.T) *listDirs {
	return walkErrors(t, errDirNotFound)
}

func testWalkRErrors(t *testing.T) *listDirs {
	return walkErrors(t, fs.ErrorDirNotFound)
}

func TestWalkErrors(t *testing.T)  { testWalkErrors(t).Walk() }
func TestWalkRErrors(t *testing.T) { testWalkRErrors(t).WalkR() }

func makeTree(level int, terminalErrors bool) (listResults, errorMap) {
	lr := listResults{}
	em := errorMap{}
	var fill func(path string, level int)
	fill = func(path string, level int) {
		de := fs.DirEntries{}
		if level > 0 {
			for _, a := range "0123456789" {
				subPath := string(a)
				if path != "" {
					subPath = path + "/" + subPath
				}
				de = append(de, mockdir.New(subPath))
				fill(subPath, level-1)
			}
		}
		lr[path] = listResult{entries: de, err: nil}
		em[path] = nil
		if level == 0 && terminalErrors {
			em[path] = errorBoom
		}
	}
	fill("", level)
	return lr, em
}

func testWalkMulti(t *testing.T) *listDirs {
	lr, em := makeTree(3, false)
	return newListDirs(t, nil, true,
		lr,
		em,
		nil,
	)
}
func TestWalkMulti(t *testing.T)  { testWalkMulti(t).Walk() }
func TestWalkRMulti(t *testing.T) { testWalkMulti(t).WalkR() }

func testWalkMultiErrors(t *testing.T) *listDirs {
	lr, em := makeTree(3, true)
	return newListDirs(t, nil, true,
		lr,
		em,
		errorBoom,
	).NoCheckMaps()
}
func TestWalkMultiErrors(t *testing.T)  { testWalkMultiErrors(t).Walk() }
func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }

// a very simple listRcallback function
func makeListRCallback(entries fs.DirEntries, err error) fs.ListRFn {
	return func(ctx context.Context, dir string, callback fs.ListRCallback) error {
		if err == nil {
			err = callback(entries)
		}
		return err
	}
}

func TestWalkRDirTree(t *testing.T) {
	for _, test := range []struct {
		entries fs.DirEntries
		want    string
		err     error
		root    string
		level   int
		exclude string
	}{
		{
			entries: fs.DirEntries{},
			want:    "/\n",
			level:   -1,
		},
		{
			entries: fs.DirEntries{mockobject.Object("a")},
			want: `/
  a
`,
			level: -1,
		},
		{
			entries: fs.DirEntries{mockobject.Object("a/b")},
			want: `/
  a/
a/
  b
`,
			level: -1,
		},
		{
			entries: fs.DirEntries{mockobject.Object("a/b/c/d")},
			want: `/
  a/
a/
  b/
a/b/
  c/
a/b/c/
  d
`,
			level: -1,
		},
		{
			entries: fs.DirEntries{mockobject.Object("a")},
			err:     errorBoom,
			level:   -1,
		},
		{
			entries: fs.DirEntries{
				mockobject.Object("0/1/2/3"),
				mockobject.Object("4/5/6/7"),
				mockobject.Object("8/9/a/b"),
				mockobject.Object("c/d/e/f"),
				mockobject.Object("g/h/i/j"),
				mockobject.Object("k/l/m/n"),
				mockobject.Object("o/p/q/r"),
				mockobject.Object("s/t/u/v"),
				mockobject.Object("w/x/y/z"),
			},
			want: `/
  0/
  4/
  8/
  c/
  g/
  k/
  o/
  s/
  w/
0/
  1/
0/1/
  2/
0/1/2/
  3
4/
  5/
4/5/
  6/
4/5/6/
  7
8/
  9/
8/9/
  a/
8/9/a/
  b
c/
  d/
c/d/
  e/
c/d/e/
  f
g/
  h/
g/h/
  i/
g/h/i/
  j
k/
  l/
k/l/
  m/
k/l/m/
  n
o/
  p/
o/p/
  q/
o/p/q/
  r
s/
  t/
s/t/
  u/
s/t/u/
  v
w/
  x/
w/x/
  y/
w/x/y/
  z
`,
			level: -1,
		},
		{
			entries: fs.DirEntries{
				mockobject.Object("a/b/c/d/e/f1"),
				mockobject.Object("a/b/c/d/e/f2"),
				mockobject.Object("a/b/c/d/e/f3"),
			},
			want: `a/b/c/
  d/
a/b/c/d/
  e/
a/b/c/d/e/
  f1
  f2
  f3
`,
			root:  "a/b/c",
			level: -1,
		},
		{
			entries: fs.DirEntries{
				mockobject.Object("A"),
				mockobject.Object("a/B"),
				mockobject.Object("a/b/C"),
				mockobject.Object("a/b/c/D"),
				mockobject.Object("a/b/c/d/E"),
			},
			want: `/
  A
  a/
a/
  B
  b/
a/b/
`,
			level: 2,
		},
		{
			entries: fs.DirEntries{
				mockobject.Object("a/b/c"),
				mockobject.Object("a/b/c/d/e"),
			},
			want: `/
  a/
a/
  b/
a/b/
`,
			level: 2,
		},
		{
			entries: fs.DirEntries{
				mockobject.Object("a/.bzEmpty"),
				mockobject.Object("a/b1/.bzEmpty"),
				mockobject.Object("a/b2/.bzEmpty"),
			},
			want: `/
  a/
a/
  .bzEmpty
  b1/
  b2/
a/b1/
  .bzEmpty
a/b2/
  .bzEmpty
`,
			level:   -1,
			exclude: ""},
		{
			entries: fs.DirEntries{
				mockobject.Object("a/.bzEmpty"),
				mockobject.Object("a/b1/.bzEmpty"),
				mockobject.Object("a/b2/.bzEmpty"),
			},
			want: `/
  a/
a/
  b1/
  b2/
a/b1/
a/b2/
`,
			level:   -1,
			exclude: ".bzEmpty",
		},
	} {
		ctx := context.Background()
		if test.exclude != "" {
			fi, err := filter.NewFilter(nil)
			require.NoError(t, err)
			require.NoError(t, fi.Add(false, test.exclude))
			// Change the active filter
			ctx = filter.ReplaceConfig(ctx, fi)

		}
		r, err := walkRDirTree(ctx, nil, test.root, test.exclude == "", test.level, makeListRCallback(test.entries, test.err))
		what := fmt.Sprintf("%+v", test)
		assert.Equal(t, test.err, err, what)
		assert.Equal(t, test.want, r.String(), what)
	}
}

func TestWalkRDirTreeExclude(t *testing.T) {
	ctx := context.Background()
	fi := filter.GetConfig(ctx)
	for _, test := range []struct {
		entries     fs.DirEntries
		want        string
		err         error
		root        string
		level       int
		excludeFile string
		includeAll  bool
	}{
		{fs.DirEntries{mockobject.Object("a"), mockobject.Object("ignore")}, "", nil, "", -1, "ignore", false},
		{fs.DirEntries{mockobject.Object("a")}, `/
  a
`, nil, "", -1, "ignore", false},
		{fs.DirEntries{
			mockobject.Object("a"),
			mockobject.Object("b/b"),
			mockobject.Object("b/.ignore"),
		}, `/
  a
`, nil, "", -1, ".ignore", false},
		{fs.DirEntries{
			mockobject.Object("a"),
			mockobject.Object("b/.ignore"),
			mockobject.Object("b/b"),
		}, `/
  a
  b/
b/
  .ignore
  b
`, nil, "", -1, ".ignore", true},
		{fs.DirEntries{
			mockobject.Object("a"),
			mockobject.Object("b/b"),
			mockobject.Object("b/c/d/e"),
			mockobject.Object("b/c/ign"),
			mockobject.Object("b/c/x"),
		}, `/
  a
  b/
b/
  b
`, nil, "", -1, "ign", false},
		{fs.DirEntries{
			mockobject.Object("a"),
			mockobject.Object("b/b"),
			mockobject.Object("b/c/d/e"),
			mockobject.Object("b/c/ign"),
			mockobject.Object("b/c/x"),
		}, `/
  a
  b/
b/
  b
  c/
b/c/
  d/
  ign
  x
b/c/d/
  e
`, nil, "", -1, "ign", true},
	} {
		fi.Opt.ExcludeFile = []string{test.excludeFile}
		r, err := walkRDirTree(context.Background(), nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err))
		assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
		assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
	}
	// Set to default value, to avoid side effects
	fi.Opt.ExcludeFile = nil
}

// countingFs counts the NewObject calls made by the exclude file check
type countingFs struct {
	fs.Fs
	newObjectCalls int
}

func (f *countingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
	f.newObjectCalls++
	return f.Fs.NewObject(ctx, remote)
}

func excludeTmpContext(t testing.TB, excludeFile ...string) context.Context {
	fi, err := filter.NewFilter(nil)
	require.NoError(t, err)
	require.NoError(t, fi.Add(false, "*.tmp"))
	fi.Opt.ExcludeFile = excludeFile
	return filter.ReplaceConfig(context.Background(), fi)
}

func TestWalkRDirTreeExcludedObjectParents(t *testing.T) {
	ctx := excludeTmpContext(t)
	entries := fs.DirEntries{
		mockdir.New("a"),
		mockobject.Object("a/1.tmp"),
		mockobject.Object("a/2.tmp"),
		mockobject.Object("b/1.tmp"),
		mockobject.Object("b/file"),
		mockobject.Object("b/2.tmp"),
		mockobject.Object("c/d/1.tmp"),
		mockobject.Object("c/d/2.tmp"),
		mockobject.Object("e/f/g/1.tmp"),
	}
	r, err := walkRDirTree(ctx, nil, "", false, -1, makeListRCallback(entries, nil))
	require.NoError(t, err)
	assert.Equal(t, `/
  a/
  b/
  c/
  e/
a/
b/
  file
c/
  d/
c/d/
e/
  f/
e/f/
  g/
e/f/g/
`, r.String())

	r, err = walkRDirTree(ctx, nil, "", false, 2, makeListRCallback(entries, nil))
	require.NoError(t, err)
	assert.Equal(t, `/
  a/
  b/
  c/
  e/
a/
b/
  file
c/
  d/
c/d/
e/
  f/
e/f/
`, r.String())
}

func TestWalkRDirTreeExcludeFileCheckedOncePerDir(t *testing.T) {
	ctx := excludeTmpContext(t, ".ignore")
	f, err := mockfs.NewFs(ctx, "mock", "", nil)
	require.NoError(t, err)
	cf := &countingFs{Fs: f}
	var entries fs.DirEntries
	for _, dir := range []string{"a", "b", "c"} {
		for i := range 10 {
			entries = append(entries, mockobject.Object(fmt.Sprintf("%s/%d.tmp", dir, i)))
		}
	}
	r, err := walkRDirTree(ctx, cf, "", false, -1, makeListRCallback(entries, nil))
	require.NoError(t, err)
	assert.Equal(t, `/
  a/
  b/
  c/
a/
b/
c/
`, r.String())
	assert.Equal(t, 3, cf.newObjectCalls)
}

// makeExcludedEntries makes n kept and n excluded objects, all in the
// root or each pair in its own directory
func makeExcludedEntries(n int, ownDir bool) fs.DirEntries {
	entries := make(fs.DirEntries, 0, 2*n)
	for i := range n {
		dir := ""
		if ownDir {
			dir = fmt.Sprintf("dir%d/", i)
		}
		entries = append(entries,
			mockobject.Object(fmt.Sprintf("%sfile%d", dir, i)),
			mockobject.Object(fmt.Sprintf("%sfile%d.tmp", dir, i)),
		)
	}
	return entries
}

func BenchmarkWalkRDirTreeExcluded(b *testing.B) {
	ctx := excludeTmpContext(b)
	for _, bench := range []struct {
		name   string
		ownDir bool
	}{
		{"SameDir", false},
		{"OwnDir", true},
	} {
		entries := makeExcludedEntries(10000, bench.ownDir)
		b.Run(bench.name, func(b *testing.B) {
			for b.Loop() {
				_, err := walkRDirTree(ctx, nil, "", false, -1, makeListRCallback(entries, nil))
				require.NoError(b, err)
			}
		})
	}
}

func TestListType(t *testing.T) {
	assert.Equal(t, true, ListObjects.Objects())
	assert.Equal(t, false, ListObjects.Dirs())
	assert.Equal(t, false, ListDirs.Objects())
	assert.Equal(t, true, ListDirs.Dirs())
	assert.Equal(t, true, ListAll.Objects())
	assert.Equal(t, true, ListAll.Dirs())

	var (
		a           = mockobject.Object("a")
		b           = mockobject.Object("b")
		dir         = mockdir.New("dir")
		adir        = mockobject.Object("dir/a")
		dir2        = mockdir.New("dir2")
		origEntries = fs.DirEntries{
			a, b, dir, adir, dir2,
		}
		dirEntries = fs.DirEntries{
			dir, dir2,
		}
		objEntries = fs.DirEntries{
			a, b, adir,
		}
	)
	copyOrigEntries := func() (out fs.DirEntries) {
		out = make(fs.DirEntries, len(origEntries))
		copy(out, origEntries)
		return out
	}

	got := copyOrigEntries()
	ListAll.Filter(&got)
	assert.Equal(t, origEntries, got)

	got = copyOrigEntries()
	ListObjects.Filter(&got)
	assert.Equal(t, objEntries, got)

	got = copyOrigEntries()
	ListDirs.Filter(&got)
	assert.Equal(t, dirEntries, got)
}

func TestListRConfinement(t *testing.T) {
	ctx := context.Background()
	f, err := mockfs.NewFs(ctx, "mock", "/", nil)
	require.NoError(t, err)
	objects := fs.DirEntries{
		mockobject.Object("ok"),
		mockobject.Object(".."),
		mockobject.Object("../escape"),
		mockdir.New("../evildir"),
		// leading-slash climbers: filterDir doesn't guard the ListR path, so
		// these reach RemoveEscaping directly and must still be dropped.
		mockobject.Object("/../slashescape"),
		mockobject.Object("//../../slashescape2"),
		mockobject.Object("dir/deep"),
	}
	var got []string
	callback := func(entries fs.DirEntries) error {
		for _, entry := range entries {
			got = append(got, entry.Remote())
		}
		return nil
	}
	doListR := func(ctx context.Context, dir string, callback fs.ListRCallback) error {
		return callback(objects)
	}
	// includeAll = true exercises the unconditional confinement - the
	// include/exclude filter block is skipped in this mode, so an escaping
	// entry would otherwise pass straight through.
	err = listR(ctx, f, "", true, ListAll, callback, doListR, false)
	require.NoError(t, err)
	require.Equal(t, []string{"ok", "dir/deep"}, got)
}

func TestListR(t *testing.T) {
	ctx := context.Background()
	objects := fs.DirEntries{
		mockobject.Object("a"),
		mockobject.Object("b"),
		mockdir.New("dir"),
		mockobject.Object("dir/a"),
		mockobject.Object("dir/b"),
		mockobject.Object("dir/c"),
	}
	f, err := mockfs.NewFs(ctx, "mock", "/", nil)
	require.NoError(t, err)
	var got []string
	clearCallback := func() {
		got = nil
	}
	callback := func(entries fs.DirEntries) error {
		for _, entry := range entries {
			got = append(got, entry.Remote())
		}
		return nil
	}
	doListR := func(ctx context.Context, dir string, callback fs.ListRCallback) error {
		var os fs.DirEntries
		for _, o := range objects {
			if dir == "" || strings.HasPrefix(o.Remote(), dir+"/") {
				os = append(os, o)
			}
		}
		return callback(os)
	}

	fi, err := filter.NewFilter(nil)
	require.NoError(t, err)
	require.NoError(t, fi.AddRule("+ b"))
	require.NoError(t, fi.AddRule("- *"))

	// Change the active filter
	ctx = filter.ReplaceConfig(ctx, fi)

	// Base case
	clearCallback()
	err = listR(ctx, f, "", true, ListAll, callback, doListR, false)
	require.NoError(t, err)
	require.Equal(t, []string{"a", "b", "dir", "dir/a", "dir/b", "dir/c"}, got)

	// Base case - with Objects
	clearCallback()
	err = listR(ctx, f, "", true, ListObjects, callback, doListR, false)
	require.NoError(t, err)
	require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/c"}, got)

	// Base case - with Dirs
	clearCallback()
	err = listR(ctx, f, "", true, ListDirs, callback, doListR, false)
	require.NoError(t, err)
	require.Equal(t, []string{"dir"}, got)

	// With filter
	clearCallback()
	err = listR(ctx, f, "", false, ListAll, callback, doListR, false)
	require.NoError(t, err)
	require.Equal(t, []string{"b", "dir", "dir/b"}, got)

	// With filter - with Objects
	clearCallback()
	err = listR(ctx, f, "", false, ListObjects, callback, doListR, false)
	require.NoError(t, err)
	require.Equal(t, []string{"b", "dir/b"}, got)

	// With filter - with Dir
	clearCallback()
	err = listR(ctx, f, "", false, ListDirs, callback, doListR, false)
	require.NoError(t, err)
	require.Equal(t, []string{"dir"}, got)

	// With filter and subdir
	clearCallback()
	err = listR(ctx, f, "dir", false, ListAll, callback, doListR, false)
	require.NoError(t, err)
	require.Equal(t, []string{"dir/b"}, got)

	// Now bucket-based
	objects = fs.DirEntries{
		mockobject.Object("a"),
		mockobject.Object("b"),
		mockobject.Object("dir/a"),
		mockobject.Object("dir/b"),
		mockobject.Object("dir/subdir/c"),
		mockdir.New("dir/subdir"),
	}

	// Base case
	clearCallback()
	err = listR(ctx, f, "", true, ListAll, callback, doListR, true)
	require.NoError(t, err)
	require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/subdir/c", "dir/subdir", "dir"}, got)

	// With filter
	clearCallback()
	err = listR(ctx, f, "", false, ListAll, callback, doListR, true)
	require.NoError(t, err)
	require.Equal(t, []string{"b", "dir/b", "dir/subdir", "dir"}, got)

	// With filter and subdir
	clearCallback()
	err = listR(ctx, f, "dir", false, ListAll, callback, doListR, true)
	require.NoError(t, err)
	require.Equal(t, []string{"dir/b", "dir/subdir"}, got)

	// With filter and subdir - with Objects
	clearCallback()
	err = listR(ctx, f, "dir", false, ListObjects, callback, doListR, true)
	require.NoError(t, err)
	require.Equal(t, []string{"dir/b"}, got)

	// With filter and subdir - with Dirs
	clearCallback()
	err = listR(ctx, f, "dir", false, ListDirs, callback, doListR, true)
	require.NoError(t, err)
	require.Equal(t, []string{"dir/subdir"}, got)
}

func TestDirMapAdd(t *testing.T) {
	type add struct {
		dir  string
		sent bool
	}
	for i, test := range []struct {
		root string
		in   []add
		want map[string]bool
	}{
		{
			root: "",
			in: []add{
				{"", true},
			},
			want: map[string]bool{},
		},
		{
			root: "",
			in: []add{
				{"a/b/c", true},
			},
			want: map[string]bool{
				"a/b/c": true,
				"a/b":   false,
				"a":     false,
			},
		},
		{
			root: "",
			in: []add{
				{"a/b/c", true},
				{"a/b", true},
			},
			want: map[string]bool{
				"a/b/c": true,
				"a/b":   true,
				"a":     false,
			},
		},
		{
			root: "",
			in: []add{
				{"a/b", true},
				{"a/b/c", false},
			},
			want: map[string]bool{
				"a/b/c": false,
				"a/b":   true,
				"a":     false,
			},
		},
		{
			root: "root",
			in: []add{
				{"root/a/b", true},
				{"root/a/b/c", false},
			},
			want: map[string]bool{
				"root/a/b/c": false,
				"root/a/b":   true,
				"root/a":     false,
			},
		},
	} {
		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
			dm := newDirMap(test.root)
			for _, item := range test.in {
				dm.add(item.dir, item.sent)
			}
			assert.Equal(t, test.want, dm.m)
		})
	}
}

func TestDirMapAddEntries(t *testing.T) {
	dm := newDirMap("")
	entries := fs.DirEntries{
		mockobject.Object("dir/a"),
		mockobject.Object("dir/b"),
		mockdir.New("dir"),
		mockobject.Object("dir2/a"),
		mockobject.Object("dir2/b"),
	}
	require.NoError(t, dm.addEntries(entries))
	assert.Equal(t, map[string]bool{"dir": true, "dir2": false}, dm.m)
}

func TestDirMapSendEntries(t *testing.T) {
	var got []string
	clearCallback := func() {
		got = nil
	}
	callback := func(entries fs.DirEntries) error {
		for _, entry := range entries {
			got = append(got, entry.Remote())
		}
		return nil
	}

	// general test
	dm := newDirMap("")
	entries := fs.DirEntries{
		mockobject.Object("dir/a"),
		mockobject.Object("dir/b"),
		mockdir.New("dir"),
		mockobject.Object("dir2/a"),
		mockobject.Object("dir2/b"),
		mockobject.Object("dir1/a"),
		mockobject.Object("dir3/b"),
	}
	require.NoError(t, dm.addEntries(entries))
	clearCallback()
	err := dm.sendEntries(callback)
	require.NoError(t, err)
	assert.Equal(t, []string{
		"dir1",
		"dir2",
		"dir3",
	}, got)

	// return error from callback
	callback2 := func(entries fs.DirEntries) error {
		return io.EOF
	}
	err = dm.sendEntries(callback2)
	require.Equal(t, io.EOF, err)

	// empty
	dm = newDirMap("")
	clearCallback()
	err = dm.sendEntries(callback)
	require.NoError(t, err)
	assert.Equal(t, []string(nil), got)
}