Upstream dundee/gdu state of the touched files before fa7a95a895

pkg/analyze/stored.go

@@ -0,0 +1,455 @@
+package analyze
+
+import (
+	"io"
+	"iter"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/dundee/gdu/v5/internal/common"
+	"github.com/dundee/gdu/v5/pkg/fs"
+	log "github.com/sirupsen/logrus"
+)
+
+// StoredAnalyzer implements Analyzer
+type StoredAnalyzer struct {
+	BaseAnalyzer
+	storage     *Storage
+	storagePath string
+}
+
+// CreateStoredAnalyzer returns Analyzer
+func CreateStoredAnalyzer(storagePath string) *StoredAnalyzer {
+	a := &StoredAnalyzer{
+		storagePath: storagePath,
+	}
+	a.Init()
+	return a
+}
+
+// AnalyzeDir analyzes given path
+func (a *StoredAnalyzer) AnalyzeDir(
+	path string, ignore common.ShouldDirBeIgnored, fileTypeFilter common.ShouldFileBeIgnored,
+) fs.Item {
+	a.ignoreDir = ignore
+	a.ignoreFileType = fileTypeFilter
+
+	a.storage = NewStorage(a.storagePath, path)
+	closeFn := a.storage.Open()
+	defer func() {
+		// nasty hack to close storage after all goroutines are done
+		// Wait returns immediately if value is 0
+		// few last goroutines might still start after that
+		time.Sleep(1 * time.Second)
+		closeFn()
+	}()
+
+	a.ignoreDir = ignore
+
+	go a.UpdateProgress()
+	dir := a.processDir(path)
+
+	a.wait.Wait()
+
+	a.progressDoneChan <- struct{}{}
+	a.doneChan.Broadcast()
+
+	return dir
+}
+
+func (a *StoredAnalyzer) processDir(path string) *StoredDir {
+	var (
+		file       fs.Item
+		err        error
+		totalUsage int64
+		info       os.FileInfo
+		dirCount   int
+	)
+
+	a.wait.Add(1)
+
+	files, err := os.ReadDir(path)
+	if err != nil {
+		log.Print(err.Error())
+	}
+
+	dir := &StoredDir{
+		Dir: &Dir{
+			File: &File{
+				Name: filepath.Base(path),
+				Flag: getDirFlag(err, len(files)),
+			},
+			BasePath:  filepath.Dir(path),
+			ItemCount: 1,
+			Files:     make(fs.Files, 0, len(files)),
+		},
+	}
+	parent := &ParentDir{Path: path}
+
+	setDirPlatformSpecificAttrs(dir.Dir, path)
+
+	for _, f := range files {
+		if a.IsCancelled() {
+			break
+		}
+		name := f.Name()
+		entryPath := filepath.Join(path, name)
+		if f.IsDir() {
+			if a.shouldSkipDir(name, entryPath) {
+				continue
+			}
+			dirCount++
+
+			subdir := &StoredDir{
+				&Dir{
+					File: &File{
+						Name: name,
+					},
+					BasePath: path,
+				},
+				nil,
+				sync.Mutex{},
+			}
+			dir.AddFile(subdir)
+
+			go func(entryPath string) {
+				concurrencyLimit <- struct{}{}
+				a.processDir(entryPath)
+				<-concurrencyLimit
+			}(entryPath)
+		} else {
+			// Apply file type filter if set
+			if a.ignoreFileType != nil && a.ignoreFileType(name) {
+				continue // Skip this file
+			}
+
+			info, err = f.Info()
+			if err != nil {
+				log.Print(err.Error())
+				continue
+			}
+
+			if a.followSymlinks && info.Mode()&os.ModeSymlink != 0 {
+				infoF, err := followSymlink(entryPath, a.gitAnnexedSize)
+				if err != nil {
+					log.Print(err.Error())
+					continue
+				}
+				if infoF != nil {
+					info = infoF
+				}
+			}
+
+			symlinkTarget := readSymlinkTarget(f.Type(), entryPath)
+
+			// Apply time filter if set
+			if a.matchesTimeFilterFn != nil && !a.matchesTimeFilterFn(info.ModTime()) {
+				continue // Skip this file
+			}
+
+			// Check if it's a zip or jar file
+			if a.archiveBrowsing && isZipFile(name) {
+				zipDir, err := processZipFile(entryPath, info)
+				if err != nil {
+					// If unable to process zip file, treat as regular file
+					log.Printf("Failed to process zip file %s: %v", entryPath, err)
+					file = &File{
+						Name:   name,
+						Flag:   getFlag(info),
+						Size:   info.Size(),
+						Parent: parent,
+					}
+				} else {
+					// Successfully processed zip file, use zip content size
+					uncompressedSize, compressedSize, err := getZipFileSize(entryPath)
+					if err == nil {
+						zipDir.Size = uncompressedSize
+						zipDir.Usage = compressedSize
+					}
+					zipDir.Parent = parent
+					file = zipDir
+				}
+			} else {
+				file = &File{
+					Name:    name,
+					Flag:    getFlag(info),
+					Size:    info.Size(),
+					Parent:  parent,
+					Symlink: symlinkTarget,
+				}
+			}
+
+			if file != nil {
+				// Only set platform-specific attributes for regular files
+				if regularFile, ok := file.(*File); ok {
+					setPlatformSpecificAttrs(regularFile, info)
+				}
+				totalUsage += file.GetUsage()
+				dir.AddFile(file)
+			}
+		}
+	}
+
+	err = a.storage.StoreDir(dir)
+	if err != nil {
+		log.Print(err.Error())
+	}
+
+	a.wait.Done()
+
+	a.progressCurrentItemName.Store(path)
+	a.progressItemCount.Add(int64(len(files)))
+	a.progressTotalUsage.Add(totalUsage)
+
+	return dir
+}
+
+// StoredDir implements Dir item stored on disk
+type StoredDir struct {
+	*Dir
+	cachedFiles fs.Files
+	dbLock      sync.Mutex
+}
+
+// GetParent returns parent dir
+func (f *StoredDir) GetParent() fs.Item {
+	if DefaultStorage.GetTopDir() == f.GetPath() {
+		return nil
+	}
+
+	if !DefaultStorage.IsOpen() {
+		closeFn := DefaultStorage.Open()
+		defer closeFn()
+	}
+
+	dir, err := DefaultStorage.GetDirForPath(f.BasePath)
+	if err != nil {
+		log.Print(err.Error())
+	}
+	return dir
+}
+
+// GetFiles returns files in directory as a sorted iterator
+// If files are already cached, use them
+// Otherwise load them from storage
+func (f *StoredDir) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
+	return func(yield func(fs.Item) bool) {
+		files := f.loadFiles()
+		sortFiles(files, sortBy, order)
+
+		for _, item := range files {
+			if !yield(item) {
+				return
+			}
+		}
+	}
+}
+
+// loadFiles loads files from storage or returns cached files
+func (f *StoredDir) loadFiles() fs.Files {
+	if f.cachedFiles != nil {
+		// Return a copy to avoid modifying cached slice
+		result := make(fs.Files, len(f.cachedFiles))
+		copy(result, f.cachedFiles)
+		return result
+	}
+
+	if !DefaultStorage.IsOpen() {
+		f.dbLock.Lock()
+		defer f.dbLock.Unlock()
+		closeFn := DefaultStorage.Open()
+		defer closeFn()
+	}
+
+	var files fs.Files
+	for _, file := range f.Files {
+		if file.IsDir() {
+			dir := &StoredDir{
+				&Dir{
+					File: &File{
+						Name: file.GetName(),
+					},
+					BasePath: f.GetPath(),
+				},
+				nil,
+				sync.Mutex{},
+			}
+
+			err := DefaultStorage.LoadDir(dir)
+			if err != nil {
+				log.Print(err.Error())
+			}
+			files = append(files, dir)
+		} else {
+			files = append(files, file)
+		}
+	}
+
+	f.cachedFiles = files
+	// Return a copy to avoid modifying cached slice
+	result := make(fs.Files, len(files))
+	copy(result, files)
+	return result
+}
+
+// RemoveFile removes file from stored directory
+// It also updates size and item count of parent directories
+func (f *StoredDir) RemoveFile(item fs.Item) {
+	if !DefaultStorage.IsOpen() {
+		f.dbLock.Lock()
+		defer f.dbLock.Unlock()
+		closeFn := DefaultStorage.Open()
+		defer closeFn()
+	}
+
+	f.Files = f.Files.Remove(item)
+	f.cachedFiles = nil
+
+	cur := f
+	for {
+		cur.ItemCount -= item.GetItemCount()
+		cur.Size -= item.GetSize()
+		cur.Usage -= item.GetUsage()
+
+		err := DefaultStorage.StoreDir(cur)
+		if err != nil {
+			log.Print(err.Error())
+		}
+
+		parent := cur.GetParent()
+		if parent == nil {
+			break
+		}
+		cur = parent.(*StoredDir)
+	}
+}
+
+// GetItemStats returns item count, apparent usage and real usage of this dir
+func (f *StoredDir) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) {
+	f.updateStats(linkedItems, filteringFiles)
+	return f.ItemCount, f.GetSize(), f.GetUsage()
+}
+
+func (f *StoredDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
+	f.updateStats(linkedItems, true)
+}
+
+// UpdateStats recursively updates size and item count
+func (f *StoredDir) UpdateStats(linkedItems fs.HardLinkedItems) {
+	f.updateStats(linkedItems, false)
+}
+
+func (f *StoredDir) updateStats(linkedItems fs.HardLinkedItems, filteringFiles bool) {
+	if !DefaultStorage.IsOpen() {
+		closeFn := DefaultStorage.Open()
+		defer closeFn()
+	}
+
+	totalSize := int64(4096)
+	totalUsage := int64(4096)
+	var itemCount int64
+	f.cachedFiles = nil
+	files := f.loadFiles()
+	for _, entry := range files {
+		count, size, usage := entry.GetItemStats(linkedItems, filteringFiles)
+		totalSize += size
+		totalUsage += usage
+		itemCount += count
+
+		if entry.GetMtime().After(f.Mtime) {
+			f.Mtime = entry.GetMtime()
+		}
+
+		switch entry.GetFlag() {
+		case '!', '.':
+			if f.Flag != '!' {
+				f.Flag = '.'
+			}
+		}
+	}
+	f.cachedFiles = nil
+	f.ItemCount = itemCount + 1
+	f.Size = totalSize
+	f.Usage = totalUsage
+	err := DefaultStorage.StoreDir(f)
+	if err != nil {
+		log.Print(err.Error())
+	}
+}
+
+// RemoveFileByName removes file by name from stored directory
+func (f *StoredDir) RemoveFileByName(name string) {
+	if !DefaultStorage.IsOpen() {
+		f.dbLock.Lock()
+		defer f.dbLock.Unlock()
+		closeFn := DefaultStorage.Open()
+		defer closeFn()
+	}
+
+	idx, ok := f.Files.FindByName(name)
+	if !ok {
+		return
+	}
+	item := f.Files[idx]
+	f.Files = append(f.Files[:idx], f.Files[idx+1:]...)
+	f.cachedFiles = nil
+
+	cur := f
+	for {
+		cur.ItemCount -= item.GetItemCount()
+		cur.Size -= item.GetSize()
+		cur.Usage -= item.GetUsage()
+
+		err := DefaultStorage.StoreDir(cur)
+		if err != nil {
+			log.Print(err.Error())
+		}
+
+		parent := cur.GetParent()
+		if parent == nil {
+			break
+		}
+		cur = parent.(*StoredDir)
+	}
+}
+
+// ParentDir represents parent directory of single file
+// It is used to get path to parent directory of a file
+type ParentDir struct {
+	Path string
+}
+
+func (p *ParentDir) GetPath() string {
+	return p.Path
+}
+func (p *ParentDir) GetName() string                                     { panic("must not be called") }
+func (p *ParentDir) GetFlag() rune                                       { panic("must not be called") }
+func (p *ParentDir) IsDir() bool                                         { panic("must not be called") }
+func (p *ParentDir) GetSize() int64                                      { panic("must not be called") }
+func (p *ParentDir) GetType() string                                     { panic("must not be called") }
+func (p *ParentDir) GetUsage() int64                                     { panic("must not be called") }
+func (p *ParentDir) GetMtime() time.Time                                 { panic("must not be called") }
+func (p *ParentDir) GetItemCount() int64                                 { panic("must not be called") }
+func (p *ParentDir) GetParent() fs.Item                                  { panic("must not be called") }
+func (p *ParentDir) SetParent(fs.Item)                                   { panic("must not be called") }
+func (p *ParentDir) GetMultiLinkedInode() uint64                         { panic("must not be called") }
+func (p *ParentDir) EncodeJSON(io.Writer, bool, fs.JSONAttributes) error { panic("must not be called") }
+func (p *ParentDir) UpdateStats(linkedItems fs.HardLinkedItems)          { panic("must not be called") }
+func (p *ParentDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
+	panic("must not be called")
+}
+func (p *ParentDir) AddFile(fs.Item)                                    { panic("must not be called") }
+func (p *ParentDir) GetFiles(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] { panic("must not be called") }
+func (p *ParentDir) GetFilesLocked(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] {
+	panic("must not be called")
+}
+func (p *ParentDir) RLock() func()                { panic("must not be called") }
+func (p *ParentDir) RemoveFile(item fs.Item)      { panic("must not be called") }
+func (p *ParentDir) RemoveFileByName(name string) { panic("must not be called") }
+func (p *ParentDir) GetItemStats(
+	linkedItems fs.HardLinkedItems, filteringFiles bool,
+) (itemCount, size, usage int64) {
+	panic("must not be called")
+}