diff --git a/pkg/analyze/stored.go b/pkg/analyze/stored.go new file mode 100644 index 0000000..7944193 --- /dev/null +++ b/pkg/analyze/stored.go @@ -0,0 +1,455 @@ +package analyze + +import ( + "io" + "iter" + "os" + "path/filepath" + "sync" + "time" + + "github.com/dundee/gdu/v5/internal/common" + "github.com/dundee/gdu/v5/pkg/fs" + log "github.com/sirupsen/logrus" +) + +// StoredAnalyzer implements Analyzer +type StoredAnalyzer struct { + BaseAnalyzer + storage *Storage + storagePath string +} + +// CreateStoredAnalyzer returns Analyzer +func CreateStoredAnalyzer(storagePath string) *StoredAnalyzer { + a := &StoredAnalyzer{ + storagePath: storagePath, + } + a.Init() + return a +} + +// AnalyzeDir analyzes given path +func (a *StoredAnalyzer) AnalyzeDir( + path string, ignore common.ShouldDirBeIgnored, fileTypeFilter common.ShouldFileBeIgnored, +) fs.Item { + a.ignoreDir = ignore + a.ignoreFileType = fileTypeFilter + + a.storage = NewStorage(a.storagePath, path) + closeFn := a.storage.Open() + defer func() { + // nasty hack to close storage after all goroutines are done + // Wait returns immediately if value is 0 + // few last goroutines might still start after that + time.Sleep(1 * time.Second) + closeFn() + }() + + a.ignoreDir = ignore + + go a.UpdateProgress() + dir := a.processDir(path) + + a.wait.Wait() + + a.progressDoneChan <- struct{}{} + a.doneChan.Broadcast() + + return dir +} + +func (a *StoredAnalyzer) processDir(path string) *StoredDir { + var ( + file fs.Item + err error + totalUsage int64 + info os.FileInfo + dirCount int + ) + + a.wait.Add(1) + + files, err := os.ReadDir(path) + if err != nil { + log.Print(err.Error()) + } + + dir := &StoredDir{ + Dir: &Dir{ + File: &File{ + Name: filepath.Base(path), + Flag: getDirFlag(err, len(files)), + }, + BasePath: filepath.Dir(path), + ItemCount: 1, + Files: make(fs.Files, 0, len(files)), + }, + } + parent := &ParentDir{Path: path} + + setDirPlatformSpecificAttrs(dir.Dir, path) + + for _, f := range files { + if a.IsCancelled() { + break + } + name := f.Name() + entryPath := filepath.Join(path, name) + if f.IsDir() { + if a.shouldSkipDir(name, entryPath) { + continue + } + dirCount++ + + subdir := &StoredDir{ + &Dir{ + File: &File{ + Name: name, + }, + BasePath: path, + }, + nil, + sync.Mutex{}, + } + dir.AddFile(subdir) + + go func(entryPath string) { + concurrencyLimit <- struct{}{} + a.processDir(entryPath) + <-concurrencyLimit + }(entryPath) + } else { + // Apply file type filter if set + if a.ignoreFileType != nil && a.ignoreFileType(name) { + continue // Skip this file + } + + info, err = f.Info() + if err != nil { + log.Print(err.Error()) + continue + } + + if a.followSymlinks && info.Mode()&os.ModeSymlink != 0 { + infoF, err := followSymlink(entryPath, a.gitAnnexedSize) + if err != nil { + log.Print(err.Error()) + continue + } + if infoF != nil { + info = infoF + } + } + + symlinkTarget := readSymlinkTarget(f.Type(), entryPath) + + // Apply time filter if set + if a.matchesTimeFilterFn != nil && !a.matchesTimeFilterFn(info.ModTime()) { + continue // Skip this file + } + + // Check if it's a zip or jar file + if a.archiveBrowsing && isZipFile(name) { + zipDir, err := processZipFile(entryPath, info) + if err != nil { + // If unable to process zip file, treat as regular file + log.Printf("Failed to process zip file %s: %v", entryPath, err) + file = &File{ + Name: name, + Flag: getFlag(info), + Size: info.Size(), + Parent: parent, + } + } else { + // Successfully processed zip file, use zip content size + uncompressedSize, compressedSize, err := getZipFileSize(entryPath) + if err == nil { + zipDir.Size = uncompressedSize + zipDir.Usage = compressedSize + } + zipDir.Parent = parent + file = zipDir + } + } else { + file = &File{ + Name: name, + Flag: getFlag(info), + Size: info.Size(), + Parent: parent, + Symlink: symlinkTarget, + } + } + + if file != nil { + // Only set platform-specific attributes for regular files + if regularFile, ok := file.(*File); ok { + setPlatformSpecificAttrs(regularFile, info) + } + totalUsage += file.GetUsage() + dir.AddFile(file) + } + } + } + + err = a.storage.StoreDir(dir) + if err != nil { + log.Print(err.Error()) + } + + a.wait.Done() + + a.progressCurrentItemName.Store(path) + a.progressItemCount.Add(int64(len(files))) + a.progressTotalUsage.Add(totalUsage) + + return dir +} + +// StoredDir implements Dir item stored on disk +type StoredDir struct { + *Dir + cachedFiles fs.Files + dbLock sync.Mutex +} + +// GetParent returns parent dir +func (f *StoredDir) GetParent() fs.Item { + if DefaultStorage.GetTopDir() == f.GetPath() { + return nil + } + + if !DefaultStorage.IsOpen() { + closeFn := DefaultStorage.Open() + defer closeFn() + } + + dir, err := DefaultStorage.GetDirForPath(f.BasePath) + if err != nil { + log.Print(err.Error()) + } + return dir +} + +// GetFiles returns files in directory as a sorted iterator +// If files are already cached, use them +// Otherwise load them from storage +func (f *StoredDir) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] { + return func(yield func(fs.Item) bool) { + files := f.loadFiles() + sortFiles(files, sortBy, order) + + for _, item := range files { + if !yield(item) { + return + } + } + } +} + +// loadFiles loads files from storage or returns cached files +func (f *StoredDir) loadFiles() fs.Files { + if f.cachedFiles != nil { + // Return a copy to avoid modifying cached slice + result := make(fs.Files, len(f.cachedFiles)) + copy(result, f.cachedFiles) + return result + } + + if !DefaultStorage.IsOpen() { + f.dbLock.Lock() + defer f.dbLock.Unlock() + closeFn := DefaultStorage.Open() + defer closeFn() + } + + var files fs.Files + for _, file := range f.Files { + if file.IsDir() { + dir := &StoredDir{ + &Dir{ + File: &File{ + Name: file.GetName(), + }, + BasePath: f.GetPath(), + }, + nil, + sync.Mutex{}, + } + + err := DefaultStorage.LoadDir(dir) + if err != nil { + log.Print(err.Error()) + } + files = append(files, dir) + } else { + files = append(files, file) + } + } + + f.cachedFiles = files + // Return a copy to avoid modifying cached slice + result := make(fs.Files, len(files)) + copy(result, files) + return result +} + +// RemoveFile removes file from stored directory +// It also updates size and item count of parent directories +func (f *StoredDir) RemoveFile(item fs.Item) { + if !DefaultStorage.IsOpen() { + f.dbLock.Lock() + defer f.dbLock.Unlock() + closeFn := DefaultStorage.Open() + defer closeFn() + } + + f.Files = f.Files.Remove(item) + f.cachedFiles = nil + + cur := f + for { + cur.ItemCount -= item.GetItemCount() + cur.Size -= item.GetSize() + cur.Usage -= item.GetUsage() + + err := DefaultStorage.StoreDir(cur) + if err != nil { + log.Print(err.Error()) + } + + parent := cur.GetParent() + if parent == nil { + break + } + cur = parent.(*StoredDir) + } +} + +// GetItemStats returns item count, apparent usage and real usage of this dir +func (f *StoredDir) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) { + f.updateStats(linkedItems, filteringFiles) + return f.ItemCount, f.GetSize(), f.GetUsage() +} + +func (f *StoredDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) { + f.updateStats(linkedItems, true) +} + +// UpdateStats recursively updates size and item count +func (f *StoredDir) UpdateStats(linkedItems fs.HardLinkedItems) { + f.updateStats(linkedItems, false) +} + +func (f *StoredDir) updateStats(linkedItems fs.HardLinkedItems, filteringFiles bool) { + if !DefaultStorage.IsOpen() { + closeFn := DefaultStorage.Open() + defer closeFn() + } + + totalSize := int64(4096) + totalUsage := int64(4096) + var itemCount int64 + f.cachedFiles = nil + files := f.loadFiles() + for _, entry := range files { + count, size, usage := entry.GetItemStats(linkedItems, filteringFiles) + totalSize += size + totalUsage += usage + itemCount += count + + if entry.GetMtime().After(f.Mtime) { + f.Mtime = entry.GetMtime() + } + + switch entry.GetFlag() { + case '!', '.': + if f.Flag != '!' { + f.Flag = '.' + } + } + } + f.cachedFiles = nil + f.ItemCount = itemCount + 1 + f.Size = totalSize + f.Usage = totalUsage + err := DefaultStorage.StoreDir(f) + if err != nil { + log.Print(err.Error()) + } +} + +// RemoveFileByName removes file by name from stored directory +func (f *StoredDir) RemoveFileByName(name string) { + if !DefaultStorage.IsOpen() { + f.dbLock.Lock() + defer f.dbLock.Unlock() + closeFn := DefaultStorage.Open() + defer closeFn() + } + + idx, ok := f.Files.FindByName(name) + if !ok { + return + } + item := f.Files[idx] + f.Files = append(f.Files[:idx], f.Files[idx+1:]...) + f.cachedFiles = nil + + cur := f + for { + cur.ItemCount -= item.GetItemCount() + cur.Size -= item.GetSize() + cur.Usage -= item.GetUsage() + + err := DefaultStorage.StoreDir(cur) + if err != nil { + log.Print(err.Error()) + } + + parent := cur.GetParent() + if parent == nil { + break + } + cur = parent.(*StoredDir) + } +} + +// ParentDir represents parent directory of single file +// It is used to get path to parent directory of a file +type ParentDir struct { + Path string +} + +func (p *ParentDir) GetPath() string { + return p.Path +} +func (p *ParentDir) GetName() string { panic("must not be called") } +func (p *ParentDir) GetFlag() rune { panic("must not be called") } +func (p *ParentDir) IsDir() bool { panic("must not be called") } +func (p *ParentDir) GetSize() int64 { panic("must not be called") } +func (p *ParentDir) GetType() string { panic("must not be called") } +func (p *ParentDir) GetUsage() int64 { panic("must not be called") } +func (p *ParentDir) GetMtime() time.Time { panic("must not be called") } +func (p *ParentDir) GetItemCount() int64 { panic("must not be called") } +func (p *ParentDir) GetParent() fs.Item { panic("must not be called") } +func (p *ParentDir) SetParent(fs.Item) { panic("must not be called") } +func (p *ParentDir) GetMultiLinkedInode() uint64 { panic("must not be called") } +func (p *ParentDir) EncodeJSON(io.Writer, bool, fs.JSONAttributes) error { panic("must not be called") } +func (p *ParentDir) UpdateStats(linkedItems fs.HardLinkedItems) { panic("must not be called") } +func (p *ParentDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) { + panic("must not be called") +} +func (p *ParentDir) AddFile(fs.Item) { panic("must not be called") } +func (p *ParentDir) GetFiles(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] { panic("must not be called") } +func (p *ParentDir) GetFilesLocked(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] { + panic("must not be called") +} +func (p *ParentDir) RLock() func() { panic("must not be called") } +func (p *ParentDir) RemoveFile(item fs.Item) { panic("must not be called") } +func (p *ParentDir) RemoveFileByName(name string) { panic("must not be called") } +func (p *ParentDir) GetItemStats( + linkedItems fs.HardLinkedItems, filteringFiles bool, +) (itemCount, size, usage int64) { + panic("must not be called") +}