package analyze import ( "io" "iter" "os" "path/filepath" "sync" "time" "github.com/dundee/gdu/v5/internal/common" "github.com/dundee/gdu/v5/pkg/fs" log "github.com/sirupsen/logrus" ) // StoredAnalyzer implements Analyzer type StoredAnalyzer struct { BaseAnalyzer storage *Storage storagePath string } // CreateStoredAnalyzer returns Analyzer func CreateStoredAnalyzer(storagePath string) *StoredAnalyzer { a := &StoredAnalyzer{ storagePath: storagePath, } a.Init() return a } // AnalyzeDir analyzes given path func (a *StoredAnalyzer) AnalyzeDir( path string, ignore common.ShouldDirBeIgnored, fileTypeFilter common.ShouldFileBeIgnored, ) fs.Item { a.ignoreDir = ignore a.ignoreFileType = fileTypeFilter a.storage = NewStorage(a.storagePath, path) closeFn := a.storage.Open() defer func() { // nasty hack to close storage after all goroutines are done // Wait returns immediately if value is 0 // few last goroutines might still start after that time.Sleep(1 * time.Second) closeFn() }() a.ignoreDir = ignore go a.UpdateProgress() dir := a.processDir(path) a.wait.Wait() a.progressDoneChan <- struct{}{} a.doneChan.Broadcast() return dir } func (a *StoredAnalyzer) processDir(path string) *StoredDir { var ( file fs.Item err error totalUsage int64 info os.FileInfo dirCount int ) a.wait.Add(1) files, err := os.ReadDir(path) if err != nil { log.Print(err.Error()) } dir := &StoredDir{ Dir: &Dir{ File: &File{ Name: filepath.Base(path), Flag: getDirFlag(err, len(files)), }, BasePath: filepath.Dir(path), ItemCount: 1, Files: make(fs.Files, 0, len(files)), }, } parent := &ParentDir{Path: path} setDirPlatformSpecificAttrs(dir.Dir, path) for _, f := range files { if a.IsCancelled() { break } name := f.Name() entryPath := filepath.Join(path, name) if f.IsDir() { if a.shouldSkipDir(name, entryPath) { continue } dirCount++ subdir := &StoredDir{ &Dir{ File: &File{ Name: name, }, BasePath: path, }, nil, sync.Mutex{}, } dir.AddFile(subdir) go func(entryPath string) { concurrencyLimit <- struct{}{} a.processDir(entryPath) <-concurrencyLimit }(entryPath) } else { // Apply file type filter if set if a.ignoreFileType != nil && a.ignoreFileType(name) { continue // Skip this file } info, err = f.Info() if err != nil { log.Print(err.Error()) continue } if a.followSymlinks && info.Mode()&os.ModeSymlink != 0 { infoF, err := followSymlink(entryPath, a.gitAnnexedSize) if err != nil { log.Print(err.Error()) continue } if infoF != nil { info = infoF } } symlinkTarget := readSymlinkTarget(f.Type(), entryPath) // Apply time filter if set if a.matchesTimeFilterFn != nil && !a.matchesTimeFilterFn(info.ModTime()) { continue // Skip this file } // Check if it's a zip or jar file if a.archiveBrowsing && isZipFile(name) { zipDir, err := processZipFile(entryPath, info) if err != nil { // If unable to process zip file, treat as regular file log.Printf("Failed to process zip file %s: %v", entryPath, err) file = &File{ Name: name, Flag: getFlag(info), Size: info.Size(), Parent: parent, } } else { // Successfully processed zip file, use zip content size uncompressedSize, compressedSize, err := getZipFileSize(entryPath) if err == nil { zipDir.Size = uncompressedSize zipDir.Usage = compressedSize } zipDir.Parent = parent file = zipDir } } else { file = &File{ Name: name, Flag: getFlag(info), Size: info.Size(), Parent: parent, Symlink: symlinkTarget, } } if file != nil { // Only set platform-specific attributes for regular files if regularFile, ok := file.(*File); ok { setPlatformSpecificAttrs(regularFile, info) } totalUsage += file.GetUsage() dir.AddFile(file) } } } err = a.storage.StoreDir(dir) if err != nil { log.Print(err.Error()) } a.wait.Done() a.progressCurrentItemName.Store(path) a.progressItemCount.Add(int64(len(files))) a.progressTotalUsage.Add(totalUsage) return dir } // StoredDir implements Dir item stored on disk type StoredDir struct { *Dir cachedFiles fs.Files dbLock sync.Mutex } // GetParent returns parent dir func (f *StoredDir) GetParent() fs.Item { if DefaultStorage.GetTopDir() == f.GetPath() { return nil } if !DefaultStorage.IsOpen() { closeFn := DefaultStorage.Open() defer closeFn() } dir, err := DefaultStorage.GetDirForPath(f.BasePath) if err != nil { log.Print(err.Error()) } return dir } // GetFiles returns files in directory as a sorted iterator // If files are already cached, use them // Otherwise load them from storage func (f *StoredDir) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] { return func(yield func(fs.Item) bool) { files := f.loadFiles() sortFiles(files, sortBy, order) for _, item := range files { if !yield(item) { return } } } } // loadFiles loads files from storage or returns cached files func (f *StoredDir) loadFiles() fs.Files { if f.cachedFiles != nil { // Return a copy to avoid modifying cached slice result := make(fs.Files, len(f.cachedFiles)) copy(result, f.cachedFiles) return result } if !DefaultStorage.IsOpen() { f.dbLock.Lock() defer f.dbLock.Unlock() closeFn := DefaultStorage.Open() defer closeFn() } var files fs.Files for _, file := range f.Files { if file.IsDir() { dir := &StoredDir{ &Dir{ File: &File{ Name: file.GetName(), }, BasePath: f.GetPath(), }, nil, sync.Mutex{}, } err := DefaultStorage.LoadDir(dir) if err != nil { log.Print(err.Error()) } files = append(files, dir) } else { files = append(files, file) } } f.cachedFiles = files // Return a copy to avoid modifying cached slice result := make(fs.Files, len(files)) copy(result, files) return result } // RemoveFile removes file from stored directory // It also updates size and item count of parent directories func (f *StoredDir) RemoveFile(item fs.Item) { if !DefaultStorage.IsOpen() { f.dbLock.Lock() defer f.dbLock.Unlock() closeFn := DefaultStorage.Open() defer closeFn() } f.Files = f.Files.Remove(item) f.cachedFiles = nil cur := f for { cur.ItemCount -= item.GetItemCount() cur.Size -= item.GetSize() cur.Usage -= item.GetUsage() err := DefaultStorage.StoreDir(cur) if err != nil { log.Print(err.Error()) } parent := cur.GetParent() if parent == nil { break } cur = parent.(*StoredDir) } } // GetItemStats returns item count, apparent usage and real usage of this dir func (f *StoredDir) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) { f.updateStats(linkedItems, filteringFiles) return f.ItemCount, f.GetSize(), f.GetUsage() } func (f *StoredDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) { f.updateStats(linkedItems, true) } // UpdateStats recursively updates size and item count func (f *StoredDir) UpdateStats(linkedItems fs.HardLinkedItems) { f.updateStats(linkedItems, false) } func (f *StoredDir) updateStats(linkedItems fs.HardLinkedItems, filteringFiles bool) { if !DefaultStorage.IsOpen() { closeFn := DefaultStorage.Open() defer closeFn() } totalSize := int64(4096) totalUsage := int64(4096) var itemCount int64 f.cachedFiles = nil files := f.loadFiles() // Snapshot the mutable stat fields under the read lock and commit them back // under the write lock, mirroring Dir.updateStats. Readers of these fields // (GetSize, GetUsage, GetFlag, GetMtime, GetItemCount) all take f.m, so the // writes here must take it too, e.g. when the stats are recomputed while a // preview reads the same directory mid-scan. f.m.RLock() mtime := f.Mtime flag := f.Flag f.m.RUnlock() for _, entry := range files { count, size, usage := entry.GetItemStats(linkedItems, filteringFiles) totalSize += size totalUsage += usage itemCount += count entryMtime := entry.GetMtime() if entryMtime.After(mtime) { mtime = entryMtime } switch entry.GetFlag() { case '!', '.': if flag != '!' { flag = '.' } } } f.cachedFiles = nil f.m.Lock() f.Mtime = mtime f.Flag = flag f.ItemCount = itemCount + 1 f.Size = totalSize f.Usage = totalUsage f.m.Unlock() err := DefaultStorage.StoreDir(f) if err != nil { log.Print(err.Error()) } } // RemoveFileByName removes file by name from stored directory func (f *StoredDir) RemoveFileByName(name string) { if !DefaultStorage.IsOpen() { f.dbLock.Lock() defer f.dbLock.Unlock() closeFn := DefaultStorage.Open() defer closeFn() } idx, ok := f.Files.FindByName(name) if !ok { return } item := f.Files[idx] f.Files = append(f.Files[:idx], f.Files[idx+1:]...) f.cachedFiles = nil cur := f for { cur.ItemCount -= item.GetItemCount() cur.Size -= item.GetSize() cur.Usage -= item.GetUsage() err := DefaultStorage.StoreDir(cur) if err != nil { log.Print(err.Error()) } parent := cur.GetParent() if parent == nil { break } cur = parent.(*StoredDir) } } // ParentDir represents parent directory of single file // It is used to get path to parent directory of a file type ParentDir struct { Path string } func (p *ParentDir) GetPath() string { return p.Path } func (p *ParentDir) GetName() string { panic("must not be called") } func (p *ParentDir) GetFlag() rune { panic("must not be called") } func (p *ParentDir) IsDir() bool { panic("must not be called") } func (p *ParentDir) GetSize() int64 { panic("must not be called") } func (p *ParentDir) GetType() string { panic("must not be called") } func (p *ParentDir) GetUsage() int64 { panic("must not be called") } func (p *ParentDir) GetMtime() time.Time { panic("must not be called") } func (p *ParentDir) GetItemCount() int64 { panic("must not be called") } func (p *ParentDir) GetParent() fs.Item { panic("must not be called") } func (p *ParentDir) SetParent(fs.Item) { panic("must not be called") } func (p *ParentDir) GetMultiLinkedInode() uint64 { panic("must not be called") } func (p *ParentDir) EncodeJSON(io.Writer, bool, fs.JSONAttributes) error { panic("must not be called") } func (p *ParentDir) UpdateStats(linkedItems fs.HardLinkedItems) { panic("must not be called") } func (p *ParentDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) { panic("must not be called") } func (p *ParentDir) AddFile(fs.Item) { panic("must not be called") } func (p *ParentDir) GetFiles(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] { panic("must not be called") } func (p *ParentDir) GetFilesLocked(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] { panic("must not be called") } func (p *ParentDir) RLock() func() { panic("must not be called") } func (p *ParentDir) RemoveFile(item fs.Item) { panic("must not be called") } func (p *ParentDir) RemoveFileByName(name string) { panic("must not be called") } func (p *ParentDir) GetItemStats( linkedItems fs.HardLinkedItems, filteringFiles bool, ) (itemCount, size, usage int64) { panic("must not be called") }