474 lines
11 KiB
Go
474 lines
11 KiB
Go
package analyze
|
|
|
|
import (
|
|
"io"
|
|
"iter"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/dundee/gdu/v5/internal/common"
|
|
"github.com/dundee/gdu/v5/pkg/fs"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// StoredAnalyzer implements Analyzer
|
|
type StoredAnalyzer struct {
|
|
BaseAnalyzer
|
|
storage *Storage
|
|
storagePath string
|
|
}
|
|
|
|
// CreateStoredAnalyzer returns Analyzer
|
|
func CreateStoredAnalyzer(storagePath string) *StoredAnalyzer {
|
|
a := &StoredAnalyzer{
|
|
storagePath: storagePath,
|
|
}
|
|
a.Init()
|
|
return a
|
|
}
|
|
|
|
// AnalyzeDir analyzes given path
|
|
func (a *StoredAnalyzer) AnalyzeDir(
|
|
path string, ignore common.ShouldDirBeIgnored, fileTypeFilter common.ShouldFileBeIgnored,
|
|
) fs.Item {
|
|
a.ignoreDir = ignore
|
|
a.ignoreFileType = fileTypeFilter
|
|
|
|
a.storage = NewStorage(a.storagePath, path)
|
|
closeFn := a.storage.Open()
|
|
defer func() {
|
|
// nasty hack to close storage after all goroutines are done
|
|
// Wait returns immediately if value is 0
|
|
// few last goroutines might still start after that
|
|
time.Sleep(1 * time.Second)
|
|
closeFn()
|
|
}()
|
|
|
|
a.ignoreDir = ignore
|
|
|
|
go a.UpdateProgress()
|
|
dir := a.processDir(path)
|
|
|
|
a.wait.Wait()
|
|
|
|
a.progressDoneChan <- struct{}{}
|
|
a.doneChan.Broadcast()
|
|
|
|
return dir
|
|
}
|
|
|
|
func (a *StoredAnalyzer) processDir(path string) *StoredDir {
|
|
var (
|
|
file fs.Item
|
|
err error
|
|
totalUsage int64
|
|
info os.FileInfo
|
|
dirCount int
|
|
)
|
|
|
|
a.wait.Add(1)
|
|
|
|
files, err := os.ReadDir(path)
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
}
|
|
|
|
dir := &StoredDir{
|
|
Dir: &Dir{
|
|
File: &File{
|
|
Name: filepath.Base(path),
|
|
Flag: getDirFlag(err, len(files)),
|
|
},
|
|
BasePath: filepath.Dir(path),
|
|
ItemCount: 1,
|
|
Files: make(fs.Files, 0, len(files)),
|
|
},
|
|
}
|
|
parent := &ParentDir{Path: path}
|
|
|
|
setDirPlatformSpecificAttrs(dir.Dir, path)
|
|
|
|
for _, f := range files {
|
|
if a.IsCancelled() {
|
|
break
|
|
}
|
|
name := f.Name()
|
|
entryPath := filepath.Join(path, name)
|
|
if f.IsDir() {
|
|
if a.shouldSkipDir(name, entryPath) {
|
|
continue
|
|
}
|
|
dirCount++
|
|
|
|
subdir := &StoredDir{
|
|
&Dir{
|
|
File: &File{
|
|
Name: name,
|
|
},
|
|
BasePath: path,
|
|
},
|
|
nil,
|
|
sync.Mutex{},
|
|
}
|
|
dir.AddFile(subdir)
|
|
|
|
go func(entryPath string) {
|
|
concurrencyLimit <- struct{}{}
|
|
a.processDir(entryPath)
|
|
<-concurrencyLimit
|
|
}(entryPath)
|
|
} else {
|
|
// Apply file type filter if set
|
|
if a.ignoreFileType != nil && a.ignoreFileType(name) {
|
|
continue // Skip this file
|
|
}
|
|
|
|
info, err = f.Info()
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
continue
|
|
}
|
|
|
|
if a.followSymlinks && info.Mode()&os.ModeSymlink != 0 {
|
|
infoF, err := followSymlink(entryPath, a.gitAnnexedSize)
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
continue
|
|
}
|
|
if infoF != nil {
|
|
info = infoF
|
|
}
|
|
}
|
|
|
|
symlinkTarget := readSymlinkTarget(f.Type(), entryPath)
|
|
|
|
// Apply time filter if set
|
|
if a.matchesTimeFilterFn != nil && !a.matchesTimeFilterFn(info.ModTime()) {
|
|
continue // Skip this file
|
|
}
|
|
|
|
// Check if it's a zip or jar file
|
|
if a.archiveBrowsing && isZipFile(name) {
|
|
zipDir, err := processZipFile(entryPath, info)
|
|
if err != nil {
|
|
// If unable to process zip file, treat as regular file
|
|
log.Printf("Failed to process zip file %s: %v", entryPath, err)
|
|
file = &File{
|
|
Name: name,
|
|
Flag: getFlag(info),
|
|
Size: info.Size(),
|
|
Parent: parent,
|
|
}
|
|
} else {
|
|
// Successfully processed zip file, use zip content size
|
|
uncompressedSize, compressedSize, err := getZipFileSize(entryPath)
|
|
if err == nil {
|
|
zipDir.Size = uncompressedSize
|
|
zipDir.Usage = compressedSize
|
|
}
|
|
zipDir.Parent = parent
|
|
file = zipDir
|
|
}
|
|
} else {
|
|
file = &File{
|
|
Name: name,
|
|
Flag: getFlag(info),
|
|
Size: info.Size(),
|
|
Parent: parent,
|
|
Symlink: symlinkTarget,
|
|
}
|
|
}
|
|
|
|
if file != nil {
|
|
// Only set platform-specific attributes for regular files
|
|
if regularFile, ok := file.(*File); ok {
|
|
setPlatformSpecificAttrs(regularFile, info)
|
|
}
|
|
totalUsage += file.GetUsage()
|
|
dir.AddFile(file)
|
|
}
|
|
}
|
|
}
|
|
|
|
err = a.storage.StoreDir(dir)
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
}
|
|
|
|
a.wait.Done()
|
|
|
|
a.progressCurrentItemName.Store(path)
|
|
a.progressItemCount.Add(int64(len(files)))
|
|
a.progressTotalUsage.Add(totalUsage)
|
|
|
|
return dir
|
|
}
|
|
|
|
// StoredDir implements Dir item stored on disk
|
|
type StoredDir struct {
|
|
*Dir
|
|
cachedFiles fs.Files
|
|
dbLock sync.Mutex
|
|
}
|
|
|
|
// GetParent returns parent dir
|
|
func (f *StoredDir) GetParent() fs.Item {
|
|
if DefaultStorage.GetTopDir() == f.GetPath() {
|
|
return nil
|
|
}
|
|
|
|
if !DefaultStorage.IsOpen() {
|
|
closeFn := DefaultStorage.Open()
|
|
defer closeFn()
|
|
}
|
|
|
|
dir, err := DefaultStorage.GetDirForPath(f.BasePath)
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
}
|
|
return dir
|
|
}
|
|
|
|
// GetFiles returns files in directory as a sorted iterator
|
|
// If files are already cached, use them
|
|
// Otherwise load them from storage
|
|
func (f *StoredDir) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
|
|
return func(yield func(fs.Item) bool) {
|
|
files := f.loadFiles()
|
|
sortFiles(files, sortBy, order)
|
|
|
|
for _, item := range files {
|
|
if !yield(item) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// loadFiles loads files from storage or returns cached files
|
|
func (f *StoredDir) loadFiles() fs.Files {
|
|
if f.cachedFiles != nil {
|
|
// Return a copy to avoid modifying cached slice
|
|
result := make(fs.Files, len(f.cachedFiles))
|
|
copy(result, f.cachedFiles)
|
|
return result
|
|
}
|
|
|
|
if !DefaultStorage.IsOpen() {
|
|
f.dbLock.Lock()
|
|
defer f.dbLock.Unlock()
|
|
closeFn := DefaultStorage.Open()
|
|
defer closeFn()
|
|
}
|
|
|
|
var files fs.Files
|
|
for _, file := range f.Files {
|
|
if file.IsDir() {
|
|
dir := &StoredDir{
|
|
&Dir{
|
|
File: &File{
|
|
Name: file.GetName(),
|
|
},
|
|
BasePath: f.GetPath(),
|
|
},
|
|
nil,
|
|
sync.Mutex{},
|
|
}
|
|
|
|
err := DefaultStorage.LoadDir(dir)
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
}
|
|
files = append(files, dir)
|
|
} else {
|
|
files = append(files, file)
|
|
}
|
|
}
|
|
|
|
f.cachedFiles = files
|
|
// Return a copy to avoid modifying cached slice
|
|
result := make(fs.Files, len(files))
|
|
copy(result, files)
|
|
return result
|
|
}
|
|
|
|
// RemoveFile removes file from stored directory
|
|
// It also updates size and item count of parent directories
|
|
func (f *StoredDir) RemoveFile(item fs.Item) {
|
|
if !DefaultStorage.IsOpen() {
|
|
f.dbLock.Lock()
|
|
defer f.dbLock.Unlock()
|
|
closeFn := DefaultStorage.Open()
|
|
defer closeFn()
|
|
}
|
|
|
|
f.Files = f.Files.Remove(item)
|
|
f.cachedFiles = nil
|
|
|
|
cur := f
|
|
for {
|
|
cur.ItemCount -= item.GetItemCount()
|
|
cur.Size -= item.GetSize()
|
|
cur.Usage -= item.GetUsage()
|
|
|
|
err := DefaultStorage.StoreDir(cur)
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
}
|
|
|
|
parent := cur.GetParent()
|
|
if parent == nil {
|
|
break
|
|
}
|
|
cur = parent.(*StoredDir)
|
|
}
|
|
}
|
|
|
|
// GetItemStats returns item count, apparent usage and real usage of this dir
|
|
func (f *StoredDir) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) {
|
|
f.updateStats(linkedItems, filteringFiles)
|
|
return f.ItemCount, f.GetSize(), f.GetUsage()
|
|
}
|
|
|
|
func (f *StoredDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
|
|
f.updateStats(linkedItems, true)
|
|
}
|
|
|
|
// UpdateStats recursively updates size and item count
|
|
func (f *StoredDir) UpdateStats(linkedItems fs.HardLinkedItems) {
|
|
f.updateStats(linkedItems, false)
|
|
}
|
|
|
|
func (f *StoredDir) updateStats(linkedItems fs.HardLinkedItems, filteringFiles bool) {
|
|
if !DefaultStorage.IsOpen() {
|
|
closeFn := DefaultStorage.Open()
|
|
defer closeFn()
|
|
}
|
|
|
|
totalSize := int64(4096)
|
|
totalUsage := int64(4096)
|
|
var itemCount int64
|
|
f.cachedFiles = nil
|
|
files := f.loadFiles()
|
|
|
|
// Snapshot the mutable stat fields under the read lock and commit them back
|
|
// under the write lock, mirroring Dir.updateStats. Readers of these fields
|
|
// (GetSize, GetUsage, GetFlag, GetMtime, GetItemCount) all take f.m, so the
|
|
// writes here must take it too, e.g. when the stats are recomputed while a
|
|
// preview reads the same directory mid-scan.
|
|
f.m.RLock()
|
|
mtime := f.Mtime
|
|
flag := f.Flag
|
|
f.m.RUnlock()
|
|
|
|
for _, entry := range files {
|
|
count, size, usage := entry.GetItemStats(linkedItems, filteringFiles)
|
|
totalSize += size
|
|
totalUsage += usage
|
|
itemCount += count
|
|
|
|
entryMtime := entry.GetMtime()
|
|
if entryMtime.After(mtime) {
|
|
mtime = entryMtime
|
|
}
|
|
|
|
switch entry.GetFlag() {
|
|
case '!', '.':
|
|
if flag != '!' {
|
|
flag = '.'
|
|
}
|
|
}
|
|
}
|
|
f.cachedFiles = nil
|
|
|
|
f.m.Lock()
|
|
f.Mtime = mtime
|
|
f.Flag = flag
|
|
f.ItemCount = itemCount + 1
|
|
f.Size = totalSize
|
|
f.Usage = totalUsage
|
|
f.m.Unlock()
|
|
|
|
err := DefaultStorage.StoreDir(f)
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
}
|
|
}
|
|
|
|
// RemoveFileByName removes file by name from stored directory
|
|
func (f *StoredDir) RemoveFileByName(name string) {
|
|
if !DefaultStorage.IsOpen() {
|
|
f.dbLock.Lock()
|
|
defer f.dbLock.Unlock()
|
|
closeFn := DefaultStorage.Open()
|
|
defer closeFn()
|
|
}
|
|
|
|
idx, ok := f.Files.FindByName(name)
|
|
if !ok {
|
|
return
|
|
}
|
|
item := f.Files[idx]
|
|
f.Files = append(f.Files[:idx], f.Files[idx+1:]...)
|
|
f.cachedFiles = nil
|
|
|
|
cur := f
|
|
for {
|
|
cur.ItemCount -= item.GetItemCount()
|
|
cur.Size -= item.GetSize()
|
|
cur.Usage -= item.GetUsage()
|
|
|
|
err := DefaultStorage.StoreDir(cur)
|
|
if err != nil {
|
|
log.Print(err.Error())
|
|
}
|
|
|
|
parent := cur.GetParent()
|
|
if parent == nil {
|
|
break
|
|
}
|
|
cur = parent.(*StoredDir)
|
|
}
|
|
}
|
|
|
|
// ParentDir represents parent directory of single file
|
|
// It is used to get path to parent directory of a file
|
|
type ParentDir struct {
|
|
Path string
|
|
}
|
|
|
|
func (p *ParentDir) GetPath() string {
|
|
return p.Path
|
|
}
|
|
func (p *ParentDir) GetName() string { panic("must not be called") }
|
|
func (p *ParentDir) GetFlag() rune { panic("must not be called") }
|
|
func (p *ParentDir) IsDir() bool { panic("must not be called") }
|
|
func (p *ParentDir) GetSize() int64 { panic("must not be called") }
|
|
func (p *ParentDir) GetType() string { panic("must not be called") }
|
|
func (p *ParentDir) GetUsage() int64 { panic("must not be called") }
|
|
func (p *ParentDir) GetMtime() time.Time { panic("must not be called") }
|
|
func (p *ParentDir) GetItemCount() int64 { panic("must not be called") }
|
|
func (p *ParentDir) GetParent() fs.Item { panic("must not be called") }
|
|
func (p *ParentDir) SetParent(fs.Item) { panic("must not be called") }
|
|
func (p *ParentDir) GetMultiLinkedInode() uint64 { panic("must not be called") }
|
|
func (p *ParentDir) EncodeJSON(io.Writer, bool, fs.JSONAttributes) error { panic("must not be called") }
|
|
func (p *ParentDir) UpdateStats(linkedItems fs.HardLinkedItems) { panic("must not be called") }
|
|
func (p *ParentDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
|
|
panic("must not be called")
|
|
}
|
|
func (p *ParentDir) AddFile(fs.Item) { panic("must not be called") }
|
|
func (p *ParentDir) GetFiles(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] { panic("must not be called") }
|
|
func (p *ParentDir) GetFilesLocked(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] {
|
|
panic("must not be called")
|
|
}
|
|
func (p *ParentDir) RLock() func() { panic("must not be called") }
|
|
func (p *ParentDir) RemoveFile(item fs.Item) { panic("must not be called") }
|
|
func (p *ParentDir) RemoveFileByName(name string) { panic("must not be called") }
|
|
func (p *ParentDir) GetItemStats(
|
|
linkedItems fs.HardLinkedItems, filteringFiles bool,
|
|
) (itemCount, size, usage int64) {
|
|
panic("must not be called")
|
|
}
|