Upstream dundee/gdu state of the touched files before fa7a95a895
This commit is contained in:
upstream committed
1 parent
606473d0fd
commit
83e8e1e984
1 file changed
+455
@@ -0,0 +1,455 @@
|
||||
package analyze
|
||||
|
||||
import (
|
||||
"io"
|
||||
"iter"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dundee/gdu/v5/internal/common"
|
||||
"github.com/dundee/gdu/v5/pkg/fs"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// StoredAnalyzer implements Analyzer
|
||||
type StoredAnalyzer struct {
|
||||
BaseAnalyzer
|
||||
storage *Storage
|
||||
storagePath string
|
||||
}
|
||||
|
||||
// CreateStoredAnalyzer returns Analyzer
|
||||
func CreateStoredAnalyzer(storagePath string) *StoredAnalyzer {
|
||||
a := &StoredAnalyzer{
|
||||
storagePath: storagePath,
|
||||
}
|
||||
a.Init()
|
||||
return a
|
||||
}
|
||||
|
||||
// AnalyzeDir analyzes given path
|
||||
func (a *StoredAnalyzer) AnalyzeDir(
|
||||
path string, ignore common.ShouldDirBeIgnored, fileTypeFilter common.ShouldFileBeIgnored,
|
||||
) fs.Item {
|
||||
a.ignoreDir = ignore
|
||||
a.ignoreFileType = fileTypeFilter
|
||||
|
||||
a.storage = NewStorage(a.storagePath, path)
|
||||
closeFn := a.storage.Open()
|
||||
defer func() {
|
||||
// nasty hack to close storage after all goroutines are done
|
||||
// Wait returns immediately if value is 0
|
||||
// few last goroutines might still start after that
|
||||
time.Sleep(1 * time.Second)
|
||||
closeFn()
|
||||
}()
|
||||
|
||||
a.ignoreDir = ignore
|
||||
|
||||
go a.UpdateProgress()
|
||||
dir := a.processDir(path)
|
||||
|
||||
a.wait.Wait()
|
||||
|
||||
a.progressDoneChan <- struct{}{}
|
||||
a.doneChan.Broadcast()
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
func (a *StoredAnalyzer) processDir(path string) *StoredDir {
|
||||
var (
|
||||
file fs.Item
|
||||
err error
|
||||
totalUsage int64
|
||||
info os.FileInfo
|
||||
dirCount int
|
||||
)
|
||||
|
||||
a.wait.Add(1)
|
||||
|
||||
files, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
}
|
||||
|
||||
dir := &StoredDir{
|
||||
Dir: &Dir{
|
||||
File: &File{
|
||||
Name: filepath.Base(path),
|
||||
Flag: getDirFlag(err, len(files)),
|
||||
},
|
||||
BasePath: filepath.Dir(path),
|
||||
ItemCount: 1,
|
||||
Files: make(fs.Files, 0, len(files)),
|
||||
},
|
||||
}
|
||||
parent := &ParentDir{Path: path}
|
||||
|
||||
setDirPlatformSpecificAttrs(dir.Dir, path)
|
||||
|
||||
for _, f := range files {
|
||||
if a.IsCancelled() {
|
||||
break
|
||||
}
|
||||
name := f.Name()
|
||||
entryPath := filepath.Join(path, name)
|
||||
if f.IsDir() {
|
||||
if a.shouldSkipDir(name, entryPath) {
|
||||
continue
|
||||
}
|
||||
dirCount++
|
||||
|
||||
subdir := &StoredDir{
|
||||
&Dir{
|
||||
File: &File{
|
||||
Name: name,
|
||||
},
|
||||
BasePath: path,
|
||||
},
|
||||
nil,
|
||||
sync.Mutex{},
|
||||
}
|
||||
dir.AddFile(subdir)
|
||||
|
||||
go func(entryPath string) {
|
||||
concurrencyLimit <- struct{}{}
|
||||
a.processDir(entryPath)
|
||||
<-concurrencyLimit
|
||||
}(entryPath)
|
||||
} else {
|
||||
// Apply file type filter if set
|
||||
if a.ignoreFileType != nil && a.ignoreFileType(name) {
|
||||
continue // Skip this file
|
||||
}
|
||||
|
||||
info, err = f.Info()
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
if a.followSymlinks && info.Mode()&os.ModeSymlink != 0 {
|
||||
infoF, err := followSymlink(entryPath, a.gitAnnexedSize)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
continue
|
||||
}
|
||||
if infoF != nil {
|
||||
info = infoF
|
||||
}
|
||||
}
|
||||
|
||||
symlinkTarget := readSymlinkTarget(f.Type(), entryPath)
|
||||
|
||||
// Apply time filter if set
|
||||
if a.matchesTimeFilterFn != nil && !a.matchesTimeFilterFn(info.ModTime()) {
|
||||
continue // Skip this file
|
||||
}
|
||||
|
||||
// Check if it's a zip or jar file
|
||||
if a.archiveBrowsing && isZipFile(name) {
|
||||
zipDir, err := processZipFile(entryPath, info)
|
||||
if err != nil {
|
||||
// If unable to process zip file, treat as regular file
|
||||
log.Printf("Failed to process zip file %s: %v", entryPath, err)
|
||||
file = &File{
|
||||
Name: name,
|
||||
Flag: getFlag(info),
|
||||
Size: info.Size(),
|
||||
Parent: parent,
|
||||
}
|
||||
} else {
|
||||
// Successfully processed zip file, use zip content size
|
||||
uncompressedSize, compressedSize, err := getZipFileSize(entryPath)
|
||||
if err == nil {
|
||||
zipDir.Size = uncompressedSize
|
||||
zipDir.Usage = compressedSize
|
||||
}
|
||||
zipDir.Parent = parent
|
||||
file = zipDir
|
||||
}
|
||||
} else {
|
||||
file = &File{
|
||||
Name: name,
|
||||
Flag: getFlag(info),
|
||||
Size: info.Size(),
|
||||
Parent: parent,
|
||||
Symlink: symlinkTarget,
|
||||
}
|
||||
}
|
||||
|
||||
if file != nil {
|
||||
// Only set platform-specific attributes for regular files
|
||||
if regularFile, ok := file.(*File); ok {
|
||||
setPlatformSpecificAttrs(regularFile, info)
|
||||
}
|
||||
totalUsage += file.GetUsage()
|
||||
dir.AddFile(file)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = a.storage.StoreDir(dir)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
}
|
||||
|
||||
a.wait.Done()
|
||||
|
||||
a.progressCurrentItemName.Store(path)
|
||||
a.progressItemCount.Add(int64(len(files)))
|
||||
a.progressTotalUsage.Add(totalUsage)
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
// StoredDir implements Dir item stored on disk
|
||||
type StoredDir struct {
|
||||
*Dir
|
||||
cachedFiles fs.Files
|
||||
dbLock sync.Mutex
|
||||
}
|
||||
|
||||
// GetParent returns parent dir
|
||||
func (f *StoredDir) GetParent() fs.Item {
|
||||
if DefaultStorage.GetTopDir() == f.GetPath() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !DefaultStorage.IsOpen() {
|
||||
closeFn := DefaultStorage.Open()
|
||||
defer closeFn()
|
||||
}
|
||||
|
||||
dir, err := DefaultStorage.GetDirForPath(f.BasePath)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// GetFiles returns files in directory as a sorted iterator
|
||||
// If files are already cached, use them
|
||||
// Otherwise load them from storage
|
||||
func (f *StoredDir) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
|
||||
return func(yield func(fs.Item) bool) {
|
||||
files := f.loadFiles()
|
||||
sortFiles(files, sortBy, order)
|
||||
|
||||
for _, item := range files {
|
||||
if !yield(item) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadFiles loads files from storage or returns cached files
|
||||
func (f *StoredDir) loadFiles() fs.Files {
|
||||
if f.cachedFiles != nil {
|
||||
// Return a copy to avoid modifying cached slice
|
||||
result := make(fs.Files, len(f.cachedFiles))
|
||||
copy(result, f.cachedFiles)
|
||||
return result
|
||||
}
|
||||
|
||||
if !DefaultStorage.IsOpen() {
|
||||
f.dbLock.Lock()
|
||||
defer f.dbLock.Unlock()
|
||||
closeFn := DefaultStorage.Open()
|
||||
defer closeFn()
|
||||
}
|
||||
|
||||
var files fs.Files
|
||||
for _, file := range f.Files {
|
||||
if file.IsDir() {
|
||||
dir := &StoredDir{
|
||||
&Dir{
|
||||
File: &File{
|
||||
Name: file.GetName(),
|
||||
},
|
||||
BasePath: f.GetPath(),
|
||||
},
|
||||
nil,
|
||||
sync.Mutex{},
|
||||
}
|
||||
|
||||
err := DefaultStorage.LoadDir(dir)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
}
|
||||
files = append(files, dir)
|
||||
} else {
|
||||
files = append(files, file)
|
||||
}
|
||||
}
|
||||
|
||||
f.cachedFiles = files
|
||||
// Return a copy to avoid modifying cached slice
|
||||
result := make(fs.Files, len(files))
|
||||
copy(result, files)
|
||||
return result
|
||||
}
|
||||
|
||||
// RemoveFile removes file from stored directory
|
||||
// It also updates size and item count of parent directories
|
||||
func (f *StoredDir) RemoveFile(item fs.Item) {
|
||||
if !DefaultStorage.IsOpen() {
|
||||
f.dbLock.Lock()
|
||||
defer f.dbLock.Unlock()
|
||||
closeFn := DefaultStorage.Open()
|
||||
defer closeFn()
|
||||
}
|
||||
|
||||
f.Files = f.Files.Remove(item)
|
||||
f.cachedFiles = nil
|
||||
|
||||
cur := f
|
||||
for {
|
||||
cur.ItemCount -= item.GetItemCount()
|
||||
cur.Size -= item.GetSize()
|
||||
cur.Usage -= item.GetUsage()
|
||||
|
||||
err := DefaultStorage.StoreDir(cur)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
}
|
||||
|
||||
parent := cur.GetParent()
|
||||
if parent == nil {
|
||||
break
|
||||
}
|
||||
cur = parent.(*StoredDir)
|
||||
}
|
||||
}
|
||||
|
||||
// GetItemStats returns item count, apparent usage and real usage of this dir
|
||||
func (f *StoredDir) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) {
|
||||
f.updateStats(linkedItems, filteringFiles)
|
||||
return f.ItemCount, f.GetSize(), f.GetUsage()
|
||||
}
|
||||
|
||||
func (f *StoredDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
|
||||
f.updateStats(linkedItems, true)
|
||||
}
|
||||
|
||||
// UpdateStats recursively updates size and item count
|
||||
func (f *StoredDir) UpdateStats(linkedItems fs.HardLinkedItems) {
|
||||
f.updateStats(linkedItems, false)
|
||||
}
|
||||
|
||||
func (f *StoredDir) updateStats(linkedItems fs.HardLinkedItems, filteringFiles bool) {
|
||||
if !DefaultStorage.IsOpen() {
|
||||
closeFn := DefaultStorage.Open()
|
||||
defer closeFn()
|
||||
}
|
||||
|
||||
totalSize := int64(4096)
|
||||
totalUsage := int64(4096)
|
||||
var itemCount int64
|
||||
f.cachedFiles = nil
|
||||
files := f.loadFiles()
|
||||
for _, entry := range files {
|
||||
count, size, usage := entry.GetItemStats(linkedItems, filteringFiles)
|
||||
totalSize += size
|
||||
totalUsage += usage
|
||||
itemCount += count
|
||||
|
||||
if entry.GetMtime().After(f.Mtime) {
|
||||
f.Mtime = entry.GetMtime()
|
||||
}
|
||||
|
||||
switch entry.GetFlag() {
|
||||
case '!', '.':
|
||||
if f.Flag != '!' {
|
||||
f.Flag = '.'
|
||||
}
|
||||
}
|
||||
}
|
||||
f.cachedFiles = nil
|
||||
f.ItemCount = itemCount + 1
|
||||
f.Size = totalSize
|
||||
f.Usage = totalUsage
|
||||
err := DefaultStorage.StoreDir(f)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveFileByName removes file by name from stored directory
|
||||
func (f *StoredDir) RemoveFileByName(name string) {
|
||||
if !DefaultStorage.IsOpen() {
|
||||
f.dbLock.Lock()
|
||||
defer f.dbLock.Unlock()
|
||||
closeFn := DefaultStorage.Open()
|
||||
defer closeFn()
|
||||
}
|
||||
|
||||
idx, ok := f.Files.FindByName(name)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
item := f.Files[idx]
|
||||
f.Files = append(f.Files[:idx], f.Files[idx+1:]...)
|
||||
f.cachedFiles = nil
|
||||
|
||||
cur := f
|
||||
for {
|
||||
cur.ItemCount -= item.GetItemCount()
|
||||
cur.Size -= item.GetSize()
|
||||
cur.Usage -= item.GetUsage()
|
||||
|
||||
err := DefaultStorage.StoreDir(cur)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
}
|
||||
|
||||
parent := cur.GetParent()
|
||||
if parent == nil {
|
||||
break
|
||||
}
|
||||
cur = parent.(*StoredDir)
|
||||
}
|
||||
}
|
||||
|
||||
// ParentDir represents parent directory of single file
|
||||
// It is used to get path to parent directory of a file
|
||||
type ParentDir struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
func (p *ParentDir) GetPath() string {
|
||||
return p.Path
|
||||
}
|
||||
func (p *ParentDir) GetName() string { panic("must not be called") }
|
||||
func (p *ParentDir) GetFlag() rune { panic("must not be called") }
|
||||
func (p *ParentDir) IsDir() bool { panic("must not be called") }
|
||||
func (p *ParentDir) GetSize() int64 { panic("must not be called") }
|
||||
func (p *ParentDir) GetType() string { panic("must not be called") }
|
||||
func (p *ParentDir) GetUsage() int64 { panic("must not be called") }
|
||||
func (p *ParentDir) GetMtime() time.Time { panic("must not be called") }
|
||||
func (p *ParentDir) GetItemCount() int64 { panic("must not be called") }
|
||||
func (p *ParentDir) GetParent() fs.Item { panic("must not be called") }
|
||||
func (p *ParentDir) SetParent(fs.Item) { panic("must not be called") }
|
||||
func (p *ParentDir) GetMultiLinkedInode() uint64 { panic("must not be called") }
|
||||
func (p *ParentDir) EncodeJSON(io.Writer, bool, fs.JSONAttributes) error { panic("must not be called") }
|
||||
func (p *ParentDir) UpdateStats(linkedItems fs.HardLinkedItems) { panic("must not be called") }
|
||||
func (p *ParentDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
|
||||
panic("must not be called")
|
||||
}
|
||||
func (p *ParentDir) AddFile(fs.Item) { panic("must not be called") }
|
||||
func (p *ParentDir) GetFiles(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] { panic("must not be called") }
|
||||
func (p *ParentDir) GetFilesLocked(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] {
|
||||
panic("must not be called")
|
||||
}
|
||||
func (p *ParentDir) RLock() func() { panic("must not be called") }
|
||||
func (p *ParentDir) RemoveFile(item fs.Item) { panic("must not be called") }
|
||||
func (p *ParentDir) RemoveFileByName(name string) { panic("must not be called") }
|
||||
func (p *ParentDir) GetItemStats(
|
||||
linkedItems fs.HardLinkedItems, filteringFiles bool,
|
||||
) (itemCount, size, usage int64) {
|
||||
panic("must not be called")
|
||||
}
|
||||
Reference in new issue
Block a user