Skip to content

Commit cfcc4c7

Browse files
laktaklaktak
committed
improve dedup messages
1 parent d8c35d9 commit cfcc4c7

File tree

3 files changed

+66
-34
lines changed

3 files changed

+66
-34
lines changed

‎cmd/chkbit/dedup.go‎

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func (m *Main) handleDedupProgress(mode1 bool) {
4545
if m.progress == Fancy {
4646
lterm.Write(termBG, termFG1, stat, lterm.ClearLine(0), lterm.Reset, "\r")
4747
} else {
48-
fmt.Print(m.dedup.NumTotal, "\r")
48+
fmt.Print(m.dedup.NumTotal(), "\r")
4949
}
5050
}
5151
case perf := <-m.dedup.PerfQueue:
@@ -58,12 +58,12 @@ func (m *Main) handleDedupProgress(mode1 bool) {
5858
stat = fmt.Sprintf("[$%s$%s$]$ %5.0f%% ", pa, pb, perf.Percent*100)
5959

6060
if mode1 {
61-
stat += fmt.Sprintf("$ # %7d ", m.dedup.NumTotal)
61+
stat += fmt.Sprintf("$ # %7d ", m.dedup.NumTotal())
6262
statF := fmt.Sprintf("%d files/s", m.fps.Last())
6363
stat += fmt.Sprintf("$ %s $%-13s ", util.Sparkline(m.fps.Stats), statF)
6464
} else {
65-
stat += fmt.Sprintf("$ # %d ", m.dedup.NumTotal)
66-
stat += fmt.Sprintf("$ %sB reclaimed ", intutil.FormatSize(m.dedup.ReclaimedTotal))
65+
stat += fmt.Sprintf("$ # %d ", m.dedup.NumTotal())
66+
stat += fmt.Sprintf("$ %sB reclaimed ", intutil.FormatSize(m.dedup.ReclaimedTotal()))
6767
}
6868

6969
stat = util.LeftTruncate(stat, m.termWidth-1+5) // extra for col tokens
@@ -83,7 +83,7 @@ func (m *Main) handleDedupProgress(mode1 bool) {
8383
if m.progress == Fancy {
8484
lterm.Write(termBG, termFG1, stat, spin, lterm.ClearLine(0), lterm.Reset, "\r")
8585
} else if m.progress == Plain {
86-
fmt.Print(m.dedup.NumTotal, "\r")
86+
fmt.Print(m.dedup.NumTotal(), "\r")
8787
}
8888
}
8989
}
@@ -170,29 +170,23 @@ func (m *Main) runDedup(command string, dd *CLIDedup, indexName string) int {
170170
return 1
171171
}
172172
if st != chkbit.IndexTypeAtom {
173-
fmt.Println("error: dedup is incompatible with split mode")
173+
m.printErr("error: dedup requires an atom index; you can create one with `chkbit fuse` while leaving your split index in place")
174174
return 1
175175
}
176176

177-
m.dedup, err = chkbit.NewDedup(root, indexName)
177+
m.dedup, err = chkbit.NewDedup(root, indexName, command == cmdDedupDetect)
178178
if err != nil {
179179
m.printError(err)
180+
if command != cmdDedupDetect && os.IsNotExist(err) {
181+
m.printStderr("Did you forget to run `chkbit detect`?")
182+
}
180183
return 1
181184
}
182185
defer m.dedup.Finish()
183186

184187
mode1 := true
185-
resultCh := make(chan error, 1)
186-
launchFunc := func() {
187-
var err error
188-
switch command {
189-
case cmdDedupDetect:
190-
err = m.dedup.DetectDupes(dd.Detect.MinSize, m.verbose)
191-
case cmdDedupRun, cmdDedupRun2:
192-
err = m.dedup.Dedup(dd.Run.Hashes, m.verbose)
193-
}
194-
resultCh <- err
195-
m.dedup.LogQueue <- nil
188+
printUpdated := func() {
189+
m.logInfo("", "last updated "+m.dedup.LastUpdated().Format(time.DateTime))
196190
}
197191

198192
switch command {
@@ -204,6 +198,7 @@ func (m *Main) runDedup(command string, dd *CLIDedup, indexName string) int {
204198
}
205199
} else {
206200
m.logInfo("", "chkbit dedup show "+argPath)
201+
printUpdated()
207202
m.showDedupStatus(list, dd.Show.Details)
208203
}
209204
}
@@ -213,11 +208,23 @@ func (m *Main) runDedup(command string, dd *CLIDedup, indexName string) int {
213208
fmt.Println(abortTip)
214209
case cmdDedupRun, cmdDedupRun2:
215210
m.logInfo("", fmt.Sprintf("chkbit dedup detect %s %s", argPath, dd.Run.Hashes))
211+
printUpdated()
216212
fmt.Println(abortTip)
217213
mode1 = false
218214
}
219215

220-
go launchFunc()
216+
resultCh := make(chan error, 1)
217+
go func() {
218+
var err error
219+
switch command {
220+
case cmdDedupDetect:
221+
err = m.dedup.DetectDupes(dd.Detect.MinSize, m.verbose)
222+
case cmdDedupRun, cmdDedupRun2:
223+
err = m.dedup.Dedup(dd.Run.Hashes, m.verbose)
224+
}
225+
resultCh <- err
226+
m.dedup.LogQueue <- nil
227+
}()
221228
m.handleDedupProgress(mode1)
222229

223230
if err = <-resultCh; err != nil {
@@ -227,14 +234,17 @@ func (m *Main) runDedup(command string, dd *CLIDedup, indexName string) int {
227234
}
228235
}
229236

230-
if m.progress == Fancy && m.dedup.NumTotal > 0 {
231-
elapsed := time.Since(m.fps.Start)
232-
elapsedS := elapsed.Seconds()
233-
m.logInfo("", fmt.Sprintf("- %s elapsed", elapsed.Truncate(time.Second)))
234-
m.logInfo("", fmt.Sprintf("- %d file(s) processed", m.dedup.NumTotal))
235-
m.logInfo("", fmt.Sprintf("- %.2f files/second", (float64(m.fps.Total)+float64(m.fps.Current))/elapsedS))
236-
if m.dedup.ReclaimedTotal > 0 {
237-
m.logInfo("", fmt.Sprintf("- %sB reclaimed", intutil.FormatSize(m.dedup.ReclaimedTotal)))
237+
switch command {
238+
case cmdDedupDetect, cmdDedupRun, cmdDedupRun2:
239+
if m.progress == Fancy {
240+
elapsed := time.Since(m.fps.Start)
241+
elapsedS := elapsed.Seconds()
242+
m.logInfo("", fmt.Sprintf("- %s elapsed", elapsed.Truncate(time.Second)))
243+
m.logInfo("", fmt.Sprintf("- %s processed", util.LangNum1Choice(m.dedup.NumTotal(), "file", "files")))
244+
m.logInfo("", fmt.Sprintf("- %.2f files/second", (float64(m.fps.Total)+float64(m.fps.Current))/elapsedS))
245+
if m.dedup.ReclaimedTotal() > 0 {
246+
m.logInfo("", fmt.Sprintf("- %sB reclaimed", intutil.FormatSize(m.dedup.ReclaimedTotal())))
247+
}
238248
}
239249
}
240250

‎cmd/chkbit/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ type CLI struct {
8888
Fuse struct {
8989
Path string `arg:"" help:"directory for the index"`
9090
Force bool `help:"force overwrite if a index already exists"`
91-
} `cmd:"" help:"merge all indexes (split&atom) under this path into an atom index"`
91+
} `cmd:"" help:"merge all indexes (split&atom) under this path into a single atom index, will NOT delete the source indexes"`
9292

9393
Dedup CLIDedup `cmd:"" help:"Deduplication commands"`
9494

‎dedup.go‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"os"
88
"path/filepath"
99
"slices"
10+
"time"
1011

1112
"github.com/laktak/chkbit/v6/intutil"
1213
bolt "go.etcd.io/bbolt"
@@ -23,12 +24,13 @@ type Dedup struct {
2324
conn *bolt.DB
2425

2526
doAbort bool
26-
NumTotal uint
27-
ReclaimedTotal uint64
27+
numTotal int
28+
reclaimedTotal uint64
2829
}
2930

3031
type ddStatus struct {
31-
Gen int `json:"gen"`
32+
Gen int `json:"gen"`
33+
Updated time.Time `json:"mod"`
3234
}
3335

3436
type ddBag struct {
@@ -77,6 +79,18 @@ func (d *Dedup) DidAbort() bool {
7779
return d.doAbort
7880
}
7981

82+
func (d *Dedup) NumTotal() int {
83+
return d.numTotal
84+
}
85+
86+
func (d *Dedup) ReclaimedTotal() uint64 {
87+
return d.reclaimedTotal
88+
}
89+
90+
func (d *Dedup) LastUpdated() time.Time {
91+
return d.status.Updated
92+
}
93+
8094
func (d *Dedup) log(stat Status, message string) {
8195
d.LogQueue <- &LogEvent{stat, message}
8296
}
@@ -86,15 +100,15 @@ func (d *Dedup) logMsg(message string) {
86100
}
87101

88102
func (d *Dedup) perfMonFiles(numFiles int, i float64, l int) {
89-
d.NumTotal += uint(numFiles)
103+
d.numTotal += numFiles
90104
pc := 0.0
91105
if l > 0 {
92106
pc = i / float64(l)
93107
}
94108
d.PerfQueue <- &DedupPerfEvent{int64(numFiles), pc}
95109
}
96110

97-
func NewDedup(path string, indexName string) (*Dedup, error) {
111+
func NewDedup(path string, indexName string, createIfNotExists bool) (*Dedup, error) {
98112
var err error
99113
d := &Dedup{
100114
rootPath: path,
@@ -104,6 +118,13 @@ func NewDedup(path string, indexName string) (*Dedup, error) {
104118
}
105119
dedupFile := filepath.Join(path, d.indexName+dedupSuffix)
106120

121+
_, err = os.Stat(dedupFile)
122+
if err != nil {
123+
if !os.IsNotExist(err) || !createIfNotExists {
124+
return nil, err
125+
}
126+
}
127+
107128
d.conn, err = bolt.Open(dedupFile, 0600, getBoltOptions(false))
108129
if err != nil {
109130
return nil, err
@@ -140,6 +161,7 @@ func (d *Dedup) Finish() error {
140161
func (d *Dedup) nextGen(tx *bolt.Tx) error {
141162
if sb := tx.Bucket(ddStatusBucketName); sb != nil {
142163
d.status.Gen += 1
164+
d.status.Updated = time.Now()
143165
if data, err := json.Marshal(&d.status); err == nil {
144166
return sb.Put(ddStatusName, data)
145167
} else {
@@ -501,7 +523,7 @@ func (d *Dedup) Dedup(hashes []string, verbose bool) error {
501523
list[i].Merged = true
502524
bag.SizeShared += uint64(bag.Size)
503525
bag.SizeExclusive -= uint64(bag.Size)
504-
d.ReclaimedTotal += reclaimed
526+
d.reclaimedTotal += reclaimed
505527
} else if IsNotSupported(err) {
506528
d.log(StatusPanic, "Dedupliate is not supported for this OS/fs, please see https://laktak.github.io/chkbit/dedup/")
507529
return err

0 commit comments

Comments
 (0)