Skip to content

Commit 3d7b8d7

Browse files
devyetiidevyetii
committed
Merge branch 'sprint-1.14' into doc/update
2 parents fcc1098 + 63789a9 commit 3d7b8d7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+1047
-645
lines changed

‎code/go/0chain.net/blobber/http.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ func initHandlers(r *mux.Router, devMode bool) {
110110
handler.SetupHandlers(r)
111111
handler.SetupSwagger()
112112
common.SetAdminCredentials(devMode)
113+
common.Set0boxDetails()
113114
}
114115

115116
func initProfHandlers(mux *http.ServeMux) {

‎code/go/0chain.net/blobbercore/allocation/allocationchange.go

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ type Result struct {
265265
PrevValidationRoot string
266266
ThumbnailHash string
267267
PrevThumbnailHash string
268+
FilestoreVersion int
268269
}
269270

270271
// TODO: Need to speed up this function
@@ -281,7 +282,7 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error {
281282

282283
err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
283284
tx := datastore.GetStore().GetTransaction(ctx)
284-
err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE).
285+
err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash", "filestore_version").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE).
285286
FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error {
286287

287288
for _, ref := range refs {
@@ -303,27 +304,27 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error {
303304
}()
304305

305306
if count == 0 && ref.PrevValidationRoot != "" {
306-
err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevValidationRoot)
307+
err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevValidationRoot, ref.FilestoreVersion)
307308
if err != nil {
308309
logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
309310
zap.String("validation_root", ref.ValidationRoot))
310311
}
311312
}
312-
err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ValidationRoot)
313+
err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ValidationRoot, ref.FilestoreVersion)
313314
if err != nil {
314315
logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error()),
315316
zap.String("validation_root", ref.ValidationRoot))
316317
}
317318

318319
if ref.ThumbnailHash != "" && ref.ThumbnailHash != ref.PrevThumbnailHash {
319320
if ref.PrevThumbnailHash != "" {
320-
err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevThumbnailHash)
321+
err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevThumbnailHash, ref.FilestoreVersion)
321322
if err != nil {
322323
logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail file: %s", err.Error()),
323324
zap.String("thumbnail_hash", ref.ThumbnailHash))
324325
}
325326
}
326-
err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ThumbnailHash)
327+
err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ThumbnailHash, ref.FilestoreVersion)
327328
if err != nil {
328329
logging.Logger.Error(fmt.Sprintf("Error while moving thumbnail file: %s", err.Error()),
329330
zap.String("thumbnail_hash", ref.ThumbnailHash))
@@ -380,15 +381,16 @@ func deleteFromFileStore(ctx context.Context, allocationID string) error {
380381
}()
381382

382383
if count == 0 {
383-
err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ValidationRoot)
384+
err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ValidationRoot,
385+
res.FilestoreVersion)
384386
if err != nil {
385387
logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
386388
zap.String("validation_root", res.ValidationRoot))
387389
}
388390
}
389391

390392
if res.ThumbnailHash != "" {
391-
err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ThumbnailHash)
393+
err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ThumbnailHash, res.FilestoreVersion)
392394
if err != nil {
393395
logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail: %s", err.Error()),
394396
zap.String("thumbnail", res.ThumbnailHash))

‎code/go/0chain.net/blobbercore/allocation/common_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,22 +37,22 @@ func (mfs *MockFileStore) CommitWrite(allocID, connID string, fileData *filestor
3737
return true, nil
3838
}
3939

40-
func (mfs *MockFileStore) MoveToFilestore(allocID, hash string) error {
40+
func (mfs *MockFileStore) MoveToFilestore(allocID, hash string, version int) error {
4141
return nil
4242
}
4343

4444
func (mfs *MockFileStore) DeleteAllocation(allocID string) {
4545
}
4646

47-
func (mfs *MockFileStore) DeleteFromFilestore(allocID, hash string) error {
47+
func (mfs *MockFileStore) DeleteFromFilestore(allocID, hash string, version int) error {
4848
return nil
4949
}
5050

5151
func (mfs *MockFileStore) DeleteTempFile(allocID, connID string, fileData *filestore.FileInputData) error {
5252
return nil
5353
}
5454

55-
func (mfs *MockFileStore) DeleteFile(allocID, contentHash string) error {
55+
func (mfs *MockFileStore) DeleteFile(allocID, contentHash string, version int) error {
5656
return nil
5757
}
5858

@@ -64,7 +64,7 @@ func (mfs *MockFileStore) GetFileBlock(rin *filestore.ReadBlockInput) (*filestor
6464
return nil, nil
6565
}
6666

67-
func (mfs *MockFileStore) GetFilePathSize(allocID, contentHash, thumbHash string) (int64, int64, error) {
67+
func (mfs *MockFileStore) GetFilePathSize(allocID, contentHash, thumbHash string, version int) (int64, int64, error) {
6868
return 0, 0, nil
6969
}
7070

@@ -112,7 +112,7 @@ func (mfs *MockFileStore) CalculateCurrentDiskCapacity() error {
112112
return nil
113113
}
114114

115-
func (mfs *MockFileStore) GetPathForFile(allocID, contentHash string) (string, error) {
115+
func (mfs *MockFileStore) GetPathForFile(allocID, contentHash string, version int) (string, error) {
116116
return "", nil
117117
}
118118

‎code/go/0chain.net/blobbercore/allocation/connection.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ func SaveFileChange(connectionID, pathHash, fileName string, cmd FileCommand, is
174174
return false, common.NewError("connection_not_found", "connection not found for save file change")
175175
}
176176
connectionObj.lock.Lock()
177+
connectionObj.UpdatedAt = time.Now()
177178
change := connectionObj.changes[pathHash]
178179
saveChange := false
179180
if change == nil {
@@ -208,7 +209,7 @@ func SaveFileChange(connectionID, pathHash, fileName string, cmd FileCommand, is
208209
change.seqPQ.Done(seqpriorityqueue.UploadData{
209210
Offset: offset,
210211
DataBytes: dataWritten,
211-
})
212+
}, contentSize)
212213
} else {
213214
change.seqPQ.Push(seqpriorityqueue.UploadData{
214215
Offset: offset,
@@ -256,7 +257,7 @@ func cleanConnectionObj() {
256257
connectionObj.cnclCtx()
257258
for _, change := range connectionObj.changes {
258259
if change.seqPQ != nil {
259-
change.seqPQ.Done(seqpriorityqueue.UploadData{})
260+
change.seqPQ.Done(seqpriorityqueue.UploadData{}, 1)
260261
}
261262
}
262263
delete(connectionProcessor, connectionID)

‎code/go/0chain.net/blobbercore/allocation/deletefilechange.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,17 +61,18 @@ func (nf *DeleteFileChange) DeleteTempFile() error {
6161
func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error {
6262
db := datastore.GetStore().GetTransaction(ctx)
6363
type Result struct {
64-
Id string
65-
ValidationRoot string
66-
ThumbnailHash string
64+
Id string
65+
ValidationRoot string
66+
ThumbnailHash string
67+
FilestoreVersion int
6768
}
6869

6970
limitCh := make(chan struct{}, 10)
7071
wg := &sync.WaitGroup{}
7172
var results []Result
7273
mut.Lock()
7374
err := db.Model(&reference.Ref{}).Unscoped().
74-
Select("id", "validation_root", "thumbnail_hash").
75+
Select("id", "validation_root", "thumbnail_hash", "filestore_version").
7576
Where("allocation_id=? AND path LIKE ? AND type=? AND deleted_at is not NULL",
7677
nf.AllocationID, nf.Path+"%", reference.FILE).
7778
FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error {
@@ -96,7 +97,7 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mut
9697
}()
9798

9899
if count == 0 {
99-
err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ValidationRoot)
100+
err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ValidationRoot, res.FilestoreVersion)
100101
if err != nil {
101102
logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
102103
zap.String("validation_root", res.ValidationRoot))

‎code/go/0chain.net/blobbercore/allocation/entity.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ type Allocation struct {
5353
BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"`
5454
BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"`
5555
LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"`
56+
LastRedeemedSeq int64 `gorm:"column:last_redeemed_sequence;default:0"`
5657
IsRedeemRequired bool `gorm:"column:is_redeem_required"`
5758
TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"`
5859
StartTime common.Timestamp `gorm:"column:start_time;not null"`

‎code/go/0chain.net/blobbercore/allocation/file_changer_update.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference
7878
nf.deleteHash = make(map[string]int)
7979

8080
if fileRef.ValidationRoot != "" && fileRef.ValidationRoot != nf.ValidationRoot {
81-
nf.deleteHash[fileRef.ValidationRoot] = int(CONTENT)
81+
nf.deleteHash[fileRef.ValidationRoot] = fileRef.FilestoreVersion
8282
}
8383

8484
fileRef.ActualFileHash = nf.ActualHash
@@ -99,13 +99,14 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference
9999
fileRef.EncryptedKeyPoint = nf.EncryptedKeyPoint
100100
fileRef.ChunkSize = nf.ChunkSize
101101
fileRef.IsPrecommit = true
102+
fileRef.FilestoreVersion = filestore.VERSION
102103

103104
return rootRef, nil
104105
}
105106

106107
func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error {
107108
db := datastore.GetStore().GetTransaction(ctx)
108-
for hash := range nf.deleteHash {
109+
for hash, version := range nf.deleteHash {
109110
var count int64
110111
mut.Lock()
111112
err := db.Table((&reference.Ref{}).TableName()).
@@ -115,7 +116,7 @@ func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mu
115116
mut.Unlock()
116117
if err == nil && count == 0 {
117118
logging.Logger.Info("Deleting content file", zap.String("validation_root", hash))
118-
if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, hash); err != nil {
119+
if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, hash, version); err != nil {
119120
logging.Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err))
120121
}
121122
}

‎code/go/0chain.net/blobbercore/allocation/file_changer_upload.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"strings"
99

1010
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
11+
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
1112

1213
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
1314
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/util"
@@ -119,6 +120,7 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference
119120
UpdatedAt: ts,
120121
HashToBeComputed: true,
121122
IsPrecommit: true,
123+
FilestoreVersion: filestore.VERSION,
122124
}
123125

124126
fileID, ok := fileIDMeta[newFile.Path]

‎code/go/0chain.net/blobbercore/allocation/repository.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ func (r *Repository) GetAllocationIds(ctx context.Context) []Res {
191191

192192
}
193193

194-
func (r *Repository) UpdateAllocationRedeem(ctx context.Context, allocationID, AllocationRoot string, allocationObj *Allocation) error {
194+
func (r *Repository) UpdateAllocationRedeem(ctx context.Context, allocationID, AllocationRoot string, allocationObj *Allocation, redeemSeq int64) error {
195195
var tx = datastore.GetStore().GetTransaction(ctx)
196196
if tx == nil {
197197
logging.Logger.Panic("no transaction in the context")
@@ -205,17 +205,20 @@ func (r *Repository) UpdateAllocationRedeem(ctx context.Context, allocationID, A
205205
allocationUpdates := make(map[string]interface{})
206206
allocationUpdates["latest_redeemed_write_marker"] = AllocationRoot
207207
allocationUpdates["is_redeem_required"] = false
208+
allocationUpdates["last_redeemed_sequence"] = redeemSeq
208209
err = tx.Model(allocationObj).Updates(allocationUpdates).Error
209210
if err != nil {
210211
return err
211212
}
212213
allocationObj.LatestRedeemedWM = AllocationRoot
213214
allocationObj.IsRedeemRequired = false
215+
allocationObj.LastRedeemedSeq = redeemSeq
214216
txnCache := cache[allocationID]
215217
txnCache.Allocation = allocationObj
216218
updateAlloc := func(a *Allocation) {
217219
a.LatestRedeemedWM = AllocationRoot
218220
a.IsRedeemRequired = false
221+
a.LastRedeemedSeq = redeemSeq
219222
}
220223
txnCache.AllocationUpdates = append(txnCache.AllocationUpdates, updateAlloc)
221224
cache[allocationID] = txnCache

‎code/go/0chain.net/blobbercore/blobberhttp/response.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,18 @@ type ConnectionResult struct {
1515

1616
// swagger:model CommitResult
1717
type CommitResult struct {
18-
AllocationRoot string `json:"allocation_root"`
19-
WriteMarker *writemarker.WriteMarker `json:"write_marker"`
20-
Success bool `json:"success"`
21-
ErrorMessage string `json:"error_msg,omitempty"`
18+
AllocationRoot string `json:"allocation_root"`
19+
WriteMarker *writemarker.WriteMarkerEntity `json:"write_marker"`
20+
Success bool `json:"success"`
21+
ErrorMessage string `json:"error_msg,omitempty"`
2222
//Result []*UploadResult `json:"result"`
2323
}
2424

2525
// swagger:model ReferencePathResult
2626
type ReferencePathResult struct {
2727
*reference.ReferencePath
2828
LatestWM *writemarker.WriteMarker `json:"latest_write_marker"`
29+
Version string `json:"version"`
2930
}
3031

3132
// swagger:model RefResult
@@ -65,4 +66,5 @@ type DownloadResponse struct {
6566
type LatestWriteMarkerResult struct {
6667
LatestWM *writemarker.WriteMarker `json:"latest_write_marker"`
6768
PrevWM *writemarker.WriteMarker `json:"prev_write_marker"`
69+
Version string `json:"version"`
6870
}

0 commit comments

Comments
 (0)