From 768cdbd8d751bebc75b60398a6fcc46ff3436737 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Sat, 19 Oct 2024 10:33:33 +0530 Subject: [PATCH 1/2] increase time gap for auth ticket and check for final --- code/go/0chain.net/blobbercore/allocation/connection.go | 5 +++++ .../blobbercore/allocation/file_changer_base.go | 4 +++- .../blobbercore/allocation/file_changer_update.go | 9 +++++++++ .../blobbercore/allocation/file_changer_upload.go | 8 ++++++++ code/go/0chain.net/blobbercore/readmarker/authticket.go | 2 +- .../blobbercore/seqpriorityqueue/seqpriorityqueue.go | 3 +++ 6 files changed, 29 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index ef7eac62f..207ae591d 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -244,6 +244,11 @@ func DeleteConnectionObjEntry(connectionID string) { connectionObj, ok := connectionProcessor[connectionID] if ok { connectionObj.cnclCtx() + for _, change := range connectionObj.changes { + if change.seqPQ != nil { + change.seqPQ.Done(seqpriorityqueue.UploadData{}, 1) + } + } } delete(connectionProcessor, connectionID) connectionObjMutex.Unlock() diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 1d86e30d1..28b6cb4db 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -122,7 +122,9 @@ func (fc *BaseFileChanger) DeleteTempFile() error { } func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { - + if !fc.IsFinal { + return nil + } if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{ StorageVersion: fc.storageVersion, diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 590d25d60..155d8f798 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -42,6 +42,9 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference if err != nil { return nil, err } + if !nf.IsFinal { + return rootRef, nil + } rootRef.HashToBeComputed = true rootRef.UpdatedAt = ts @@ -114,6 +117,9 @@ func (nf *UpdateFileChanger) ApplyChangeV2(ctx context.Context, allocationRoot, if nf.AllocationID == "" { return 0, common.NewError("invalid_allocation_id", "Allocation ID is empty") } + if !nf.IsFinal { + return 0, nil + } nf.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) @@ -190,6 +196,9 @@ func (nf *UpdateFileChanger) ApplyChangeV2(ctx context.Context, allocationRoot, } func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { + if !nf.IsFinal { + return nil + } return nf.BaseFileChanger.CommitToFileStore(ctx, mut) } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 59936dd10..bd51ad85d 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -34,6 +34,10 @@ type UploadFileChanger struct { func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { + if !nf.IsFinal { + return rootRef, nil + } + totalRefs, err := reference.CountRefs(ctx, nf.AllocationID) if err != nil { return nil, err @@ -148,6 +152,10 @@ func (nf *UploadFileChanger) ApplyChangeV2(ctx context.Context, allocationRoot, if nf.AllocationID == "" { return 0, common.NewError("invalid_allocation_id", "Allocation ID is empty") } + if !nf.IsFinal { + return 0, nil + } + //find if ref exists var refResult struct { ID int64 diff --git a/code/go/0chain.net/blobbercore/readmarker/authticket.go b/code/go/0chain.net/blobbercore/readmarker/authticket.go index af05624a6..e2c0a9033 100644 --- a/code/go/0chain.net/blobbercore/readmarker/authticket.go +++ b/code/go/0chain.net/blobbercore/readmarker/authticket.go @@ -67,7 +67,7 @@ func (authToken *AuthTicket) Verify(allocationObj *allocation.Allocation, client if authToken.OwnerID != allocationObj.OwnerID { return common.NewError("invalid_parameters", "Invalid auth ticket. Owner ID mismatch") } - if authToken.Timestamp > (common.Now() + 2) { + if authToken.Timestamp > (common.Now() + 120) { return common.NewError("invalid_parameters", "Invalid auth ticket. Timestamp in future") } diff --git a/code/go/0chain.net/blobbercore/seqpriorityqueue/seqpriorityqueue.go b/code/go/0chain.net/blobbercore/seqpriorityqueue/seqpriorityqueue.go index bbb3cb194..c9ea2b18a 100644 --- a/code/go/0chain.net/blobbercore/seqpriorityqueue/seqpriorityqueue.go +++ b/code/go/0chain.net/blobbercore/seqpriorityqueue/seqpriorityqueue.go @@ -70,6 +70,9 @@ func (pq *SeqPriorityQueue) Push(v UploadData) { } func (pq *SeqPriorityQueue) Done(v UploadData, dataSize int64) { + if pq.done { + return + } pq.lock.Lock() pq.done = true pq.dataSize = dataSize From 56cac1243c7e352258d279760028246d7339d2c9 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Sat, 19 Oct 2024 10:59:14 +0530 Subject: [PATCH 2/2] fix ut --- .../blobbercore/allocation/file_changer_upload_test.go | 1 + code/go/0chain.net/blobbercore/allocation/multiop_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go index 4ea4e3bce..5ca7e4086 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go @@ -118,6 +118,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { Size: 2310, ChunkSize: 65536, ConnectionID: "connection_id", + IsFinal: true, }, } diff --git a/code/go/0chain.net/blobbercore/allocation/multiop_test.go b/code/go/0chain.net/blobbercore/allocation/multiop_test.go index 917cd9030..d52fddd52 100644 --- a/code/go/0chain.net/blobbercore/allocation/multiop_test.go +++ b/code/go/0chain.net/blobbercore/allocation/multiop_test.go @@ -87,6 +87,7 @@ func uploadChanges(allocationID, validationRoot string, fileIDMeta map[string]st ValidationRoot: validationRoot, Size: 2310, ChunkSize: 65536, + IsFinal: true, }, } fileIDMeta[filepath.Join("/", change.Filename)] = randName() @@ -112,6 +113,7 @@ func getUpdateChange(uploadChanger *UploadFileChanger) *UpdateFileChanger { ValidationRoot: "updated_validation_root", Size: uploadChanger.Size, ChunkSize: uploadChanger.ChunkSize, + IsFinal: true, }, } }