Skip to content

Commit e655d47

Browse files
authored
Merge branch 'sprint-1.16' into doc/update
2 parents f4f0132 + d16854c commit e655d47

File tree

22 files changed

+88
-65
lines changed

22 files changed

+88
-65
lines changed

.github/workflows/gosdk.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ jobs:
1818
create-pr:
1919
runs-on: [arc-runner]
2020
steps:
21-
- name: Setup go 1.20
21+
- name: Setup go 1.21
2222
uses: actions/setup-go@v2
2323
with:
24-
go-version: '1.20'
24+
go-version: '1.21'
2525

2626
- name: Checkout
2727
uses: actions/checkout@v4

code/go/0chain.net/blobbercore/allocation/allocationchange.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func GetAllocationChanges(ctx context.Context, connectionID, allocationID, clien
135135
allocationID,
136136
clientID,
137137
DeletedConnection,
138-
).Preload("Changes").First(cc).Error
138+
).Preload("Changes").Take(cc).Error
139139

140140
if err == nil {
141141
cc.ComputeProperties()

code/go/0chain.net/blobbercore/allocation/connection.go

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"sync"
66
"time"
77

8-
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
98
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
109
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
1110
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/seqpriorityqueue"
@@ -166,7 +165,7 @@ func UpdateConnectionObjSize(connectionID string, addSize int64) {
166165
connectionObj.UpdatedAt = time.Now()
167166
}
168167

169-
func SaveFileChange(connectionID, pathHash, fileName string, cmd FileCommand, isFinal bool, contentSize int64, offset, dataWritten int64) (bool, error) {
168+
func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string, cmd FileCommand, isFinal bool, contentSize, offset, dataWritten, addSize int64) (bool, error) {
170169
connectionObjMutex.RLock()
171170
connectionObj := connectionProcessor[connectionID]
172171
connectionObjMutex.RUnlock()
@@ -175,31 +174,32 @@ func SaveFileChange(connectionID, pathHash, fileName string, cmd FileCommand, is
175174
}
176175
connectionObj.lock.Lock()
177176
connectionObj.UpdatedAt = time.Now()
178-
change := connectionObj.changes[pathHash]
179177
saveChange := false
178+
change := connectionObj.changes[pathHash]
180179
if change == nil {
181180
change = &ConnectionChange{}
182181
connectionObj.changes[pathHash] = change
183-
err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
184-
dbConnectionObj, err := GetAllocationChanges(ctx, connectionID, connectionObj.AllocationID, connectionObj.ClientID)
185-
if err != nil {
186-
return err
187-
}
188-
return cmd.UpdateChange(ctx, dbConnectionObj)
189-
})
182+
change.lock.Lock()
183+
defer change.lock.Unlock()
184+
connectionObj.lock.Unlock()
185+
dbConnectionObj, err := GetAllocationChanges(ctx, connectionID, connectionObj.AllocationID, connectionObj.ClientID)
190186
if err != nil {
191-
connectionObj.lock.Unlock()
192-
return false, err
187+
return saveChange, err
188+
}
189+
err = cmd.UpdateChange(ctx, dbConnectionObj)
190+
if err != nil {
191+
return saveChange, err
193192
}
194193
hasher := filestore.GetNewCommitHasher(contentSize)
195194
change.hasher = hasher
196195
change.seqPQ = seqpriorityqueue.NewSeqPriorityQueue(contentSize)
197196
go hasher.Start(connectionObj.ctx, connectionID, connectionObj.AllocationID, fileName, pathHash, change.seqPQ)
198197
saveChange = true
198+
} else {
199+
change.lock.Lock()
200+
defer change.lock.Unlock()
201+
connectionObj.lock.Unlock()
199202
}
200-
connectionObj.lock.Unlock()
201-
change.lock.Lock()
202-
defer change.lock.Unlock()
203203
if change.isFinalized {
204204
return false, nil
205205
}
@@ -210,6 +210,9 @@ func SaveFileChange(connectionID, pathHash, fileName string, cmd FileCommand, is
210210
Offset: offset,
211211
DataBytes: dataWritten,
212212
}, contentSize)
213+
if addSize != 0 {
214+
UpdateConnectionObjSize(connectionID, addSize)
215+
}
213216
} else {
214217
change.seqPQ.Push(seqpriorityqueue.UploadData{
215218
Offset: offset,

code/go/0chain.net/blobbercore/allocation/file_changer_base.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ type FileCommand interface {
8989
IsValidated(ctx context.Context, req *http.Request, allocationObj *Allocation, clientID string) error
9090

9191
// ProcessContent flush file to FileStorage
92-
ProcessContent(allocationObj *Allocation) (UploadResult, error)
92+
ProcessContent(ctx context.Context, allocationObj *Allocation) (UploadResult, error)
9393

9494
// ProcessThumbnail flush thumbnail file to FileStorage if it has.
9595
ProcessThumbnail(allocationObj *Allocation) error

code/go/0chain.net/blobbercore/challenge/protocol.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,19 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error {
286286
return
287287
}
288288

289+
if validationTicket.ValidatorID != validatorID {
290+
numFailed++
291+
logging.Logger.Error(
292+
"[challenge]resp: ",
293+
zap.String("validator",
294+
validatorID),
295+
zap.Any("resp", string(resp)),
296+
zap.Any("error", "Validator ID mismatch"),
297+
)
298+
updateMapAndSlice(validatorID, i, nil)
299+
return
300+
}
301+
289302
logging.Logger.Info(
290303
"[challenge]resp: Got response from the validator.",
291304
zap.Any("validator_response", validationTicket),

code/go/0chain.net/blobbercore/datastore/postgres.go

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ func (store *postgresStore) Open() error {
6767
}
6868

6969
sqldb.SetMaxIdleConns(100)
70-
sqldb.SetMaxOpenConns(200)
71-
sqldb.SetConnMaxLifetime(20 * time.Minute)
70+
sqldb.SetMaxOpenConns(400)
71+
sqldb.SetConnMaxLifetime(30 * time.Minute)
7272
sqldb.SetConnMaxIdleTime(5 * time.Minute)
7373
// Enable Logger, show detailed log
7474
//db.LogMode(true)
@@ -85,12 +85,7 @@ func (store *postgresStore) Close() {
8585
}
8686

8787
func (store *postgresStore) CreateTransaction(ctx context.Context) context.Context {
88-
//conn := ctx.Value(ContextKeyTransaction)
89-
//if conn != nil {
90-
// return ctx
91-
//}
92-
93-
db := store.db.Begin()
88+
db := store.db.WithContext(ctx).Begin()
9489
return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(db))
9590
}
9691

@@ -104,10 +99,13 @@ func (store *postgresStore) GetTransaction(ctx context.Context) *EnhancedDB {
10499
}
105100

106101
func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error) error {
107-
ctx := store.CreateTransaction(context.TODO())
108-
defer ctx.Done()
109-
102+
timeoutctx, cancel := context.WithTimeout(context.TODO(), 45*time.Second)
103+
defer cancel()
104+
ctx := store.CreateTransaction(timeoutctx)
110105
tx := store.GetTransaction(ctx)
106+
if tx.Error != nil {
107+
return tx.Error
108+
}
111109
err := f(ctx)
112110
if err != nil {
113111
tx.Rollback()
@@ -124,6 +122,9 @@ func (store *postgresStore) WithTransaction(ctx context.Context, f func(ctx cont
124122
if tx == nil {
125123
ctx = store.CreateTransaction(ctx)
126124
tx = store.GetTransaction(ctx)
125+
if tx.Error != nil {
126+
return tx.Error
127+
}
127128
}
128129

129130
err := f(ctx)

code/go/0chain.net/blobbercore/filestore/storage.go

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i
106106
_ = os.Remove(tempFilePath)
107107
return nil, common.NewError("file_size_mismatch", "File size is greater than expected")
108108
}
109-
logging.Logger.Info("temp_file_write: ", zap.String("filePath", fileData.Path), zap.Int64("currentSize", currentSize), zap.Int64("initialSize", initialSize), zap.Int64("writtenSize", writtenSize), zap.Int64("offset", fileData.UploadOffset), zap.Bool("ChunkUploaded", fileRef.ChunkUploaded))
109+
logging.Logger.Info("temp_file_write: ", zap.String("filePath", fileData.Path), zap.Int64("currentSize", currentSize), zap.Int64("initialSize", initialSize), zap.Int64("writtenSize", writtenSize), zap.Int64("offset", fileData.UploadOffset), zap.String("tempFilePath", tempFilePath))
110110
fileRef.Size = writtenSize
111111
fileRef.Name = fileData.Name
112112
fileRef.Path = fileData.Path
@@ -182,12 +182,6 @@ func (fs *FileStore) DeletePreCommitDir(allocID string) error {
182182
if err != nil {
183183
return common.NewError("pre_commit_dir_deletion_error", err.Error())
184184
}
185-
tempDir := fs.getAllocTempDir(allocID)
186-
err = os.RemoveAll(tempDir)
187-
if err != nil {
188-
return common.NewError("temp_dir_deletion_error", err.Error())
189-
}
190-
191185
return nil
192186
}
193187

@@ -398,7 +392,7 @@ func (fs *FileStore) DeleteTempFile(allocID, conID string, fd *FileInputData) er
398392
return common.NewError("invalid_allocation_id", "Allocation id cannot be empty")
399393
}
400394
fileObjectPath := fs.getTempPathForFile(allocID, fd.Name, encryption.Hash(fd.Path), conID)
401-
395+
logging.Logger.Info("deleting_temp_file", zap.String("fileObjectPath", fileObjectPath), zap.String("allocation_id", allocID), zap.String("connection_id", conID))
402396
finfo, err := os.Stat(fileObjectPath)
403397
if err != nil {
404398
if errors.Is(err, os.ErrNotExist) {

code/go/0chain.net/blobbercore/filestore/tree_validation.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil
427427
tempFilePath := GetFileStore().GetTempFilePath(allocID, connID, fileName, filePathHash)
428428
f, err := os.Open(tempFilePath)
429429
if err != nil {
430+
logging.Logger.Error("hasher_open", zap.Error(err), zap.String("tempFilePath", tempFilePath))
430431
c.hashErr = err
431432
return
432433
}

code/go/0chain.net/blobbercore/handler/file_command_delete.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func (cmd *DeleteFileCommand) UpdateChange(ctx context.Context, connectionObj *a
6868
}
6969

7070
// ProcessContent flush file to FileStorage
71-
func (cmd *DeleteFileCommand) ProcessContent(allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
71+
func (cmd *DeleteFileCommand) ProcessContent(_ context.Context, allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
7272
deleteSize := cmd.existingFileRef.Size
7373
connectionID := cmd.connectionID
7474
cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionID,

code/go/0chain.net/blobbercore/handler/file_command_update.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request
106106
}
107107

108108
// ProcessContent flush file to FileStorage
109-
func (cmd *UpdateFileCommand) ProcessContent(allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
109+
func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
110110
result := allocation.UploadResult{}
111111

112112
result.Filename = cmd.fileChanger.Filename
@@ -147,7 +147,6 @@ func (cmd *UpdateFileCommand) ProcessContent(allocationObj *allocation.Allocatio
147147
if fileOutputData.ContentSize != cmd.fileChanger.Size {
148148
return result, common.NewError("upload_error", fmt.Sprintf("File size mismatch. Expected: %d, Actual: %d", cmd.fileChanger.Size, fileOutputData.ContentSize))
149149
}
150-
allocation.UpdateConnectionObjSize(connID, cmd.fileChanger.Size-cmd.existingFileRef.Size)
151150
}
152151

153152
if cmd.thumbFile != nil {
@@ -157,7 +156,7 @@ func (cmd *UpdateFileCommand) ProcessContent(allocationObj *allocation.Allocatio
157156
}
158157
}
159158

160-
saveChange, err := allocation.SaveFileChange(connID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size)
159+
saveChange, err := allocation.SaveFileChange(ctx, connID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size)
161160
if err != nil {
162161
return result, err
163162
}

0 commit comments

Comments
 (0)