Skip to content

Commit 3a0dd3c

Browse files
dabasovdabasov
authored
Merge pull request #1430 from 0chain/feat/ignore-ref
Ignore ref field in refpath response and db connection fix
2 parents 3314e6a + 6ceae9f commit 3a0dd3c

10 files changed

+39
-38
lines changed

‎code/go/0chain.net/blobbercore/allocation/allocationchange.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func GetAllocationChanges(ctx context.Context, connectionID, allocationID, clien
135135
allocationID,
136136
clientID,
137137
DeletedConnection,
138-
).Preload("Changes").First(cc).Error
138+
).Preload("Changes").Take(cc).Error
139139

140140
if err == nil {
141141
cc.ComputeProperties()

‎code/go/0chain.net/blobbercore/allocation/connection.go

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"sync"
66
"time"
77

8-
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
98
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
109
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
1110
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/seqpriorityqueue"
@@ -166,7 +165,7 @@ func UpdateConnectionObjSize(connectionID string, addSize int64) {
166165
connectionObj.UpdatedAt = time.Now()
167166
}
168167

169-
func SaveFileChange(connectionID, pathHash, fileName string, cmd FileCommand, isFinal bool, contentSize, offset, dataWritten, addSize int64) (bool, error) {
168+
func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string, cmd FileCommand, isFinal bool, contentSize, offset, dataWritten, addSize int64) (bool, error) {
170169
connectionObjMutex.RLock()
171170
connectionObj := connectionProcessor[connectionID]
172171
connectionObjMutex.RUnlock()
@@ -175,31 +174,32 @@ func SaveFileChange(connectionID, pathHash, fileName string, cmd FileCommand, is
175174
}
176175
connectionObj.lock.Lock()
177176
connectionObj.UpdatedAt = time.Now()
178-
change := connectionObj.changes[pathHash]
179177
saveChange := false
178+
change := connectionObj.changes[pathHash]
180179
if change == nil {
181180
change = &ConnectionChange{}
182181
connectionObj.changes[pathHash] = change
183-
err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
184-
dbConnectionObj, err := GetAllocationChanges(ctx, connectionID, connectionObj.AllocationID, connectionObj.ClientID)
185-
if err != nil {
186-
return err
187-
}
188-
return cmd.UpdateChange(ctx, dbConnectionObj)
189-
})
182+
change.lock.Lock()
183+
defer change.lock.Unlock()
184+
connectionObj.lock.Unlock()
185+
dbConnectionObj, err := GetAllocationChanges(ctx, connectionID, connectionObj.AllocationID, connectionObj.ClientID)
190186
if err != nil {
191-
connectionObj.lock.Unlock()
192-
return false, err
187+
return saveChange, err
188+
}
189+
err = cmd.UpdateChange(ctx, dbConnectionObj)
190+
if err != nil {
191+
return saveChange, err
193192
}
194193
hasher := filestore.GetNewCommitHasher(contentSize)
195194
change.hasher = hasher
196195
change.seqPQ = seqpriorityqueue.NewSeqPriorityQueue(contentSize)
197196
go hasher.Start(connectionObj.ctx, connectionID, connectionObj.AllocationID, fileName, pathHash, change.seqPQ)
198197
saveChange = true
198+
} else {
199+
connectionObj.lock.Unlock()
200+
change.lock.Lock()
201+
defer change.lock.Unlock()
199202
}
200-
connectionObj.lock.Unlock()
201-
change.lock.Lock()
202-
defer change.lock.Unlock()
203203
if change.isFinalized {
204204
return false, nil
205205
}

‎code/go/0chain.net/blobbercore/allocation/file_changer_base.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ type FileCommand interface {
8888
IsValidated(ctx context.Context, req *http.Request, allocationObj *Allocation, clientID string) error
8989

9090
// ProcessContent flush file to FileStorage
91-
ProcessContent(allocationObj *Allocation) (UploadResult, error)
91+
ProcessContent(ctx context.Context, allocationObj *Allocation) (UploadResult, error)
9292

9393
// ProcessThumbnail flush thumbnail file to FileStorage if it has.
9494
ProcessThumbnail(allocationObj *Allocation) error

‎code/go/0chain.net/blobbercore/datastore/postgres.go

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ func (store *postgresStore) Open() error {
6767
}
6868

6969
sqldb.SetMaxIdleConns(100)
70-
sqldb.SetMaxOpenConns(200)
71-
sqldb.SetConnMaxLifetime(20 * time.Minute)
70+
sqldb.SetMaxOpenConns(400)
71+
sqldb.SetConnMaxLifetime(30 * time.Minute)
7272
sqldb.SetConnMaxIdleTime(5 * time.Minute)
7373
// Enable Logger, show detailed log
7474
//db.LogMode(true)
@@ -85,12 +85,7 @@ func (store *postgresStore) Close() {
8585
}
8686

8787
func (store *postgresStore) CreateTransaction(ctx context.Context) context.Context {
88-
//conn := ctx.Value(ContextKeyTransaction)
89-
//if conn != nil {
90-
// return ctx
91-
//}
92-
93-
db := store.db.Begin()
88+
db := store.db.WithContext(ctx).Begin()
9489
return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(db))
9590
}
9691

@@ -104,10 +99,13 @@ func (store *postgresStore) GetTransaction(ctx context.Context) *EnhancedDB {
10499
}
105100

106101
func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error) error {
107-
ctx := store.CreateTransaction(context.TODO())
108-
defer ctx.Done()
109-
102+
timeoutctx, cancel := context.WithTimeout(context.TODO(), 45*time.Second)
103+
defer cancel()
104+
ctx := store.CreateTransaction(timeoutctx)
110105
tx := store.GetTransaction(ctx)
106+
if tx.Error != nil {
107+
return tx.Error
108+
}
111109
err := f(ctx)
112110
if err != nil {
113111
tx.Rollback()
@@ -124,6 +122,9 @@ func (store *postgresStore) WithTransaction(ctx context.Context, f func(ctx cont
124122
if tx == nil {
125123
ctx = store.CreateTransaction(ctx)
126124
tx = store.GetTransaction(ctx)
125+
if tx.Error != nil {
126+
return tx.Error
127+
}
127128
}
128129

129130
err := f(ctx)

‎code/go/0chain.net/blobbercore/handler/file_command_delete.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func (cmd *DeleteFileCommand) UpdateChange(ctx context.Context, connectionObj *a
6868
}
6969

7070
// ProcessContent flush file to FileStorage
71-
func (cmd *DeleteFileCommand) ProcessContent(allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
71+
func (cmd *DeleteFileCommand) ProcessContent(_ context.Context, allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
7272
deleteSize := cmd.existingFileRef.Size
7373
connectionID := cmd.connectionID
7474
cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionID,

‎code/go/0chain.net/blobbercore/handler/file_command_update.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request
106106
}
107107

108108
// ProcessContent flush file to FileStorage
109-
func (cmd *UpdateFileCommand) ProcessContent(allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
109+
func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
110110
result := allocation.UploadResult{}
111111

112112
result.Filename = cmd.fileChanger.Filename
@@ -156,7 +156,7 @@ func (cmd *UpdateFileCommand) ProcessContent(allocationObj *allocation.Allocatio
156156
}
157157
}
158158

159-
saveChange, err := allocation.SaveFileChange(connID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size)
159+
saveChange, err := allocation.SaveFileChange(ctx, connID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size)
160160
if err != nil {
161161
return result, err
162162
}

‎code/go/0chain.net/blobbercore/handler/file_command_upload.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request
122122
}
123123

124124
// ProcessContent flush file to FileStorage
125-
func (cmd *UploadFileCommand) ProcessContent(allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
125+
func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj *allocation.Allocation) (allocation.UploadResult, error) {
126126
logging.Logger.Info("UploadFileCommand.ProcessContent", zap.Any("fileChanger", cmd.fileChanger.Path), zap.Any("uploadOffset", cmd.fileChanger.UploadOffset), zap.Any("isFinal", cmd.fileChanger.IsFinal))
127127
result := allocation.UploadResult{}
128128
defer cmd.contentFile.Close()
@@ -174,7 +174,7 @@ func (cmd *UploadFileCommand) ProcessContent(allocationObj *allocation.Allocatio
174174
}
175175
}
176176

177-
saveChange, err := allocation.SaveFileChange(connectionID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size)
177+
saveChange, err := allocation.SaveFileChange(ctx, connectionID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size)
178178
if err != nil {
179179
logging.Logger.Error("UploadFileCommand.ProcessContent", zap.Error(err))
180180
return result, err

‎code/go/0chain.net/blobbercore/handler/handler_test.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -847,7 +847,6 @@ func TestHandlers_Requiring_Signature(t *testing.T) {
847847
sqlmock.NewRows([]string{"found"}).
848848
AddRow(false),
849849
)
850-
mock.ExpectBegin()
851850
mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocation_connections" WHERE`)).
852851
WithArgs(connectionID, alloc.ID, alloc.OwnerID, allocation.DeletedConnection).
853852
WillReturnRows(

‎code/go/0chain.net/blobbercore/handler/object_operation_handler.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1285,7 +1285,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*all
12851285
return nil, err
12861286
}
12871287
// call process content, which writes to file checks if conn obj needs to be updated and if commit hasher needs to be called
1288-
res, err := cmd.ProcessContent(allocationObj)
1288+
res, err := cmd.ProcessContent(ctx, allocationObj)
12891289
if err != nil {
12901290
return nil, err
12911291
}
@@ -1415,8 +1415,9 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob
14151415
}
14161416

14171417
elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM
1418-
c := datastore.GetStore().CreateTransaction(context.TODO())
1419-
defer c.Done()
1418+
timeoutCtx, cancel := context.WithTimeout(ctx, 45*time.Second)
1419+
defer cancel()
1420+
c := datastore.GetStore().CreateTransaction(timeoutCtx)
14201421
txn := datastore.GetStore().GetTransaction(c)
14211422
err = allocation.ApplyRollback(c, allocationID)
14221423
if err != nil {

‎code/go/0chain.net/blobbercore/reference/referencepath.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
type ReferencePath struct {
1717
Meta map[string]interface{} `json:"meta_data"`
1818
List []*ReferencePath `json:"list,omitempty"`
19-
Ref *Ref
19+
Ref *Ref `json:"-"`
2020
}
2121

2222
func GetReferencePath(ctx context.Context, allocationID, path string) (*Ref, error) {

0 commit comments

Comments
 (0)