Skip to content

update gosdk #185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Mar 15, 2025
Merged

update gosdk #185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9f27e88
update gosdk
Hitenjain14 Feb 25, 2025
451bf12
add batch logs
Hitenjain14 Feb 26, 2025
2dbdf5b
process half batch
Hitenjain14 Feb 26, 2025
b1c7dc5
remove half batch
Hitenjain14 Feb 26, 2025
75bb73d
add retries for ref path resp
Hitenjain14 Mar 1, 2025
15a23ae
increase download chan size
Hitenjain14 Mar 1, 2025
09bd0bf
add retry for unexpected eof
Hitenjain14 Mar 11, 2025
3ad90d4
add option to time request
Hitenjain14 Mar 12, 2025
f9a4cfb
check if bit is set before releasing buffer
Hitenjain14 Mar 12, 2025
93b945f
update gosdk
Hitenjain14 Mar 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cmd/gateway/zcn/gateway-zcn.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func (z *ZCN) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, erro
if err != nil {
return nil, err
}
log.Println("0chain gosdk initialized: ", allocationID, "compress: ", compress, "encrypt: ", encrypt)
log.Println("0chain gosdk initialized: ", allocationID, "compress: ", compress, "encrypt: ", encrypt, serverConfig.UploadWorkers, serverConfig.DownloadWorkers, serverConfig.MaxConcurrentRequests)
if serverConfig.UploadWorkers > 0 {
sdk.SetHighModeWorkers(serverConfig.UploadWorkers)
}
Expand All @@ -162,6 +162,7 @@ func (z *ZCN) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, erro
sdk.SetSingleClietnMode(true)
sdk.SetShouldVerifyHash(false)
sdk.SetSaveProgress(false)
sdk.SetShouldTimeRequest(false)
allocation.SetCheckStatus(true)
zob := &zcnObjects{
alloc: allocation,
Expand Down
6 changes: 4 additions & 2 deletions cmd/gateway/zcn/worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ var (
)

func IntiBatchUploadWorkers(ctx context.Context, alloc *sdk.Allocation, waitTime, maxOperations, maxWorkers int) {

log.Println("start upload worker: ", waitTime, maxOperations, maxWorkers)
batchUploadChan = make(chan sdk.OperationRequest, maxOperations*maxWorkers)
workerChan = make(chan []sdk.OperationRequest, maxWorkers)

Expand All @@ -33,7 +33,7 @@ func IntiBatchUploadWorkers(ctx context.Context, alloc *sdk.Allocation, waitTime
// process the batch upload or wait for more operations
opRequest = append(opRequest, op)
if len(opRequest) == maxOperations || iterations > 2 {
log.Println("process batch for time condition")
log.Println("process batch for time condition", iterations)
workerChan <- opRequest
opRequest = make([]sdk.OperationRequest, 0, 5)
iterations = 0
Expand Down Expand Up @@ -68,6 +68,7 @@ func batchUploadWorker(ctx context.Context, alloc *sdk.Allocation, opsChan chan
return
case ops := <-opsChan:
// process the batch upload or wait for more operations
now := time.Now()
log.Println("processing batch upload: ", len(ops))
err := alloc.DoMultiOperation(ops)
if err != nil {
Expand All @@ -80,6 +81,7 @@ func batchUploadWorker(ctx context.Context, alloc *sdk.Allocation, opsChan chan
for _, op := range ops {
op.CancelCauseFunc(err)
}
log.Println("finish batch upload: ", len(ops), " time", time.Since(now).Milliseconds())
}
}
}
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ toolchain go1.22.5
require (
cloud.google.com/go/storage v1.28.1
github.com/0chain/errors v1.0.3
github.com/0chain/gosdk v1.19.6
github.com/0chain/gosdk v1.19.10
github.com/Azure/azure-pipeline-go v0.2.2
github.com/Azure/azure-storage-blob-go v0.10.0
github.com/Shopify/sarama v1.28.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ github.com/0chain/common v1.18.3 h1:42dYOv2KyMTSanuS67iDtfv+ErbSRqR8NJ3MG72MwaI=
github.com/0chain/common v1.18.3/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw=
github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
github.com/0chain/gosdk v1.19.6 h1:GfNdBBC0uQqaooT4Vh6B7FP1m76lyTagugizwfYYleQ=
github.com/0chain/gosdk v1.19.6/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw=
github.com/0chain/gosdk v1.19.10 h1:etxcGRcPebsJi/GUgmT0LyOm+pFsIb/kDodGGLPQu+w=
github.com/0chain/gosdk v1.19.10/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw=
github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU=
github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=
Expand Down
6 changes: 4 additions & 2 deletions go.work.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,8 +515,10 @@ github.com/0chain/gosdk v1.15.5 h1:e+khk21/YrjlxuvQPFCI2JsQbgSCyr/Y/Km82HCH+MM=
github.com/0chain/gosdk v1.15.5/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg=
github.com/0chain/gosdk v1.16.0 h1:3CKuU9i9d+X2/htJOIyxNwviKG5H/lp+q8ogzCgnoQI=
github.com/0chain/gosdk v1.16.0/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg=
github.com/0chain/gosdk v1.19.6 h1:GfNdBBC0uQqaooT4Vh6B7FP1m76lyTagugizwfYYleQ=
github.com/0chain/gosdk v1.19.6/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw=
github.com/0chain/gosdk v1.19.8 h1:xjC0WA90hd8ytPfdp7zzSe2sP/Cp8ZgSNqKMPeZD06E=
github.com/0chain/gosdk v1.19.8/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw=
github.com/0chain/gosdk v1.19.10 h1:etxcGRcPebsJi/GUgmT0LyOm+pFsIb/kDodGGLPQu+w=
github.com/0chain/gosdk v1.19.10/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw=
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4=
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/Azure/azure-amqp-common-go/v2 v2.1.0 h1:+QbFgmWCnPzdaRMfsI0Yb6GrRdBj5jVL8N3EXuEUcBQ=
Expand Down