diff --git a/code/go/0chain.net/blobber/logging.go b/code/go/0chain.net/blobber/logging.go
index 339e8807c..98d38e38e 100644
--- a/code/go/0chain.net/blobber/logging.go
+++ b/code/go/0chain.net/blobber/logging.go
@@ -5,7 +5,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk_common/zcncore"
)
func setupLogging() {
diff --git a/code/go/0chain.net/blobber/settings.go b/code/go/0chain.net/blobber/settings.go
index eb90322b2..72f631f16 100644
--- a/code/go/0chain.net/blobber/settings.go
+++ b/code/go/0chain.net/blobber/settings.go
@@ -9,7 +9,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk_common/core/transaction"
)
type storageScCB struct {
@@ -58,20 +58,24 @@ func (ssc *storageScCB) OnInfoAvailable(op int, status int, info string, errStr
}
func setStorageScConfigFromChain() error {
- cb := &storageScCB{
- done: make(chan struct{}),
+ conf, err := transaction.GetConfig("storage_sc_config")
+ if err != nil {
+ return err
}
- err := zcncore.GetStorageSCConfig(cb)
+
+ maxChallengeCompletionRoundsString := conf.Fields["max_challenge_completion_rounds"]
+ maxChallengeCompletionRoundsInt, err := strconv.ParseInt(maxChallengeCompletionRoundsString, 10, 64)
if err != nil {
return err
}
- <-cb.done
- if cb.err != nil {
+ maxFileSizeString := conf.Fields["max_file_size"]
+ maxFileSizeInt64, err := strconv.ParseInt(maxFileSizeString, 10, 64)
+ if err != nil {
return err
}
- config.StorageSCConfig.ChallengeCompletionTime = cb.cct
- config.StorageSCConfig.MaxFileSize = cb.mfs
+ config.StorageSCConfig.ChallengeCompletionTime = maxChallengeCompletionRoundsInt
+ config.StorageSCConfig.MaxFileSize = maxFileSizeInt64
return nil
}
diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go
index 8417a81ea..87d95b979 100644
--- a/code/go/0chain.net/blobber/zcn.go
+++ b/code/go/0chain.net/blobber/zcn.go
@@ -2,6 +2,8 @@ package main
import (
"fmt"
+ "time"
+
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler"
@@ -10,10 +12,9 @@ import (
handleCommon "github.com/0chain/blobber/code/go/0chain.net/core/common/handler"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
- "github.com/0chain/gosdk/zboxcore/sdk"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/zcncore"
"go.uber.org/zap"
- "time"
)
func registerOnChain() error {
@@ -82,16 +83,28 @@ func setupServerChain() error {
serverChain := chain.NewChainFromConfig()
chain.SetServerChain(serverChain)
- if err := zcncore.InitZCNSDK(serverChain.BlockWorker, config.Configuration.SignatureScheme); err != nil {
+ //options := []int{
+ // 0,
+ // 10, // MinConfirmation
+ // 20, // MinSubmit
+ // 3, // ConfirmationChainLength
+ // 3, // SharderConsensous
+ // 1, // QuerySleepTime
+ // 0, // VerifyOptimistic
+ //}
+
+ err := client.InitSDK("{}", serverChain.BlockWorker, config.Configuration.ChainID, config.Configuration.SignatureScheme, 0, false)
+ if err != nil {
return err
}
- if err := zcncore.SetWalletInfo(node.Self.GetWalletString(), false); err != nil {
+
+ err = zcncore.SetGeneralWalletInfo(node.Self.GetWalletString(), config.Configuration.SignatureScheme)
+ if err != nil {
return err
}
- if err := sdk.InitStorageSDK(node.Self.GetWalletString(), serverChain.BlockWorker, config.Configuration.ChainID, config.Configuration.SignatureScheme,
- nil, 0); err != nil {
- return err
+ if node.Self.GetWallet().IsSplit {
+ zcncore.RegisterZauthServer(serverChain.ZauthServer)
}
fmt.Print(" [OK]\n")
diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go
index 87669bb3d..b1395caa4 100644
--- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go
+++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go
@@ -13,12 +13,11 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"golang.org/x/sync/errgroup"
"go.uber.org/zap"
"gorm.io/gorm"
- "gorm.io/gorm/clause"
)
const (
@@ -32,8 +31,8 @@ const (
type AllocationChangeProcessor interface {
CommitToFileStore(ctx context.Context, mut *sync.Mutex) error
DeleteTempFile() error
- ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, allocationRoot string,
- ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error)
+ ApplyChange(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error
GetPath() []string
Marshal() (string, error)
Unmarshal(string) error
@@ -74,6 +73,7 @@ type AllocationChange struct {
Input string `gorm:"column:input"`
FilePath string `gorm:"-"`
LookupHash string `gorm:"column:lookup_hash;size:64"`
+ AllocationID string `gorm:"-" json:"-"`
datastore.ModelWithTS
}
@@ -230,8 +230,8 @@ func (cc *AllocationChangeCollector) ComputeProperties() {
switch change.Operation {
case constants.FileOperationInsert:
acp = new(UploadFileChanger)
- case constants.FileOperationUpdate:
- acp = new(UpdateFileChanger)
+ // case constants.FileOperationUpdate:
+ // acp = new(UpdateFileChanger)
case constants.FileOperationDelete:
acp = new(DeleteFileChange)
case constants.FileOperationRename:
@@ -254,29 +254,36 @@ func (cc *AllocationChangeCollector) ComputeProperties() {
}
}
-func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocationRoot, prevAllocationRoot string,
- ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) {
- rootRef, err := cc.GetRootRef(ctx)
- if err != nil {
- return rootRef, err
- }
- if rootRef.Hash != prevAllocationRoot {
- return rootRef, common.NewError("invalid_prev_root", "Invalid prev root")
- }
+func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64) error {
+ now := time.Now()
+ collector := reference.NewCollector(len(cc.Changes))
+ timeoutctx, cancel := context.WithTimeout(ctx, time.Second*60)
+ defer cancel()
+ eg, egCtx := errgroup.WithContext(timeoutctx)
+ eg.SetLimit(10)
for idx, change := range cc.Changes {
- changeProcessor := cc.AllocationChanges[idx]
- _, err := changeProcessor.ApplyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta)
- if err != nil {
- return rootRef, err
+ select {
+ case <-egCtx.Done():
+ return egCtx.Err()
+ default:
+ changeIndex := idx
+ eg.Go(func() error {
+ change.AllocationID = cc.AllocationID
+ changeProcessor := cc.AllocationChanges[changeIndex]
+ return changeProcessor.ApplyChange(ctx, ts, allocationVersion, collector)
+ })
}
}
- collector := reference.NewCollector(len(cc.Changes))
- _, err = rootRef.CalculateHash(ctx, true, collector)
+ err := eg.Wait()
if err != nil {
- return rootRef, err
+ return err
}
- err = collector.Finalize(ctx)
- return rootRef, err
+ elapsedApplyChanges := time.Since(now)
+ err = collector.Finalize(ctx, cc.AllocationID, allocationVersion)
+ elapsedFinalize := time.Since(now) - elapsedApplyChanges
+ logging.Logger.Info("ApplyChanges", zap.String("allocation_id", cc.AllocationID), zap.Duration("apply_changes", elapsedApplyChanges), zap.Duration("finalize", elapsedFinalize), zap.Int("changes", len(cc.Changes)))
+ return err
}
func (a *AllocationChangeCollector) CommitToFileStore(ctx context.Context) error {
@@ -304,154 +311,107 @@ func (a *AllocationChangeCollector) DeleteChanges(ctx context.Context) {
}
type Result struct {
- Id string
- ValidationRoot string
- PrevValidationRoot string
- ThumbnailHash string
- PrevThumbnailHash string
- FilestoreVersion int
+ LookupHash string
}
// TODO: Need to speed up this function
-func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error {
+func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context, allocationVersion int64) error {
logging.Logger.Info("Move to filestore", zap.String("allocation_id", a.AllocationID))
- err := deleteFromFileStore(ctx, a.AllocationID)
+ var (
+ refs []*reference.Ref
+ useRefCache bool
+ deletedRefs []*reference.Ref
+ )
+ refCache := reference.GetRefCache(a.AllocationID)
+ defer reference.DeleteRefCache(a.AllocationID)
+ if refCache != nil && refCache.AllocationVersion == allocationVersion {
+ useRefCache = true
+ refs = refCache.CreatedRefs
+ deletedRefs = refCache.DeletedRefs
+ } else if refCache != nil && refCache.AllocationVersion != allocationVersion {
+ logging.Logger.Error("Ref cache is not valid", zap.String("allocation_id", a.AllocationID), zap.String("ref_cache_version", fmt.Sprintf("%d", refCache.AllocationVersion)), zap.String("allocation_version", fmt.Sprintf("%d", allocationVersion)))
+ } else {
+ logging.Logger.Error("Ref cache is nil", zap.String("allocation_id", a.AllocationID))
+ }
+ err := deleteFromFileStore(a.AllocationID, deletedRefs, useRefCache)
if err != nil {
return err
}
- var refs []*Result
- limitCh := make(chan struct{}, 10)
- wg := &sync.WaitGroup{}
- err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+ limitCh := make(chan struct{}, 12)
+ wg := &sync.WaitGroup{}
+ if !useRefCache {
tx := datastore.GetStore().GetTransaction(ctx)
- err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash", "filestore_version").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE).
- FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error {
-
- for _, ref := range refs {
-
- var count int64
- if ref.PrevValidationRoot != "" {
- tx.Model(&reference.Ref{}).
- Where("allocation_id=? AND validation_root=?", a.AllocationID, ref.PrevValidationRoot).
- Count(&count)
- }
-
- limitCh <- struct{}{}
- wg.Add(1)
-
- go func(ref *Result) {
- defer func() {
- <-limitCh
- wg.Done()
- }()
-
- if count == 0 && ref.PrevValidationRoot != "" {
- err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevValidationRoot, ref.FilestoreVersion)
- if err != nil {
- logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
- zap.String("validation_root", ref.ValidationRoot))
- }
- }
- err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ValidationRoot, ref.FilestoreVersion)
- if err != nil {
- logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error()),
- zap.String("validation_root", ref.ValidationRoot))
- }
-
- if ref.ThumbnailHash != "" && ref.ThumbnailHash != ref.PrevThumbnailHash {
- if ref.PrevThumbnailHash != "" {
- err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevThumbnailHash, ref.FilestoreVersion)
- if err != nil {
- logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail file: %s", err.Error()),
- zap.String("thumbnail_hash", ref.ThumbnailHash))
- }
- }
- err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ThumbnailHash, ref.FilestoreVersion)
- if err != nil {
- logging.Logger.Error(fmt.Sprintf("Error while moving thumbnail file: %s", err.Error()),
- zap.String("thumbnail_hash", ref.ThumbnailHash))
- }
- }
-
- }(ref)
- }
-
- return nil
- }).Error
-
- wg.Wait()
-
+ err = tx.Model(&reference.Ref{}).Select("lookup_hash").Where("allocation_id=? AND allocation_version=? AND type=?", a.AllocationID, allocationVersion, reference.FILE).Find(&refs).Error
if err != nil {
- logging.Logger.Error("Error while moving to filestore", zap.Error(err))
+ logging.Logger.Error("Error while moving files to filestore", zap.Error(err))
return err
}
+ }
- return tx.Exec("UPDATE reference_objects SET is_precommit=?, prev_validation_root=validation_root, prev_thumbnail_hash=thumbnail_hash WHERE allocation_id=? AND is_precommit=? AND deleted_at is NULL", false, a.AllocationID, true).Error
- })
- return err
+ for _, ref := range refs {
+
+ limitCh <- struct{}{}
+ wg.Add(1)
+ refLookupHash := ref.LookupHash
+ go func() {
+ defer func() {
+ <-limitCh
+ wg.Done()
+ }()
+ err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, refLookupHash, filestore.VERSION)
+ if err != nil {
+ logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error()))
+ }
+
+ }()
+ }
+
+ wg.Wait()
+ return nil
}
-func deleteFromFileStore(ctx context.Context, allocationID string) error {
- limitCh := make(chan struct{}, 10)
+func deleteFromFileStore(allocationID string, deletedRefs []*reference.Ref, useRefCache bool) error {
+ limitCh := make(chan struct{}, 12)
wg := &sync.WaitGroup{}
- var results []Result
+ var results []*reference.Ref
+ if useRefCache {
+ results = deletedRefs
+ }
return datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
db := datastore.GetStore().GetTransaction(ctx)
+ if !useRefCache {
+ err := db.Model(&reference.Ref{}).Unscoped().Select("lookup_hash").
+ Where("allocation_id=? AND type=? AND deleted_at is not NULL", allocationID, reference.FILE).
+ Find(&results).Error
+ if err != nil && err != gorm.ErrRecordNotFound {
+ logging.Logger.Error("DeleteFromFileStore", zap.Error(err))
+ return err
+ }
+ }
- err := db.Model(&reference.Ref{}).Unscoped().Select("id", "validation_root", "thumbnail_hash", "filestore_version").
- Where("allocation_id=? AND is_precommit=? AND type=? AND deleted_at is not NULL", allocationID, true, reference.FILE).
- FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error {
-
- for _, res := range results {
- var count int64
- tx.Model(&reference.Ref{}).
- Where("allocation_id=? AND validation_root=?", allocationID, res.ValidationRoot).
- Count(&count)
-
- if count != 0 && res.ThumbnailHash == "" {
- continue
- }
-
- limitCh <- struct{}{}
- wg.Add(1)
-
- go func(res Result, count int64) {
- defer func() {
- <-limitCh
- wg.Done()
- }()
-
- if count == 0 {
- err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ValidationRoot,
- res.FilestoreVersion)
- if err != nil {
- logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
- zap.String("validation_root", res.ValidationRoot))
- }
- }
-
- if res.ThumbnailHash != "" {
- err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ThumbnailHash, res.FilestoreVersion)
- if err != nil {
- logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail: %s", err.Error()),
- zap.String("thumbnail", res.ThumbnailHash))
- }
- }
-
- }(res, count)
-
+ for _, res := range results {
+ limitCh <- struct{}{}
+ wg.Add(1)
+ resLookupHash := res.LookupHash
+ go func() {
+ defer func() {
+ <-limitCh
+ wg.Done()
+ }()
+
+ err := filestore.GetFileStore().DeleteFromFilestore(allocationID, resLookupHash,
+ filestore.VERSION)
+ if err != nil {
+ logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
+ zap.String("lookup_hash", res.LookupHash))
}
- return nil
- }).Error
+ }()
- wg.Wait()
- if err != nil && err != gorm.ErrRecordNotFound {
- logging.Logger.Error("DeleteFromFileStore", zap.Error(err))
- return err
}
+ wg.Wait()
return db.Model(&reference.Ref{}).Unscoped().
Delete(&reference.Ref{},
diff --git a/code/go/0chain.net/blobbercore/allocation/common_test.go b/code/go/0chain.net/blobbercore/allocation/common_test.go
index 372cbaee0..9cb6429fb 100644
--- a/code/go/0chain.net/blobbercore/allocation/common_test.go
+++ b/code/go/0chain.net/blobbercore/allocation/common_test.go
@@ -21,11 +21,9 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string,
b := bytes.NewBuffer(make([]byte, 0))
n, _ := io.Copy(b, infile)
return &filestore.FileOutputData{
- Name: fileData.Name,
- Path: fileData.Path,
- FixedMerkleRoot: "",
- ValidationRoot: fileData.ValidationRoot,
- Size: n,
+ Name: fileData.Name,
+ Path: fileData.Path,
+ Size: n,
}, nil
}
diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go
index ef7eac62f..47e418712 100644
--- a/code/go/0chain.net/blobbercore/allocation/connection.go
+++ b/code/go/0chain.net/blobbercore/allocation/connection.go
@@ -68,10 +68,10 @@ func SaveFileChanger(connectionID string, fileChanger *BaseFileChanger) error {
return common.NewError("connection_not_found", "connection not found")
}
connectionObj.lock.Lock()
- if connectionObj.changes[fileChanger.PathHash] == nil {
+ if connectionObj.changes[fileChanger.LookupHash] == nil {
return common.NewError("connection_change_not_found", "connection change not found")
}
- connectionObj.changes[fileChanger.PathHash].baseChanger = fileChanger
+ connectionObj.changes[fileChanger.LookupHash].baseChanger = fileChanger
connectionObj.lock.Unlock()
return nil
}
@@ -190,7 +190,7 @@ func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string
if err != nil {
return saveChange, err
}
- hasher := filestore.GetNewCommitHasher(contentSize)
+ hasher := filestore.NewCommitHasher(contentSize)
change.hasher = hasher
change.seqPQ = seqpriorityqueue.NewSeqPriorityQueue(contentSize)
go hasher.Start(connectionObj.ctx, connectionID, connectionObj.AllocationID, fileName, pathHash, change.seqPQ)
@@ -211,6 +211,12 @@ func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string
DataBytes: dataWritten,
}, contentSize)
if addSize != 0 {
+ //check if reference exists and get the size
+ existingSize, err := reference.GetObjectSizeByLookupHash(ctx, pathHash)
+ if err != nil {
+ return saveChange, err
+ }
+ addSize -= existingSize
UpdateConnectionObjSize(connectionID, addSize)
}
} else {
@@ -240,6 +246,7 @@ func GetHasher(connectionID, pathHash string) *filestore.CommitHasher {
// DeleteConnectionObjEntry remove the connectionID entry from map
// If the given connectionID is not present, then it is no-op.
func DeleteConnectionObjEntry(connectionID string) {
+ logging.Logger.Info("DeleteConnectionObjEntry", zap.String("connection_id", connectionID))
connectionObjMutex.Lock()
connectionObj, ok := connectionProcessor[connectionID]
if ok {
@@ -256,6 +263,7 @@ func cleanConnectionObj() {
for connectionID, connectionObj := range connectionProcessor {
diff := time.Since(connectionObj.UpdatedAt)
if diff >= ConnectionObjTimeout {
+ logging.Logger.Info("cleanConnectionObj", zap.String("connection_id", connectionID), zap.Duration("diff", diff))
// Stop the context and hash worker
connectionObj.cnclCtx()
for _, change := range connectionObj.changes {
diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go
index 9f849df4e..f119bf33a 100644
--- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go
+++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go
@@ -2,16 +2,17 @@ package allocation
import (
"context"
+ "database/sql"
"encoding/json"
- "fmt"
"path/filepath"
"strings"
"sync"
- "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
-
+ "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+ "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
+ "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
)
type CopyFileChange struct {
@@ -19,122 +20,77 @@ type CopyFileChange struct {
AllocationID string `json:"allocation_id"`
SrcPath string `json:"path"`
DestPath string `json:"dest_path"`
+ Type string `json:"type"`
+ CustomMeta string `json:"custom_meta"`
}
func (rf *CopyFileChange) DeleteTempFile() error {
return nil
}
-func (rf *CopyFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
- allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) {
-
- totalRefs, err := reference.CountRefs(ctx, rf.AllocationID)
- if err != nil {
- return nil, err
- }
-
- if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs {
- return nil, common.NewErrorf("max_alloc_dir_files_reached",
- "maximum files and directories already reached: %v", err)
- }
-
- srcRef, err := rootRef.GetSrcPath(rf.SrcPath)
- if err != nil {
- return nil, err
- }
+func (rf *CopyFileChange) ApplyChange(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error {
+ srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath)
+ destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath)
- rootRef.UpdatedAt = ts
- rootRef.HashToBeComputed = true
+ var (
+ srcRef *reference.Ref
+ err error
+ )
- dirRef := rootRef
- fields, err := common.GetPathFields(rf.DestPath)
- if err != nil {
- return nil, err
- }
-
- for i := 0; i < len(fields); i++ {
- found := false
- for _, child := range dirRef.Children {
- if child.Name == fields[i] {
- if child.Type == reference.DIRECTORY {
- child.HashToBeComputed = true
- dirRef = child
- dirRef.UpdatedAt = ts
- found = true
- } else {
- return nil, common.NewError("invalid_path",
- fmt.Sprintf("%s is of file type", child.Path))
- }
- }
+ err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+ srcRef, err = reference.GetReferenceByLookupHash(ctx, rf.AllocationID, srcLookUpHash)
+ if err != nil {
+ return err
}
-
- if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir {
- return nil, common.NewErrorf("max_objects_in_dir_reached",
- "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir)
+ exist, err := reference.IsRefExist(ctx, rf.AllocationID, rf.DestPath)
+ if err != nil {
+ return err
+ }
+ if exist {
+ return common.NewError("invalid_reference_path", "file already exists")
}
- if !found {
- newRef := reference.NewDirectoryRef()
- newRef.AllocationID = rf.AllocationID
- newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/"))
- fileID, ok := fileIDMeta[newRef.Path]
- if !ok || fileID == "" {
- return nil, common.NewError("invalid_parameter",
- fmt.Sprintf("file path %s has no entry in file ID meta", newRef.Path))
+ rf.Type = srcRef.Type
+ if srcRef.Type == reference.DIRECTORY {
+ isEmpty, err := reference.IsDirectoryEmpty(ctx, srcRef.ID)
+ if err != nil {
+ return err
+ }
+ if !isEmpty {
+ return common.NewError("invalid_reference_path", "directory is not empty")
}
- newRef.FileID = fileID
- newRef.ParentPath = filepath.Join("/", strings.Join(fields[:i], "/"))
- newRef.Name = fields[i]
- newRef.HashToBeComputed = true
- newRef.CreatedAt = ts
- newRef.UpdatedAt = ts
- dirRef.AddChild(newRef)
- dirRef = newRef
}
+ return nil
+ }, &sql.TxOptions{
+ ReadOnly: true,
+ })
+ if err != nil {
+ return err
}
- _, err = rf.processCopyRefs(ctx, srcRef, dirRef, allocationRoot, ts, fileIDMeta)
+ parentDir, err := reference.Mkdir(ctx, rf.AllocationID, filepath.Dir(rf.DestPath), allocationVersion, ts, collector)
if err != nil {
- return nil, err
+ return err
}
- return rootRef, err
-}
-
-func (rf *CopyFileChange) processCopyRefs(
- ctx context.Context, srcRef, destRef *reference.Ref,
- allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string,
-) (
- fileRefs []*reference.Ref, err error,
-) {
-
- newRef := *srcRef
- newRef.ID = 0
- newRef.Path = filepath.Join(destRef.Path, srcRef.Name)
- fileID, ok := fileIDMeta[newRef.Path]
- if !ok || fileID == "" {
- return nil, common.NewError("invalid_parameter",
- fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path))
- }
- newRef.FileID = fileID
- newRef.ParentPath = destRef.Path
- newRef.CreatedAt = ts
- newRef.UpdatedAt = ts
- newRef.HashToBeComputed = true
- destRef.AddChild(&newRef)
- if newRef.Type == reference.DIRECTORY {
- for _, childRef := range srcRef.Children {
- fRefs, err := rf.processCopyRefs(ctx, childRef, &newRef, allocationRoot, ts, fileIDMeta)
- if err != nil {
- return nil, err
- }
- fileRefs = append(fileRefs, fRefs...)
- }
- } else {
- fileRefs = append(fileRefs, &newRef)
+ srcRef.ID = 0
+ srcRef.ParentID = &parentDir.ID
+ srcRef.Path = rf.DestPath
+ srcRef.LookupHash = destLookUpHash
+ srcRef.CreatedAt = ts
+ srcRef.UpdatedAt = ts
+ srcRef.ParentPath = filepath.Dir(rf.DestPath)
+ srcRef.Name = filepath.Base(rf.DestPath)
+ srcRef.PathLevel = len(strings.Split(strings.TrimRight(rf.DestPath, "/"), "/"))
+ srcRef.FileMetaHash = encryption.FastHash(srcRef.GetFileHashData())
+ if rf.CustomMeta != "" {
+ srcRef.CustomMeta = rf.CustomMeta
}
+ srcRef.AllocationVersion = allocationVersion
+ collector.CreateRefRecord(srcRef)
- return
+ return nil
}
func (rf *CopyFileChange) Marshal() (string, error) {
@@ -151,7 +107,12 @@ func (rf *CopyFileChange) Unmarshal(input string) error {
}
func (rf *CopyFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error {
- return nil
+ if rf.Type == reference.DIRECTORY {
+ return nil
+ }
+ srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath)
+ destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath)
+ return filestore.GetFileStore().CopyFile(rf.AllocationID, srcLookUpHash, destLookUpHash)
}
func (rf *CopyFileChange) GetPath() []string {
diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go
index f0ab47414..121cad230 100644
--- a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go
+++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go
@@ -15,9 +15,9 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/constants"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/constants"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
mocket "github.com/selvatico/go-mocket"
"go.uber.org/zap"
"google.golang.org/grpc/metadata"
@@ -38,7 +38,7 @@ func TestBlobberCore_CopyFile(t *testing.T) {
ts := time.Now().Add(time.Hour)
alloc := makeTestAllocation(common.Timestamp(ts.Unix()))
alloc.OwnerPublicKey = sch.GetPublicKey()
- alloc.OwnerID = client.GetClientID()
+ alloc.OwnerID = client.Id()
testCases := []struct {
name string
diff --git a/code/go/0chain.net/blobbercore/allocation/dao.go b/code/go/0chain.net/blobbercore/allocation/dao.go
index a62005e4f..6f67b1943 100644
--- a/code/go/0chain.net/blobbercore/allocation/dao.go
+++ b/code/go/0chain.net/blobbercore/allocation/dao.go
@@ -5,7 +5,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/errors"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"gorm.io/gorm"
)
diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go
index 4caf0d1e3..18c4c7a31 100644
--- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go
+++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go
@@ -3,7 +3,6 @@ package allocation
import (
"context"
"encoding/json"
- "path/filepath"
"sync"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
@@ -20,18 +19,15 @@ type DeleteFileChange struct {
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
- Hash string `json:"hash"`
+ LookupHash string `json:"lookup_hash"`
+ Type string `json:"type"`
}
-func (nf *DeleteFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
- allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) {
-
- err := reference.DeleteObject(ctx, rootRef, nf.AllocationID, filepath.Clean(nf.Path), ts)
- if err != nil {
- return nil, err
- }
-
- return nil, nil
+func (nf *DeleteFileChange) ApplyChange(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error {
+ collector.LockTransaction()
+ defer collector.UnlockTransaction()
+ return reference.DeleteObject(ctx, nf.AllocationID, nf.LookupHash, nf.Type, ts, allocationVersion, collector)
}
func (nf *DeleteFileChange) Marshal() (string, error) {
diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go
index b94dc9d7b..7d8ba36f8 100644
--- a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go
+++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go
@@ -13,8 +13,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
mocket "github.com/selvatico/go-mocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -32,7 +32,7 @@ func TestBlobberCore_DeleteFile(t *testing.T) {
ts := time.Now().Add(time.Hour)
alloc := makeTestAllocation(common.Timestamp(ts.Unix()))
alloc.OwnerPublicKey = sch.GetPublicKey()
- alloc.OwnerID = client.GetClientID()
+ alloc.OwnerID = client.Id()
testCases := []struct {
name string
context metadata.MD
diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go
index 98e7db8c7..fdb19dadf 100644
--- a/code/go/0chain.net/blobbercore/allocation/entity.go
+++ b/code/go/0chain.net/blobbercore/allocation/entity.go
@@ -44,23 +44,26 @@ type Allocation struct {
Tx string `gorm:"column:tx;size:64;not null;unique;index:idx_unique_allocations_tx,unique"`
TotalSize int64 `gorm:"column:size;not null;default:0"`
UsedSize int64 `gorm:"column:used_size;not null;default:0"`
+ PrevUsedSize int64 `gorm:"column:prev_used_size;not null;default:0"`
OwnerID string `gorm:"column:owner_id;size:64;not null"`
OwnerPublicKey string `gorm:"column:owner_public_key;size:512;not null"`
RepairerID string `gorm:"column:repairer_id;size:64;not null"`
Expiration common.Timestamp `gorm:"column:expiration_date;not null"`
// AllocationRoot allcation_root of last write_marker
- AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"`
- FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"`
- BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"`
- BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"`
- LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"`
- LastRedeemedSeq int64 `gorm:"column:last_redeemed_sequence;default:0"`
- IsRedeemRequired bool `gorm:"column:is_redeem_required"`
- TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"`
- StartTime common.Timestamp `gorm:"column:start_time;not null"`
+ AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"`
+ FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"`
+ BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"`
+ BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"`
+ PrevBlobberSizeUsed int64 `gorm:"column:prev_blobber_size_used;not null;default:0"`
+ LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"`
+ LastRedeemedSeq int64 `gorm:"column:last_redeemed_sequence;default:0"`
+ IsRedeemRequired bool `gorm:"column:is_redeem_required"`
+ TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"`
+ StartTime common.Timestamp `gorm:"column:start_time;not null"`
// Ending and cleaning
- CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"`
- Finalized bool `gorm:"column:finalized;not null;default:false"`
+ CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"`
+ Finalized bool `gorm:"column:finalized;not null;default:false"`
+ AllocationVersion int64 `gorm:"column:allocation_version;not null;default:0"`
// FileOptions to define file restrictions on an allocation for third-parties
// default 00000000 for all crud operations suggesting only owner has the below listed abilities.
diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go
index 9c93a5bb2..f89725e14 100644
--- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go
+++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go
@@ -8,10 +8,11 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+ "go.uber.org/zap"
)
-// swagger:model BaseFileChanger
+// swagger:model BaseFileChanger
// BaseFileChanger base file change processor
type BaseFileChanger struct {
//client side: unmarshal them from 'updateMeta'/'uploadMeta'
@@ -34,16 +35,12 @@ type BaseFileChanger struct {
//client side:
MimeType string `json:"mimetype,omitempty"`
//client side:
- //client side:
- FixedMerkleRoot string `json:"fixed_merkle_root,omitempty"`
//server side: update them by ChangeProcessor
- AllocationID string `json:"allocation_id"`
- //client side:
- ValidationRootSignature string `json:"validation_root_signature,omitempty"`
- //client side:
- ValidationRoot string `json:"validation_root,omitempty"`
- Size int64 `json:"size"`
+ AllocationID string `json:"allocation_id"`
+ DataHash string `json:"data_hash"`
+ DataHashSignature string `json:"data_hash_signature"`
+ Size int64 `json:"size"`
//server side:
ThumbnailHash string `json:"thumbnail_content_hash,omitempty"`
ThumbnailSize int64 `json:"thumbnail_size"`
@@ -60,16 +57,15 @@ type BaseFileChanger struct {
ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of CompactMerkleTree
ChunkHash string `json:"chunk_hash,omitempty"`
UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to
- PathHash string `json:"-"` // hash of path
+ CanUpdate bool `json:"can_update"` // can file be updated or not
+ LookupHash string `json:"-"` // hash of allocationID+path
}
// swagger:model UploadResult
type UploadResult struct {
- Filename string `json:"filename"`
- Size int64 `json:"size"`
- Hash string `json:"hash"`
- ValidationRoot string `json:"validation_root"`
- FixedMerkleRoot string `json:"fixed_merkle_root"`
+ Filename string `json:"filename"`
+ Size int64 `json:"size"`
+ Hash string `json:"hash"`
// UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer.
UploadLength int64 `json:"upload_length"`
@@ -108,7 +104,8 @@ func (fc *BaseFileChanger) DeleteTempFile() error {
fileInputData := &filestore.FileInputData{}
fileInputData.Name = fc.Filename
fileInputData.Path = fc.Path
- fileInputData.ValidationRoot = fc.ValidationRoot
+ fileInputData.DataHash = fc.DataHash
+ fileInputData.LookupHash = fc.LookupHash
err := filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fc.ConnectionID, fileInputData)
if fc.ThumbnailSize > 0 {
fileInputData := &filestore.FileInputData{}
@@ -122,11 +119,15 @@ func (fc *BaseFileChanger) DeleteTempFile() error {
func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error {
+ if fc.LookupHash == "" {
+ fc.LookupHash = reference.GetReferenceLookup(fc.AllocationID, fc.Path)
+ }
if fc.ThumbnailSize > 0 {
fileInputData := &filestore.FileInputData{}
fileInputData.Name = fc.ThumbnailFilename
fileInputData.Path = fc.Path
fileInputData.ThumbnailHash = fc.ThumbnailHash
+ fileInputData.LookupHash = fc.LookupHash
fileInputData.ChunkSize = fc.ChunkSize
fileInputData.IsThumbnail = true
_, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData)
@@ -137,13 +138,14 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mute
fileInputData := &filestore.FileInputData{}
fileInputData.Name = fc.Filename
fileInputData.Path = fc.Path
- fileInputData.ValidationRoot = fc.ValidationRoot
- fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot
+ fileInputData.DataHash = fc.DataHash
+ fileInputData.LookupHash = fc.LookupHash
fileInputData.ChunkSize = fc.ChunkSize
fileInputData.Size = fc.Size
- fileInputData.Hasher = GetHasher(fc.ConnectionID, encryption.Hash(fc.Path))
+ fileInputData.Hasher = GetHasher(fc.ConnectionID, fc.LookupHash)
if fileInputData.Hasher == nil {
- return common.NewError("invalid_parameters", "Invalid parameters. Error getting hasher for commit.")
+ logging.Logger.Error("CommitToFileStore: Error getting hasher", zap.String("connection_id", fc.ConnectionID), zap.String("lookup_hash", fc.LookupHash))
+ return common.NewError("file_store_error", "Error getting hasher")
}
_, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData)
if err != nil {
diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go
index e1e1282db..b8faf2677 100644
--- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go
+++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go
@@ -74,19 +74,14 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference
fileRef.HashToBeComputed = true
nf.deleteHash = make(map[string]int)
- if fileRef.ValidationRoot != "" && fileRef.ValidationRoot != nf.ValidationRoot {
- nf.deleteHash[fileRef.ValidationRoot] = fileRef.FilestoreVersion
- }
-
fileRef.ActualFileHash = nf.ActualHash
fileRef.ActualFileHashSignature = nf.ActualFileHashSignature
fileRef.ActualFileSize = nf.ActualSize
fileRef.MimeType = nf.MimeType
- fileRef.ValidationRootSignature = nf.ValidationRootSignature
- fileRef.ValidationRoot = nf.ValidationRoot
fileRef.CustomMeta = nf.CustomMeta
- fileRef.FixedMerkleRoot = nf.FixedMerkleRoot
- fileRef.AllocationRoot = allocationRoot
+ fileRef.DataHash = nf.DataHash
+ fileRef.DataHashSignature = nf.DataHashSignature
+ fileRef.LookupHash = nf.LookupHash
fileRef.Size = nf.Size
fileRef.ThumbnailHash = nf.ThumbnailHash
fileRef.ThumbnailSize = nf.ThumbnailSize
@@ -95,7 +90,6 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference
fileRef.EncryptedKey = nf.EncryptedKey
fileRef.EncryptedKeyPoint = nf.EncryptedKeyPoint
fileRef.ChunkSize = nf.ChunkSize
- fileRef.IsPrecommit = true
fileRef.FilestoreVersion = filestore.VERSION
return rootRef, nil
diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go
index 3ad14d9db..50e11b0f4 100644
--- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go
+++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go
@@ -2,18 +2,24 @@ package allocation
import (
"context"
+ "database/sql"
"encoding/json"
- "fmt"
+ "math"
"path/filepath"
"strings"
+ "time"
- "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
+ "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
+ "go.uber.org/zap"
+ "gorm.io/gorm"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/util"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
+ "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
)
// swagger:model UploadFileChanger
@@ -23,93 +29,27 @@ type UploadFileChanger struct {
}
// ApplyChange update references, and create a new FileRef
-func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
- allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) {
+func (nf *UploadFileChanger) applyChange(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error {
- totalRefs, err := reference.CountRefs(ctx, nf.AllocationID)
- if err != nil {
- return nil, err
- }
-
- if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs {
- return nil, common.NewErrorf("max_alloc_dir_files_reached",
- "maximum files and directories already reached: %v", err)
- }
-
- fields, err := common.GetPathFields(filepath.Dir(nf.Path))
- if err != nil {
- return nil, err
- }
- if rootRef.CreatedAt == 0 {
- rootRef.CreatedAt = ts
- }
-
- rootRef.UpdatedAt = ts
- rootRef.HashToBeComputed = true
-
- dirRef := rootRef
- for i := 0; i < len(fields); i++ {
- found := false
- for _, child := range dirRef.Children {
- if child.Name == fields[i] {
- if child.Type != reference.DIRECTORY {
- return nil, common.NewError("invalid_reference_path", "Reference path has invalid ref type")
- }
- dirRef = child
- dirRef.UpdatedAt = ts
- dirRef.HashToBeComputed = true
- found = true
- }
- }
-
- if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir {
- return nil, common.NewErrorf("max_objects_in_dir_reached",
- "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir)
- }
-
- if !found {
- newRef := reference.NewDirectoryRef()
- newRef.AllocationID = dirRef.AllocationID
- newRef.Path = "/" + strings.Join(fields[:i+1], "/")
- fileID, ok := fileIDMeta[newRef.Path]
- if !ok || fileID == "" {
- return nil, common.NewError("invalid_parameter",
- fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path))
- }
- newRef.FileID = fileID
- newRef.ParentPath = "/" + strings.Join(fields[:i], "/")
- newRef.Name = fields[i]
- newRef.CreatedAt = ts
- newRef.UpdatedAt = ts
- newRef.HashToBeComputed = true
-
- dirRef.AddChild(newRef)
- dirRef = newRef
- }
- }
-
- for _, child := range dirRef.Children {
- if child.Name == nf.Filename {
- return nil, common.NewError("duplicate_file", "File already exists")
- }
+ if nf.AllocationID == "" {
+ return common.NewError("invalid_parameter", "allocation_id is required")
}
-
+ now := time.Now()
+ parentPath := filepath.Dir(nf.Path)
+ nf.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path)
newFile := &reference.Ref{
ActualFileHash: nf.ActualHash,
ActualFileHashSignature: nf.ActualFileHashSignature,
ActualFileSize: nf.ActualSize,
- AllocationID: dirRef.AllocationID,
- ValidationRoot: nf.ValidationRoot,
- ValidationRootSignature: nf.ValidationRootSignature,
+ AllocationID: nf.AllocationID,
CustomMeta: nf.CustomMeta,
- FixedMerkleRoot: nf.FixedMerkleRoot,
Name: nf.Filename,
Path: nf.Path,
- ParentPath: dirRef.Path,
+ ParentPath: parentPath,
Type: reference.FILE,
Size: nf.Size,
MimeType: nf.MimeType,
- AllocationRoot: allocationRoot,
ThumbnailHash: nf.ThumbnailHash,
ThumbnailSize: nf.ThumbnailSize,
ActualThumbnailHash: nf.ActualThumbnailHash,
@@ -119,21 +59,63 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference
ChunkSize: nf.ChunkSize,
CreatedAt: ts,
UpdatedAt: ts,
- HashToBeComputed: true,
- IsPrecommit: true,
+ LookupHash: nf.LookupHash,
+ DataHash: nf.DataHash,
+ DataHashSignature: nf.DataHashSignature,
+ PathLevel: len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")),
+ NumBlocks: int64(math.Ceil(float64(nf.Size*1.0) / float64(nf.ChunkSize))),
FilestoreVersion: filestore.VERSION,
+ AllocationVersion: allocationVersion,
+ NumUpdates: 1,
}
-
- fileID, ok := fileIDMeta[newFile.Path]
- if !ok || fileID == "" {
- return nil, common.NewError("invalid_parameter",
- fmt.Sprintf("file path %s has no entry in fileID meta", newFile.Path))
+ newFile.FileMetaHash = encryption.FastHash(newFile.GetFileMetaHashData())
+ elapsedNewFile := time.Since(now)
+ // find if ref exists
+ var refResult struct {
+ ID int64
+ Type string
+ NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"`
}
- newFile.FileID = fileID
- dirRef.AddChild(newFile)
+ err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+ tx := datastore.GetStore().GetTransaction(ctx)
+ return tx.Model(&reference.Ref{}).Select("id", "type", "num_of_updates").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error
+ }, &sql.TxOptions{
+ ReadOnly: true,
+ })
+ if err != nil && err != gorm.ErrRecordNotFound {
+ return err
+ }
- return rootRef, nil
+ if refResult.ID > 0 {
+ if !nf.CanUpdate {
+ return common.NewError("prohibited_allocation_file_options", "Cannot update data in this allocation.")
+ }
+ if refResult.Type != reference.FILE {
+ return common.NewError("invalid_reference_path", "Directory already exists with the same path")
+ }
+ deleteRecord := &reference.Ref{
+ ID: refResult.ID,
+ LookupHash: newFile.LookupHash,
+ Type: refResult.Type,
+ }
+ collector.DeleteRefRecord(deleteRecord)
+ newFile.NumUpdates = refResult.NumUpdates + 1
+ }
+ elapsedNewFileRecord := time.Since(now) - elapsedNewFile
+ // get parent id
+ parent := filepath.Dir(nf.Path)
+ // create or get parent directory
+ parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent, allocationVersion, ts, collector)
+ if err != nil {
+ return err
+ }
+ elapsedMkdir := time.Since(now) - elapsedNewFileRecord - elapsedNewFile
+ newFile.ParentID = &parentRef.ID
+ collector.CreateRefRecord(newFile)
+ elapsedCreateRefRecord := time.Since(now) - elapsedMkdir - elapsedNewFileRecord - elapsedNewFile
+ logging.Logger.Info("UploadFileChanger", zap.Duration("elapsedNewFile", elapsedNewFile), zap.Duration("elapsedNewFileRecord", elapsedNewFileRecord), zap.Duration("elapsedMkdir", elapsedMkdir), zap.Duration("elapsedCreateRefRecord", elapsedCreateRefRecord), zap.Duration("elapsedTotal", time.Since(now)))
+ return err
}
// Marshal marshal and change to persistent to postgres
diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go
index 5a41a40aa..7cb2fea21 100644
--- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go
+++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go
@@ -5,11 +5,12 @@ package allocation
import (
"context"
+
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
)
-func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
- allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) {
- return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta)
+func (nf *UploadFileChanger) ApplyChange(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error {
+ return nf.applyChange(ctx, ts, allocationVersion, collector)
}
diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go
index 4ea4e3bce..f1f8964c1 100644
--- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go
+++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go
@@ -17,9 +17,9 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/constants"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/constants"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
mocket "github.com/selvatico/go-mocket"
"go.uber.org/zap"
"google.golang.org/grpc/metadata"
@@ -39,7 +39,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) {
ts := time.Now().Add(time.Hour)
alloc := makeTestAllocation(common.Timestamp(ts.Unix()))
alloc.OwnerPublicKey = sch.GetPublicKey()
- alloc.OwnerID = client.GetClientID()
+ alloc.OwnerID = client.Id()
testCases := []struct {
name string
@@ -106,7 +106,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) {
changes: make(map[string]*ConnectionChange),
}
connectionProcessor["connection_id"].changes[pathHash] = &ConnectionChange{
- hasher: filestore.GetNewCommitHasher(2310),
+ hasher: filestore.NewCommitHasher(2310),
}
change := &UploadFileChanger{
BaseFileChanger: BaseFileChanger{
diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange.go b/code/go/0chain.net/blobbercore/allocation/movefilechange.go
index 7d3aa9c3e..52d587029 100644
--- a/code/go/0chain.net/blobbercore/allocation/movefilechange.go
+++ b/code/go/0chain.net/blobbercore/allocation/movefilechange.go
@@ -2,15 +2,17 @@ package allocation
import (
"context"
+ "database/sql"
"encoding/json"
- "fmt"
"path/filepath"
"strings"
"sync"
- "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
+ "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+ "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
+ "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
)
type MoveFileChange struct {
@@ -18,140 +20,74 @@ type MoveFileChange struct {
AllocationID string `json:"allocation_id"`
SrcPath string `json:"path"`
DestPath string `json:"dest_path"`
+ Type string `json:"type"`
}
func (rf *MoveFileChange) DeleteTempFile() error {
return nil
}
-func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
- allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) {
-
- srcRef, err := rootRef.GetSrcPath(rf.SrcPath)
- if err != nil {
- return nil, err
- }
-
- rootRef.UpdatedAt = ts
- rootRef.HashToBeComputed = true
-
- srcParentPath, srcFileName := filepath.Split(rf.SrcPath)
- srcFields, err := common.GetPathFields(srcParentPath)
- if err != nil {
- return nil, err
- }
- dirRef := rootRef
- for i := 0; i < len(srcFields); i++ {
- found := false
- for _, child := range dirRef.Children {
- if child.Name == srcFields[i] {
- dirRef = child
- found = true
- dirRef.HashToBeComputed = true
- break
- }
+func (rf *MoveFileChange) ApplyChange(cctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error {
+ var err error
+ srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath)
+ destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath)
+ var srcRef *reference.Ref
+ err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+ srcRef, err = reference.GetReferenceByLookupHash(ctx, rf.AllocationID, srcLookUpHash)
+ if err != nil {
+ return err
}
- if !found {
- return nil, common.NewError("invalid_reference_path",
- fmt.Sprintf("path %s does not exist", strings.Join(srcFields[:i+1], "/")))
+ exist, err := reference.IsRefExist(ctx, rf.AllocationID, rf.DestPath)
+ if err != nil {
+ return err
}
- }
-
- var removed bool
- for i, child := range dirRef.Children {
- if child.Name == srcFileName {
- dirRef.RemoveChild(i)
- removed = true
- break
+ if exist {
+ return common.NewError("invalid_reference_path", "file already exists")
}
- }
- if !removed {
- return nil, common.NewError("incomplete_move",
- "move operation rejected as it cannot be completed")
- }
-
- dirRef = rootRef
- fields, err := common.GetPathFields(rf.DestPath)
- if err != nil {
- return nil, err
- }
-
- for i := 0; i < len(fields); i++ {
- found := false
- for _, child := range dirRef.Children {
- if child.Name == fields[i] {
- if child.Type == reference.DIRECTORY {
- child.HashToBeComputed = true
- dirRef = child
- dirRef.UpdatedAt = ts
- found = true
- } else {
- return nil, common.NewError("invalid_path",
- fmt.Sprintf("%s is of file type", child.Path))
- }
+ if srcRef.Type == reference.DIRECTORY {
+ isEmpty, err := reference.IsDirectoryEmpty(cctx, srcRef.ID)
+ if err != nil {
+ return err
}
- }
-
- if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir {
- return nil, common.NewErrorf("max_objects_in_dir_reached",
- "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir)
- }
-
- if !found {
- newRef := reference.NewDirectoryRef()
- newRef.AllocationID = rf.AllocationID
- newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/"))
- newRef.ParentPath = filepath.Join("/", strings.Join(fields[:i], "/"))
- newRef.Name = fields[i]
- newRef.HashToBeComputed = true
- newRef.CreatedAt = ts
- newRef.UpdatedAt = ts
- fileID, ok := fileIDMeta[newRef.Path]
- if !ok || fileID == "" {
- return nil, common.NewError("invalid_parameter",
- fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path))
+ if !isEmpty {
+ return common.NewError("invalid_reference_path", "directory is not empty")
}
- newRef.FileID = fileID
- dirRef.AddChild(newRef)
- dirRef = newRef
}
+ return nil
+ }, &sql.TxOptions{
+ ReadOnly: true,
+ })
+ if err != nil {
+ return err
}
- fileRefs := rf.processMoveRefs(ctx, srcRef, dirRef, allocationRoot, ts, true)
-
- for _, fileRef := range fileRefs {
- fileRef.IsPrecommit = true
+ rf.Type = srcRef.Type
+ parentDir, err := reference.Mkdir(cctx, rf.AllocationID, filepath.Dir(rf.DestPath), allocationVersion, ts, collector)
+ if err != nil {
+ return err
}
- return rootRef, nil
-}
-func (rf *MoveFileChange) processMoveRefs(
- ctx context.Context, srcRef, destRef *reference.Ref,
- allocationRoot string, ts common.Timestamp, toAdd bool) (fileRefs []*reference.Ref) {
- if srcRef.Type == reference.DIRECTORY {
- srcRef.Path = filepath.Join(destRef.Path, srcRef.Name)
- srcRef.ParentPath = destRef.Path
- srcRef.UpdatedAt = ts
- srcRef.HashToBeComputed = true
- if toAdd {
- destRef.AddChild(srcRef)
- }
-
- for _, childRef := range srcRef.Children {
- fileRefs = append(fileRefs, rf.processMoveRefs(ctx, childRef, srcRef, allocationRoot, ts, false)...)
- }
- } else if srcRef.Type == reference.FILE {
- srcRef.ParentPath = destRef.Path
- srcRef.Path = filepath.Join(destRef.Path, srcRef.Name)
- srcRef.UpdatedAt = ts
- srcRef.HashToBeComputed = true
- if toAdd {
- destRef.AddChild(srcRef)
- }
- fileRefs = append(fileRefs, srcRef)
+ deleteRef := &reference.Ref{
+ ID: srcRef.ID,
+ LookupHash: srcLookUpHash,
+ Type: srcRef.Type,
}
+ collector.DeleteRefRecord(deleteRef)
+
+ srcRef.ID = 0
+ srcRef.ParentID = &parentDir.ID
+ srcRef.Path = rf.DestPath
+ srcRef.ParentPath = filepath.Dir(rf.DestPath)
+ srcRef.Name = filepath.Base(rf.DestPath)
+ srcRef.LookupHash = destLookUpHash
+ srcRef.CreatedAt = ts
+ srcRef.UpdatedAt = ts
+ srcRef.PathLevel = len(strings.Split(strings.TrimRight(rf.DestPath, "/"), "/"))
+ srcRef.FileMetaHash = encryption.FastHash(srcRef.GetFileHashData())
+ srcRef.AllocationVersion = allocationVersion
+ collector.CreateRefRecord(srcRef)
- return
-
+ return nil
}
func (rf *MoveFileChange) Marshal() (string, error) {
@@ -168,7 +104,12 @@ func (rf *MoveFileChange) Unmarshal(input string) error {
}
func (rf *MoveFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error {
- return nil
+ if rf.Type == reference.DIRECTORY {
+ return nil
+ }
+ srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath)
+ destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath)
+ return filestore.GetFileStore().CopyFile(rf.AllocationID, srcLookUpHash, destLookUpHash)
}
func (rf *MoveFileChange) GetPath() []string {
diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go
index befa83566..0e89ec582 100644
--- a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go
+++ b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go
@@ -16,8 +16,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
"go.uber.org/zap"
"google.golang.org/grpc/metadata"
)
@@ -37,7 +37,7 @@ func TestBlobberCore_MoveFile(t *testing.T) {
ts := time.Now().Add(time.Hour)
alloc := makeTestAllocation(common.Timestamp(ts.Unix()))
alloc.OwnerPublicKey = sch.GetPublicKey()
- alloc.OwnerID = client.GetClientID()
+ alloc.OwnerID = client.Id()
testCases := []struct {
name string
diff --git a/code/go/0chain.net/blobbercore/allocation/multiop_test.go b/code/go/0chain.net/blobbercore/allocation/multiop_test.go
index 917cd9030..8cde60fc5 100644
--- a/code/go/0chain.net/blobbercore/allocation/multiop_test.go
+++ b/code/go/0chain.net/blobbercore/allocation/multiop_test.go
@@ -13,8 +13,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
mocket "github.com/selvatico/go-mocket"
"github.com/stretchr/testify/require"
)
@@ -30,7 +30,7 @@ func TestMultiOp(t *testing.T) {
ts := time.Now().Add(time.Hour)
alloc := makeTestAllocation(common.Timestamp(ts.Unix()))
alloc.OwnerPublicKey = sch.GetPublicKey()
- alloc.OwnerID = client.GetClientID()
+ alloc.OwnerID = client.Id()
datastore.MocketTheStore(t, true)
ctx := datastore.GetStore().CreateTransaction(context.TODO())
setupDbMock()
diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go
index 48f434b71..e5bf4051e 100644
--- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go
+++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go
@@ -3,16 +3,17 @@ package allocation
import (
"context"
"encoding/json"
- "fmt"
"path/filepath"
"strings"
"sync"
- "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
-
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/util"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
+ "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+ "go.uber.org/zap"
+ "gorm.io/gorm"
)
type NewDir struct {
@@ -20,72 +21,58 @@ type NewDir struct {
Path string `json:"filepath" validation:"required"`
AllocationID string `json:"allocation_id"`
CustomMeta string `json:"custom_meta,omitempty"`
+ MimeType string `json:"mimetype,omitempty"`
}
-func (nf *NewDir) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
- allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) {
-
- totalRefs, err := reference.CountRefs(ctx, nf.AllocationID)
- if err != nil {
- return nil, err
- }
-
- if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs {
- return nil, common.NewErrorf("max_alloc_dir_files_reached",
- "maximum files and directories already reached: %v", err)
- }
-
- err = nf.Unmarshal(change.Input)
- if err != nil {
- return nil, err
- }
-
- if rootRef.CreatedAt == 0 {
- rootRef.CreatedAt = ts
- }
- rootRef.UpdatedAt = ts
- rootRef.HashToBeComputed = true
- fields, err := common.GetPathFields(nf.Path)
- if err != nil {
- return nil, err
+func (nf *NewDir) ApplyChange(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error {
+ parentPath := filepath.Dir(nf.Path)
+ parentPathLookup := reference.GetReferenceLookup(nf.AllocationID, parentPath)
+ parentRef, err := reference.GetFullReferenceByLookupHashWithNewTransaction(parentPathLookup)
+ if err != nil && err != gorm.ErrRecordNotFound {
+ return err
}
- dirRef := rootRef
- for i := 0; i < len(fields); i++ {
- found := false
- for _, child := range dirRef.Children {
- if child.Name == fields[i] {
- dirRef = child
- dirRef.HashToBeComputed = true
- dirRef.UpdatedAt = ts
- found = true
- break
- }
+ if parentRef == nil || parentRef.ID == 0 {
+ _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, allocationVersion, ts, collector)
+ } else {
+ collector.LockTransaction()
+ defer collector.UnlockTransaction()
+ dirLookupHash := reference.GetReferenceLookup(nf.AllocationID, nf.Path)
+ dRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, nf.AllocationID, dirLookupHash, []string{"id"})
+ if err != nil && err != gorm.ErrRecordNotFound {
+ logging.Logger.Error("ApplyChange:Newdir", zap.Error(err))
+ return err
}
-
- if !found {
- newRef := reference.NewDirectoryRef()
- newRef.AllocationID = nf.AllocationID
- newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/"))
- newRef.PathLevel = len(fields) + 1
- newRef.ParentPath = filepath.Dir(newRef.Path)
- newRef.Name = fields[i]
- newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, newRef.Path)
- newRef.CreatedAt = ts
- newRef.UpdatedAt = ts
- newRef.HashToBeComputed = true
- fileID, ok := fileIDMeta[newRef.Path]
- if !ok || fileID == "" {
- return nil, common.NewError("invalid_parameter",
- fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path))
- }
- newRef.FileID = fileID
+ err = nil
+ // already exists
+ if dRef != nil && dRef.ID != 0 {
+ return nil
+ }
+ parentIDRef := &parentRef.ID
+ newRef := reference.NewDirectoryRef()
+ newRef.AllocationID = nf.AllocationID
+ newRef.Path = nf.Path
+ if newRef.Path != "/" {
+ newRef.ParentPath = parentPath
+ }
+ newRef.Name = filepath.Base(nf.Path)
+ newRef.PathLevel = len(strings.Split(strings.TrimRight(nf.Path, "/"), "/"))
+ newRef.ParentID = parentIDRef
+ newRef.LookupHash = dirLookupHash
+ newRef.CreatedAt = ts
+ newRef.UpdatedAt = ts
+ newRef.FileMetaHash = encryption.FastHash(newRef.GetFileMetaHashData())
+ if nf.CustomMeta != "" {
newRef.CustomMeta = nf.CustomMeta
- dirRef.AddChild(newRef)
- dirRef = newRef
}
+ if nf.MimeType != "" {
+ newRef.MimeType = nf.MimeType
+ }
+ newRef.AllocationVersion = allocationVersion
+ collector.CreateRefRecord(newRef)
}
- return rootRef, nil
+ return err
}
func (nd *NewDir) Marshal() (string, error) {
diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go
index fbcee3759..774027eee 100644
--- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go
+++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go
@@ -6,31 +6,37 @@ import (
"path/filepath"
"sync"
+ "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
+ "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"go.uber.org/zap"
)
type RenameFileChange struct {
- ConnectionID string `json:"connection_id"`
- AllocationID string `json:"allocation_id"`
- Path string `json:"path"`
- NewName string `json:"new_name"`
- Name string `json:"name"`
- Type string `json:"type"`
+ ConnectionID string `json:"connection_id"`
+ AllocationID string `json:"allocation_id"`
+ Path string `json:"path"`
+ NewName string `json:"new_name"`
+ Name string `json:"name"`
+ Type string `json:"type"`
+ CustomMeta string `json:"custom_meta"`
+ MimeType string `json:"mimetype"`
+ newLookupHash string `json:"-"`
}
func (rf *RenameFileChange) DeleteTempFile() error {
return nil
}
-func (rf *RenameFileChange) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
- allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) {
-
+func (rf *RenameFileChange) applyChange(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error {
+ collector.LockTransaction()
+ defer collector.UnlockTransaction()
if rf.Path == "/" {
- return nil, common.NewError("invalid_operation", "cannot rename root path")
+ return common.NewError("invalid_operation", "cannot rename root path")
}
newPath := filepath.Join(filepath.Dir(rf.Path), rf.NewName)
@@ -40,79 +46,45 @@ func (rf *RenameFileChange) applyChange(ctx context.Context, rootRef *reference.
}
if isFilePresent {
- return nil, common.NewError("invalid_reference_path", "file already exists")
- }
-
- affectedRef, err := rootRef.GetSrcPath(rf.Path)
- if err != nil {
- return nil, err
- }
- affectedRef.HashToBeComputed = true
- affectedRef.Name = rf.NewName
- affectedRef.Path = newPath
- affectedRef.UpdatedAt = ts
- if affectedRef.Type == reference.FILE {
- affectedRef.IsPrecommit = true
- } else {
- rf.processChildren(ctx, affectedRef, ts)
+ return common.NewError("invalid_reference_path", "file already exists")
}
-
- parentPath := filepath.Dir(rf.Path)
- fields, err := common.GetPathFields(parentPath)
+ oldFileLookupHash := reference.GetReferenceLookup(rf.AllocationID, rf.Path)
+ ref, err := reference.GetReferenceByLookupHash(ctx, rf.AllocationID, oldFileLookupHash)
if err != nil {
- return nil, err
+ return common.NewError("invalid_reference_path", err.Error())
}
-
- rootRef.UpdatedAt = ts
- rootRef.HashToBeComputed = true
- dirRef := rootRef
-
- for i := 0; i < len(fields); i++ {
- found := false
- for _, child := range dirRef.Children {
- if child.Name == fields[i] {
- dirRef = child
- dirRef.UpdatedAt = ts
- dirRef.HashToBeComputed = true
- found = true
- break
- }
+ if ref.Type == reference.DIRECTORY {
+ isEmpty, err := reference.IsDirectoryEmpty(ctx, ref.ID)
+ if err != nil {
+ return common.NewError("invalid_reference_path", err.Error())
}
-
- if !found {
- return nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber")
+ if !isEmpty {
+ return common.NewError("invalid_reference_path", "Directory is not empty")
}
}
-
- found := false
- for i, child := range dirRef.Children {
- if child.Path == rf.Path {
- dirRef.RemoveChild(i)
- dirRef.AddChild(affectedRef)
- found = true
- break
- }
+ rf.Type = ref.Type
+ deleteRef := &reference.Ref{
+ ID: ref.ID,
+ LookupHash: oldFileLookupHash,
+ Type: ref.Type,
}
- if !found {
- return nil, common.NewError("file_not_found", "File to rename not found in blobber")
+ collector.DeleteRefRecord(deleteRef)
+ ref.Name = rf.NewName
+ ref.Path = newPath
+ ref.ID = 0
+ ref.LookupHash = reference.GetReferenceLookup(rf.AllocationID, newPath)
+ ref.UpdatedAt = ts
+ ref.FileMetaHash = encryption.Hash(ref.GetFileMetaHashData())
+ if rf.CustomMeta != "" {
+ ref.CustomMeta = rf.CustomMeta
}
-
- return rootRef, nil
-}
-
-func (rf *RenameFileChange) processChildren(ctx context.Context, curRef *reference.Ref, ts common.Timestamp) {
- for _, childRef := range curRef.Children {
- childRef.UpdatedAt = ts
- childRef.HashToBeComputed = true
- newPath := filepath.Join(curRef.Path, childRef.Name)
- childRef.UpdatePath(newPath, curRef.Path)
- if childRef.Type == reference.FILE {
- childRef.IsPrecommit = true
- }
- if childRef.Type == reference.DIRECTORY {
- rf.processChildren(ctx, childRef, ts)
- }
+ if rf.MimeType != "" {
+ ref.MimeType = rf.MimeType
}
+ ref.AllocationVersion = allocationVersion
+ collector.CreateRefRecord(ref)
+ rf.newLookupHash = ref.LookupHash
+ return nil
}
func (rf *RenameFileChange) Marshal() (string, error) {
@@ -129,7 +101,18 @@ func (rf *RenameFileChange) Unmarshal(input string) error {
}
func (rf *RenameFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error {
- return nil
+ if rf.Type == reference.DIRECTORY {
+ return nil
+ }
+ if rf.newLookupHash == "" {
+ return common.NewError("invalid_reference_path", "new lookup hash is empty")
+ }
+ oldFileLookupHash := reference.GetReferenceLookup(rf.AllocationID, rf.Path)
+ err := filestore.GetFileStore().CopyFile(rf.AllocationID, oldFileLookupHash, rf.newLookupHash)
+ if err != nil {
+ logging.Logger.Error("CommitToFileStore: CopyFile", zap.Error(err))
+ }
+ return err
}
func (rf *RenameFileChange) GetPath() []string {
diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go
index 9c773cab7..424ffe7ab 100644
--- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go
+++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go
@@ -10,8 +10,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
)
-func (rf *RenameFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange,
- allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) {
+func (rf *RenameFileChange) ApplyChange(ctx context.Context,
+ ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error {
- return rf.applyChange(ctx, rootRef, change, allocationRoot, ts, nil)
+ return rf.applyChange(ctx, ts, allocationVersion, collector)
}
diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go
index c895c7003..66b44da29 100644
--- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go
+++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go
@@ -17,10 +17,11 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/config"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
- zencryption "github.com/0chain/gosdk/zboxcore/encryption"
- "github.com/0chain/gosdk/zcncore"
+ coreNetwork "github.com/0chain/gosdk/core/conf"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
+ zencryption "github.com/0chain/gosdk_common/zboxcore/encryption"
+ "github.com/0chain/gosdk_common/zcncore"
"github.com/DATA-DOG/go-sqlmock"
mocket "github.com/selvatico/go-mocket"
"github.com/stretchr/testify/require"
@@ -55,7 +56,7 @@ func setup(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if err := zcncore.SetWalletInfo(string(wBlob), true); err != nil {
+ if err := zcncore.SetWalletInfo(string(wBlob), "bls0chain", true); err != nil {
t.Fatal(err)
}
@@ -66,10 +67,10 @@ func setup(t *testing.T) {
},
),
)
- server := httptest.NewServer(
+ _ = httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
- n := zcncore.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}}
+ n := coreNetwork.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}}
blob, err := json.Marshal(n)
if err != nil {
t.Fatal(err)
@@ -81,10 +82,6 @@ func setup(t *testing.T) {
},
),
)
-
- if err := zcncore.InitZCNSDK(server.URL, "ed25519"); err != nil {
- t.Fatal(err)
- }
}
func setupMockForFileManagerInit(mock sqlmock.Sqlmock) {
mock.ExpectBegin()
@@ -151,7 +148,7 @@ func TestBlobberCore_RenameFile(t *testing.T) {
ts := time.Now().Add(time.Hour)
alloc := makeTestAllocation(common.Timestamp(ts.Unix()))
alloc.OwnerPublicKey = sch.GetPublicKey()
- alloc.OwnerID = client.GetClientID()
+ alloc.OwnerID = client.Id()
testCases := []struct {
name string
diff --git a/code/go/0chain.net/blobbercore/allocation/rollback.go b/code/go/0chain.net/blobbercore/allocation/rollback.go
index fe25adaac..d3070ad96 100644
--- a/code/go/0chain.net/blobbercore/allocation/rollback.go
+++ b/code/go/0chain.net/blobbercore/allocation/rollback.go
@@ -8,23 +8,23 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
)
-func ApplyRollback(ctx context.Context, allocationID string) error {
+func ApplyRollback(ctx context.Context, allocationID string, allocationVersion int64) error {
db := datastore.GetStore().GetTransaction(ctx)
- // delete all is_precommit rows
+ // delete all current allocation version rows
err := db.Model(&reference.Ref{}).Unscoped().
Delete(&reference.Ref{},
- "allocation_id=? AND is_precommit=? AND deleted_at IS NULL",
- allocationID, true).Error
+ "allocation_id=? AND allocation_version=? AND deleted_at IS NULL",
+ allocationID, allocationVersion).Error
if err != nil {
return err
}
// err = db.Exec("UPDATE file_stats SET deleted_at=NULL WHERE ref_id IN (SELECT id FROM reference_objects WHERE allocation_id=? AND deleted_at IS NOT NULL)", allocationID).Error
// revive soft deleted ref rows
- err = db.Exec("UPDATE reference_objects SET deleted_at=NULL,is_precommit=? WHERE allocation_id=? AND deleted_at IS NOT NULL", false, allocationID).Error
+ err = db.Exec("UPDATE reference_objects SET deleted_at=NULL WHERE allocation_id=? AND deleted_at IS NOT NULL", allocationID).Error
return err
}
diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go
index da41fcc62..07c789d78 100644
--- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go
+++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go
@@ -12,8 +12,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
mocket "github.com/selvatico/go-mocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -36,7 +36,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) {
ts := time.Now().Add(time.Hour)
alloc := makeTestAllocation(common.Timestamp(ts.Unix()))
alloc.OwnerPublicKey = sch.GetPublicKey()
- alloc.OwnerID = client.GetClientID()
+ alloc.OwnerID = client.Id()
testCases := []struct {
name string
@@ -344,7 +344,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) {
changes: make(map[string]*ConnectionChange),
}
connectionProcessor["connection_id"].changes[pathHash] = &ConnectionChange{
- hasher: filestore.GetNewCommitHasher(2310),
+ hasher: filestore.NewCommitHasher(2310),
}
change := &UpdateFileChanger{
BaseFileChanger: BaseFileChanger{
diff --git a/code/go/0chain.net/blobbercore/allocation/workers.go b/code/go/0chain.net/blobbercore/allocation/workers.go
index b5475b81d..97f916630 100644
--- a/code/go/0chain.net/blobbercore/allocation/workers.go
+++ b/code/go/0chain.net/blobbercore/allocation/workers.go
@@ -12,6 +12,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
+ coreTxn "github.com/0chain/gosdk_common/core/transaction"
+
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
@@ -306,20 +308,15 @@ type finalizeRequest struct {
}
func sendFinalizeAllocation(allocationID string) {
- var tx, err = transaction.NewTransactionEntity()
- if err != nil {
- logging.Logger.Error("creating new transaction entity", zap.Error(err))
- return
- }
-
var request finalizeRequest
request.AllocationID = allocationID
- err = tx.ExecuteSmartContract(
+ _, _, _, _, err := coreTxn.SmartContractTxn(
transaction.STORAGE_CONTRACT_ADDRESS,
- transaction.FINALIZE_ALLOCATION,
- request,
- 0)
+ coreTxn.SmartContractTxnData{
+ Name: transaction.FINALIZE_ALLOCATION,
+ InputArgs: request,
+ }, true)
if err != nil {
logging.Logger.Error("sending finalize allocation", zap.Error(err))
return
diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go
index c462fdb23..fb64be0da 100644
--- a/code/go/0chain.net/blobbercore/blobberhttp/response.go
+++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go
@@ -35,7 +35,6 @@ type RefResult struct {
OffsetPath string `json:"offset_path,omitempty"` //used for pagination; index for path is created in database
OffsetDate common.Timestamp `json:"offset_date,omitempty"` //used for pagination; idex for updated_at is created in database
Refs *[]reference.PaginatedRef `json:"refs"`
- LatestWM *writemarker.WriteMarker `json:"latest_write_marker"`
}
// swagger:model RecentRefResult
@@ -50,9 +49,9 @@ type ObjectPathResult struct {
// swagger:model ListResult
type ListResult struct {
- AllocationRoot string `json:"allocation_root"`
- Meta map[string]interface{} `json:"meta_data"`
- Entities []map[string]interface{} `json:"list"`
+ AllocationVersion int64 `json:"allocation_version"`
+ Meta map[string]interface{} `json:"meta_data"`
+ Entities []map[string]interface{} `json:"list"`
}
// swagger:model DownloadResponse
@@ -70,3 +69,7 @@ type LatestWriteMarkerResult struct {
PrevWM *writemarker.WriteMarker `json:"prev_write_marker"`
Version string `json:"version"`
}
+
+type LatestVersionMarkerResult struct {
+ VersionMarker *writemarker.VersionMarker `json:"version_marker"`
+}
diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go
index e8f37cee7..5fb55d152 100644
--- a/code/go/0chain.net/blobbercore/challenge/challenge.go
+++ b/code/go/0chain.net/blobbercore/challenge/challenge.go
@@ -9,6 +9,7 @@ import (
"time"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
+ coreTxn "github.com/0chain/gosdk_common/core/transaction"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
@@ -162,7 +163,7 @@ func validateOnValidators(ctx context.Context, c *ChallengeEntity) error {
return nil
}
-func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*transaction.Transaction, error) {
+func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*coreTxn.Transaction, error) {
createdTime := common.ToTime(c.CreatedAt)
logging.Logger.Info("[challenge]verify: ",
@@ -186,13 +187,6 @@ func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*transactio
return nil, nil
}
- txn, err := transaction.NewTransactionEntity()
- if err != nil {
- logging.Logger.Error("[challenge]createTxn", zap.Error(err))
- c.CancelChallenge(ctx, err)
- return nil, nil
- }
-
sn := &ChallengeResponse{}
sn.ChallengeID = c.ChallengeID
for _, vt := range c.ValidationTickets {
@@ -201,14 +195,17 @@ func (c *ChallengeEntity) getCommitTransaction(ctx context.Context) (*transactio
}
}
- err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.CHALLENGE_RESPONSE, sn, 0)
+ _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{
+ Name: transaction.CHALLENGE_RESPONSE,
+ InputArgs: sn,
+ }, false)
if err != nil {
logging.Logger.Info("Failed submitting challenge to the mining network", zap.String("err:", err.Error()))
c.CancelChallenge(ctx, err)
return nil, nil
}
- err = UpdateChallengeTimingTxnSubmission(c.ChallengeID, txn.CreationDate)
+ err = UpdateChallengeTimingTxnSubmission(c.ChallengeID, common.Timestamp(txn.CreationDate))
if err != nil {
logging.Logger.Error("[challengetiming]txn_submission",
zap.Any("challenge_id", c.ChallengeID),
diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go
index 55b8940c0..8d669d2e9 100644
--- a/code/go/0chain.net/blobbercore/challenge/protocol.go
+++ b/code/go/0chain.net/blobbercore/challenge/protocol.go
@@ -14,13 +14,13 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker"
- "github.com/0chain/blobber/code/go/0chain.net/core/chain"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/lock"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
"github.com/0chain/blobber/code/go/0chain.net/core/util"
- sdkUtil "github.com/0chain/gosdk/core/util"
+ coretxn "github.com/0chain/gosdk_common/core/transaction"
+ sdkUtil "github.com/0chain/gosdk_common/core/util"
"github.com/remeh/sizedwaitgroup"
"gorm.io/gorm"
@@ -109,9 +109,6 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error {
blockNum := int64(0)
var objectPath *reference.ObjectPath
if rootRef != nil {
- if rootRef.Hash != allocationObj.AllocationRoot {
- logging.Logger.Error("root_mismatch", zap.Any("allocation_root", allocationObj.AllocationRoot), zap.Any("latest_write_marker", wms[len(wms)-1].WM.AllocationRoot), zap.Any("root_ref_hash", rootRef.Hash))
- }
if rootRef.NumBlocks > 0 {
r := rand.New(rand.NewSource(cr.RandomNumber))
blockNum = r.Int63n(rootRef.NumBlocks)
@@ -350,11 +347,11 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error {
return nil
}
-func (cr *ChallengeEntity) VerifyChallengeTransaction(ctx context.Context, txn *transaction.Transaction) error {
+func (cr *ChallengeEntity) VerifyChallengeTransaction(ctx context.Context, txn *coretxn.Transaction) error {
if len(cr.LastCommitTxnIDs) > 0 {
for _, lastTxn := range cr.LastCommitTxnIDs {
logging.Logger.Info("[challenge]commit: Verifying the transaction : " + lastTxn)
- t, err := transaction.VerifyTransaction(lastTxn, chain.GetServerChain())
+ t, err := coretxn.VerifyTransaction(lastTxn)
if err == nil {
cr.SaveChallengeResult(ctx, t, false)
return nil
@@ -365,11 +362,11 @@ func (cr *ChallengeEntity) VerifyChallengeTransaction(ctx context.Context, txn *
logging.Logger.Info("Verifying challenge response to blockchain.", zap.String("txn", txn.Hash), zap.String("challenge_id", cr.ChallengeID))
var (
- t *transaction.Transaction
+ t *coretxn.Transaction
err error
)
for i := 0; i < 3; i++ {
- t, err = transaction.VerifyTransactionWithNonce(txn.Hash, txn.GetTransaction().GetTransactionNonce())
+ t, err = coretxn.VerifyTransaction(txn.Hash)
if err == nil {
break
}
@@ -406,7 +403,7 @@ func IsEntityNotFoundError(err error) bool {
return strings.Contains(err.Error(), EntityNotFound)
}
-func (cr *ChallengeEntity) SaveChallengeResult(ctx context.Context, t *transaction.Transaction, toAdd bool) {
+func (cr *ChallengeEntity) SaveChallengeResult(ctx context.Context, t *coretxn.Transaction, toAdd bool) {
cr.statusMutex.Lock()
cr.Status = Committed
cr.statusMutex.Unlock()
diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go
index e6fbe14e2..c4fb76c7c 100644
--- a/code/go/0chain.net/blobbercore/challenge/worker.go
+++ b/code/go/0chain.net/blobbercore/challenge/worker.go
@@ -2,15 +2,15 @@ package challenge
import (
"context"
+ "encoding/json"
"sync"
"time"
- "github.com/0chain/gosdk/zcncore"
-
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/blobber/code/go/0chain.net/core/transaction"
+ "github.com/0chain/gosdk_common/core/screstapi"
+ coreTxn "github.com/0chain/gosdk_common/core/transaction"
"github.com/emirpasic/gods/maps/treemap"
"go.uber.org/zap"
"golang.org/x/sync/semaphore"
@@ -54,8 +54,8 @@ const batchSize = 5
// SetupWorkers start challenge workers
func SetupWorkers(ctx context.Context) {
- go startPullWorker(ctx)
- go startWorkers(ctx)
+ // go startPullWorker(ctx)
+ // go startWorkers(ctx)
}
func startPullWorker(ctx context.Context) {
@@ -95,7 +95,15 @@ func getRoundWorker(ctx context.Context) {
}
func setRound() {
- currentRound, _ := zcncore.GetRoundFromSharders()
+ res, err := screstapi.MakeSCRestAPICall("", "/v1/current-round", nil, "")
+ if err != nil {
+ logging.Logger.Error("getRoundWorker", zap.Error(err))
+ }
+ var currentRound int64
+ err = json.Unmarshal(res, ¤tRound)
+ if err != nil {
+ logging.Logger.Error("getRoundWorker", zap.Error(err))
+ }
if roundInfo.LastRoundDiff == 0 {
roundInfo.LastRoundDiff = 1000
@@ -183,7 +191,7 @@ func commitOnChainWorker(ctx context.Context) {
for _, challenge := range challenges {
chall := challenge
var (
- txn *transaction.Transaction
+ txn *coreTxn.Transaction
err error
)
_ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go
index 7e9d2dccc..73c9e3bdf 100644
--- a/code/go/0chain.net/blobbercore/config/config.go
+++ b/code/go/0chain.net/blobbercore/config/config.go
@@ -32,9 +32,10 @@ func SetupDefaultConfig() {
viper.SetDefault("rate_limiters.block_limit_request", 500)
viper.SetDefault("rate_limiters.block_limit_monthly", 31250000)
viper.SetDefault("rate_limiters.upload_limit_monthly", 31250000)
- viper.SetDefault("rate_limiters.commit_limit_monthly", 30000)
+ viper.SetDefault("rate_limiters.commit_limit_monthly", 1000000000)
viper.SetDefault("rate_limiters.commit_limit_daily", 1600)
viper.SetDefault("rate_limiters.commit_zero_limit_daily", 400)
+ viper.SetDefault("rate_limiters.max_connection_changes", 100)
viper.SetDefault("healthcheck.frequency", "60s")
@@ -121,6 +122,7 @@ type Config struct {
CommitLimitDaily int64
CommitZeroLimitDaily int64
ChallengeCleanupGap int64
+ MaxConnectionChanges int
HealthCheckWorkerFreq time.Duration
@@ -292,7 +294,6 @@ func ReadConfig(deploymentMode int) {
} else if Configuration.MinConfirmation > 100 {
Configuration.MinConfirmation = 100
}
-
Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily")
Configuration.BlockLimitRequest = viper.GetInt64("rate_limiters.block_limit_request")
Configuration.BlockLimitMonthly = viper.GetInt64("rate_limiters.block_limit_monthly")
@@ -300,6 +301,7 @@ func ReadConfig(deploymentMode int) {
Configuration.CommitLimitMonthly = viper.GetInt64("rate_limiters.commit_limit_monthly")
Configuration.CommitLimitDaily = viper.GetInt64("rate_limiters.commit_limit_daily")
Configuration.CommitZeroLimitDaily = viper.GetInt64("rate_limiters.commit_zero_limit_daily")
+ Configuration.MaxConnectionChanges = viper.GetInt("rate_limiters.max_connection_changes")
Configuration.IsEnterprise = viper.GetBool("is_enterprise")
}
diff --git a/code/go/0chain.net/blobbercore/config/settings.go b/code/go/0chain.net/blobbercore/config/settings.go
index f12f26cca..7fd0e9e30 100644
--- a/code/go/0chain.net/blobbercore/config/settings.go
+++ b/code/go/0chain.net/blobbercore/config/settings.go
@@ -8,8 +8,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
"github.com/0chain/errors"
- "github.com/0chain/gosdk/constants"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk/zboxcore/sdk"
+ "github.com/0chain/gosdk_common/constants"
"go.uber.org/zap"
"gorm.io/gorm"
)
@@ -102,7 +102,7 @@ func Update(ctx context.Context, db *gorm.DB) error {
}
// ReloadFromChain load and refresh latest settings from blockchain
-func ReloadFromChain(ctx context.Context, db *gorm.DB) (*zcncore.Blobber, error) {
+func ReloadFromChain(ctx context.Context, db *gorm.DB) (*sdk.Blobber, error) {
if db == nil {
return nil, errors.Throw(constants.ErrInvalidParameter, "db")
}
@@ -115,7 +115,7 @@ func ReloadFromChain(ctx context.Context, db *gorm.DB) (*zcncore.Blobber, error)
}
Configuration.Capacity = int64(b.Capacity)
- Configuration.NumDelegates = *b.StakePoolSettings.NumDelegates
+ Configuration.NumDelegates = b.StakePoolSettings.NumDelegates
if token, err := b.Terms.ReadPrice.ToToken(); err != nil {
return nil, err
@@ -129,6 +129,6 @@ func ReloadFromChain(ctx context.Context, db *gorm.DB) (*zcncore.Blobber, error)
Configuration.WritePrice = token
}
- Configuration.ServiceCharge = *b.StakePoolSettings.ServiceCharge
+ Configuration.ServiceCharge = b.StakePoolSettings.ServiceCharge
return b, Update(ctx, db)
}
diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go
index 8f3ca5c41..1c6ef1bb3 100644
--- a/code/go/0chain.net/blobbercore/convert/convert.go
+++ b/code/go/0chain.net/blobbercore/convert/convert.go
@@ -280,13 +280,9 @@ func convertFileRefToFileMetaDataGRPC(fileref *reference.Ref) *blobbergrpc.FileM
LookupHash: fileref.LookupHash,
Name: fileref.Name,
Path: fileref.Path,
- Hash: fileref.Hash,
NumBlocks: fileref.NumBlocks,
- PathHash: fileref.PathHash,
CustomMeta: fileref.CustomMeta,
- ValidationRoot: fileref.ValidationRoot,
Size: fileref.Size,
- FixedMerkleRoot: fileref.FixedMerkleRoot,
ActualFileSize: fileref.ActualFileSize,
ActualFileHash: fileref.ActualFileHash,
MimeType: fileref.MimeType,
@@ -306,9 +302,7 @@ func convertDirRefToDirMetaDataGRPC(dirref *reference.Ref) *blobbergrpc.DirMetaD
LookupHash: dirref.LookupHash,
Name: dirref.Name,
Path: dirref.Path,
- Hash: dirref.Hash,
NumBlocks: dirref.NumBlocks,
- PathHash: dirref.PathHash,
Size: dirref.Size,
CreatedAt: int64(dirref.CreatedAt),
UpdatedAt: int64(dirref.UpdatedAt),
@@ -336,13 +330,9 @@ func convertFileMetaDataGRPCToFileRef(metaData *blobbergrpc.FileMetaData) *refer
LookupHash: metaData.LookupHash,
Name: metaData.Name,
Path: metaData.Path,
- Hash: metaData.Hash,
NumBlocks: metaData.NumBlocks,
- PathHash: metaData.PathHash,
CustomMeta: metaData.CustomMeta,
- ValidationRoot: metaData.ValidationRoot,
Size: metaData.Size,
- FixedMerkleRoot: metaData.FixedMerkleRoot,
ActualFileSize: metaData.ActualFileSize,
ActualFileHash: metaData.ActualFileHash,
MimeType: metaData.MimeType,
@@ -362,9 +352,7 @@ func convertDirMetaDataGRPCToDirRef(dirref *blobbergrpc.DirMetaData) *reference.
LookupHash: dirref.LookupHash,
Name: dirref.Name,
Path: dirref.Path,
- Hash: dirref.Hash,
NumBlocks: dirref.NumBlocks,
- PathHash: dirref.PathHash,
Size: dirref.Size,
CreatedAt: common.Timestamp(dirref.CreatedAt),
UpdatedAt: common.Timestamp(dirref.UpdatedAt),
diff --git a/code/go/0chain.net/blobbercore/convert/response_creator.go b/code/go/0chain.net/blobbercore/convert/response_creator.go
index 828bd2bcb..7e8dbbfdc 100644
--- a/code/go/0chain.net/blobbercore/convert/response_creator.go
+++ b/code/go/0chain.net/blobbercore/convert/response_creator.go
@@ -62,7 +62,6 @@ func ListEntitesResponseCreator(r interface{}) *blobbergrpc.ListEntitiesResponse
}
resp.MetaData = FileRefToFileRefGRPC(reference.ListingDataToRef(httpResp.Meta))
- resp.AllocationRoot = httpResp.AllocationRoot
return &resp
}
@@ -157,12 +156,10 @@ func CopyObjectResponseCreator(r interface{}) *blobbergrpc.CopyObjectResponse {
httpResp, _ := r.(*allocation.UploadResult)
return &blobbergrpc.CopyObjectResponse{
- Filename: httpResp.Filename,
- Size: httpResp.Size,
- ValidationRoot: httpResp.ValidationRoot,
- FixedMerkleRoot: httpResp.FixedMerkleRoot,
- UploadLength: httpResp.UploadLength,
- UploadOffset: httpResp.UploadOffset,
+ Filename: httpResp.Filename,
+ Size: httpResp.Size,
+ UploadLength: httpResp.UploadLength,
+ UploadOffset: httpResp.UploadOffset,
}
}
@@ -173,12 +170,10 @@ func RenameObjectResponseCreator(r interface{}) *blobbergrpc.RenameObjectRespons
httpResp, _ := r.(*allocation.UploadResult)
return &blobbergrpc.RenameObjectResponse{
- Filename: httpResp.Filename,
- Size: httpResp.Size,
- ValidationRoot: httpResp.ValidationRoot,
- FixedMerkleRoot: httpResp.FixedMerkleRoot,
- UploadLength: httpResp.UploadLength,
- UploadOffset: httpResp.UploadOffset,
+ Filename: httpResp.Filename,
+ Size: httpResp.Size,
+ UploadLength: httpResp.UploadLength,
+ UploadOffset: httpResp.UploadOffset,
}
}
@@ -211,11 +206,9 @@ func UploadFileResponseCreator(r interface{}) *blobbergrpc.UploadFileResponse {
httpResp, _ := r.(*allocation.UploadResult)
return &blobbergrpc.UploadFileResponse{
- Filename: httpResp.Filename,
- Size: httpResp.Size,
- ValidationRoot: httpResp.ValidationRoot,
- FixedMerkleRoot: httpResp.FixedMerkleRoot,
- UploadLength: httpResp.UploadLength,
- UploadOffset: httpResp.UploadOffset,
+ Filename: httpResp.Filename,
+ Size: httpResp.Size,
+ UploadLength: httpResp.UploadLength,
+ UploadOffset: httpResp.UploadOffset,
}
}
diff --git a/code/go/0chain.net/blobbercore/convert/response_handler.go b/code/go/0chain.net/blobbercore/convert/response_handler.go
index 3092c6b6d..8fb3374df 100644
--- a/code/go/0chain.net/blobbercore/convert/response_handler.go
+++ b/code/go/0chain.net/blobbercore/convert/response_handler.go
@@ -42,9 +42,8 @@ func ListEntitesResponseHandler(resp *blobbergrpc.ListEntitiesResponse) *blobber
}
return &blobberhttp.ListResult{
- AllocationRoot: resp.AllocationRoot,
- Meta: FileRefGRPCToFileRef(resp.MetaData).GetListingData(ctx),
- Entities: entities,
+ Meta: FileRefGRPCToFileRef(resp.MetaData).GetListingData(ctx),
+ Entities: entities,
}
}
@@ -119,11 +118,9 @@ func GetCommitMetaTxnHandlerResponse(response *blobbergrpc.CommitMetaTxnResponse
func CopyObjectResponseHandler(copyObjectResponse *blobbergrpc.CopyObjectResponse) *allocation.UploadResult {
return &allocation.UploadResult{
- Filename: copyObjectResponse.Filename,
- Size: copyObjectResponse.Size,
- ValidationRoot: copyObjectResponse.ValidationRoot,
- FixedMerkleRoot: copyObjectResponse.FixedMerkleRoot,
- UploadLength: copyObjectResponse.UploadLength,
- UploadOffset: copyObjectResponse.UploadOffset,
+ Filename: copyObjectResponse.Filename,
+ Size: copyObjectResponse.Size,
+ UploadLength: copyObjectResponse.UploadLength,
+ UploadOffset: copyObjectResponse.UploadOffset,
}
}
diff --git a/code/go/0chain.net/blobbercore/datastore/mocket.go b/code/go/0chain.net/blobbercore/datastore/mocket.go
index b560b0841..fb9b3d4cc 100644
--- a/code/go/0chain.net/blobbercore/datastore/mocket.go
+++ b/code/go/0chain.net/blobbercore/datastore/mocket.go
@@ -72,7 +72,7 @@ func (store *Mocket) Close() {
}
}
-func (store *Mocket) CreateTransaction(ctx context.Context,opts ...*sql.TxOptions) context.Context {
+func (store *Mocket) CreateTransaction(ctx context.Context, opts ...*sql.TxOptions) context.Context {
db := store.db.Begin()
return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(db))
}
@@ -86,7 +86,7 @@ func (store *Mocket) GetTransaction(ctx context.Context) *EnhancedDB {
return nil
}
-func (store *Mocket) WithNewTransaction(f func(ctx context.Context) error) error {
+func (store *Mocket) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error {
ctx := store.CreateTransaction(context.TODO())
defer ctx.Done()
diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go
index 241701dfe..47e46623b 100644
--- a/code/go/0chain.net/blobbercore/datastore/postgres.go
+++ b/code/go/0chain.net/blobbercore/datastore/postgres.go
@@ -11,6 +11,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"gorm.io/driver/postgres"
"gorm.io/gorm"
+ "moul.io/zapgorm2"
)
// postgresStore store implementation for postgres
@@ -46,6 +47,11 @@ func (p *postgresStore) GetPgDB() (*gorm.DB, error) {
}
func (store *postgresStore) Open() error {
+ gormLogger := zapgorm2.New(logging.Logger)
+ gormLogger.SlowThreshold = 100 * time.Millisecond
+ gormLogger.IgnoreRecordNotFoundError = true
+ gormLogger.SkipCallerLookup = true
+ gormLogger.SetAsDefault()
db, err := gorm.Open(postgres.Open(fmt.Sprintf(
"host=%v port=%v user=%v dbname=%v password=%v sslmode=disable",
config.Configuration.DBHost, config.Configuration.DBPort,
@@ -53,6 +59,7 @@ func (store *postgresStore) Open() error {
config.Configuration.DBPassword)), &gorm.Config{
SkipDefaultTransaction: true, // https://gorm.io/docs/performance.html#Disable-Default-Transaction
PrepareStmt: true, //https://gorm.io/docs/performance.html#Caches-Prepared-Statement
+ Logger: gormLogger,
})
if err != nil {
return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err)
@@ -99,7 +106,7 @@ func (store *postgresStore) GetTransaction(ctx context.Context) *EnhancedDB {
return nil
}
-func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error) error {
+func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error {
timeoutctx, cancel := context.WithTimeout(context.TODO(), 45*time.Second)
defer cancel()
ctx := store.CreateTransaction(timeoutctx)
diff --git a/code/go/0chain.net/blobbercore/datastore/sqlmock.go b/code/go/0chain.net/blobbercore/datastore/sqlmock.go
index a7bcdeac5..354cc69e5 100644
--- a/code/go/0chain.net/blobbercore/datastore/sqlmock.go
+++ b/code/go/0chain.net/blobbercore/datastore/sqlmock.go
@@ -81,7 +81,7 @@ func (store *Sqlmock) GetTransaction(ctx context.Context) *EnhancedDB {
return nil
}
-func (store *Sqlmock) WithNewTransaction(f func(ctx context.Context) error) error {
+func (store *Sqlmock) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error {
ctx := store.CreateTransaction(context.TODO())
defer ctx.Done()
diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go
index a097be572..bedc36896 100644
--- a/code/go/0chain.net/blobbercore/datastore/store.go
+++ b/code/go/0chain.net/blobbercore/datastore/store.go
@@ -45,7 +45,7 @@ type Store interface {
CreateTransaction(ctx context.Context, opts ...*sql.TxOptions) context.Context
// GetTransaction get transaction from context
GetTransaction(ctx context.Context) *EnhancedDB
- WithNewTransaction(f func(ctx context.Context) error) error
+ WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error
WithTransaction(ctx context.Context, f func(ctx context.Context) error) error
// Get db connection with user that creates roles and databases. Its dialactor does not contain database name
GetPgDB() (*gorm.DB, error)
diff --git a/code/go/0chain.net/blobbercore/filestore/state.go b/code/go/0chain.net/blobbercore/filestore/state.go
index b098a34c8..f68b556f1 100644
--- a/code/go/0chain.net/blobbercore/filestore/state.go
+++ b/code/go/0chain.net/blobbercore/filestore/state.go
@@ -57,8 +57,6 @@ func (fs *FileStore) initMap() error {
return errors.New("could not get db client")
}
- limitCh := make(chan struct{}, 50)
- wg := &sync.WaitGroup{}
var dbAllocations []*dbAllocation
err := db.Model(&dbAllocation{}).FindInBatches(&dbAllocations, 1000, func(tx *gorm.DB, batch int) error {
@@ -78,18 +76,12 @@ func (fs *FileStore) initMap() error {
if err != nil {
return err
}
-
- limitCh <- struct{}{}
- wg.Add(1)
- go fs.getTemporaryStorageDetails(ctx, &a, dbAlloc.ID, limitCh, wg)
-
}
fs.setAllocations(allocsMap)
return nil
}).Error
- wg.Wait()
return err
})
return err
diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go
index 63bd60e8a..083f3d3e3 100644
--- a/code/go/0chain.net/blobbercore/filestore/storage.go
+++ b/code/go/0chain.net/blobbercore/filestore/storage.go
@@ -45,7 +45,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/core/util"
+ "github.com/0chain/gosdk_common/core/util"
"go.uber.org/zap"
"golang.org/x/crypto/sha3"
"golang.org/x/sys/unix"
@@ -57,10 +57,15 @@ const (
MerkleChunkSize = 64
ChunkSize = 64 * KB
BufferSize = 80 * ChunkSize
+ ThumbnailSuffix = "_thumbnail"
)
func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, infile multipart.File) (*FileOutputData, error) {
- tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileData.FilePathHash, conID)
+ fileHash := fileData.LookupHash
+ if fileData.IsThumbnail {
+ fileHash = fileData.LookupHash + ThumbnailSuffix
+ }
+ tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileHash, conID)
var (
initialSize int64
)
@@ -157,6 +162,17 @@ func (fs *FileStore) MoveToFilestore(allocID, hash string, version int) error {
}
_ = os.Rename(preCommitPath, fPath)
+
+ // Check if thumbnail exists
+ thumbPath := fs.getPreCommitPathForFile(allocID, hash+ThumbnailSuffix, version)
+ if _, err := os.Stat(thumbPath); err == nil {
+ thumbFilePath, err := fs.GetPathForFile(allocID, hash+ThumbnailSuffix, version)
+ if err != nil {
+ return common.NewError("get_file_path_error", err.Error())
+ }
+ _ = os.Rename(thumbPath, thumbFilePath)
+ }
+
return nil
}
@@ -179,6 +195,14 @@ func (fs *FileStore) DeleteFromFilestore(allocID, hash string, version int) erro
}
fs.incrDecrAllocFileSizeAndNumber(allocID, -stat.Size(), -1)
+ thumbPath, err := fs.GetPathForFile(allocID, hash+ThumbnailSuffix, version)
+ if err != nil {
+ return common.NewError("get_file_path_error", err.Error())
+ }
+ if _, err := os.Stat(thumbPath); err == nil {
+ os.Remove(thumbPath) //nolint:errcheck
+ }
+
return nil
}
@@ -193,15 +217,13 @@ func (fs *FileStore) DeletePreCommitDir(allocID string) error {
}
func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) (_ bool, err error) {
-
- logging.Logger.Info("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData))
- filePathHash := encryption.Hash(fileData.Path)
- tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, filePathHash, conID)
-
- fileHash := fileData.ValidationRoot
+ now := time.Now()
+ logging.Logger.Debug("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData))
+ fileHash := fileData.LookupHash
if fileData.IsThumbnail {
- fileHash = fileData.ThumbnailHash
+ fileHash = fileData.LookupHash + ThumbnailSuffix
}
+ tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileHash, conID)
preCommitPath := fs.getPreCommitPathForFile(allocID, fileHash, VERSION)
@@ -255,7 +277,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData)
return true, nil
}
- key := getKey(allocID, fileData.ValidationRoot)
+ key := getKey(allocID, fileData.LookupHash)
l, _ := contentHashMapLock.GetLock(key)
l.Lock()
defer func() {
@@ -270,41 +292,18 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData)
}
fileSize := rStat.Size()
- now := time.Now()
ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
defer cancel()
- err = fileData.Hasher.Wait(ctx, conID, allocID, fileData.Name, filePathHash)
+ err = fileData.Hasher.Wait(ctx)
if err != nil {
return false, common.NewError("hasher_wait_error", err.Error())
}
- elapsedWait := time.Since(now)
- _, err = r.Seek(fileSize, io.SeekStart)
- if err != nil {
- return false, common.NewError("seek_error", err.Error())
- }
- fmtRootBytes, err := fileData.Hasher.fmt.CalculateRootAndStoreNodes(r)
- if err != nil {
- return false, common.NewError("fmt_hash_calculation_error", err.Error())
+ md5Hash := fileData.Hasher.GetMd5Hash()
+ if md5Hash != fileData.DataHash {
+ return false, common.NewError("hash_mismatch",
+ fmt.Sprintf("calculated hash does not match with expected hash. Expected %s, got %s.",
+ fileData.DataHash, md5Hash))
}
-
- validationRootBytes, err := fileData.Hasher.vt.CalculateRootAndStoreNodes(r, fileSize)
- if err != nil {
- return false, common.NewError("validation_hash_calculation_error", err.Error())
- }
- fmtRoot := hex.EncodeToString(fmtRootBytes)
- validationRoot := hex.EncodeToString(validationRootBytes)
- elapsedRoot := time.Since(now) - elapsedWait
- if fmtRoot != fileData.FixedMerkleRoot {
- err = common.NewError("fixed_merkle_root_mismatch",
- fmt.Sprintf("Expected %s got %s", fileData.FixedMerkleRoot, fmtRoot))
- return false, err
- }
- if validationRoot != fileData.ValidationRoot {
- err = common.NewError("validation_root_mismatch",
- "calculated validation root does not match with client's validation root")
- return false, err
- }
-
err = os.Rename(tempFilePath, preCommitPath)
if err != nil {
return false, common.NewError("write_error", err.Error())
@@ -321,10 +320,81 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData)
// 5. Move: It is Copy + Delete. Delete will not delete file if ref exists in database. i.e. copy would create
// ref that refers to this file therefore it will be skipped
fs.incrDecrAllocFileSizeAndNumber(allocID, fileSize, 1)
- logging.Logger.Info("Committing write done", zap.String("file_path", fileData.Path), zap.Duration("elapsed_wait", elapsedWait), zap.Duration("elapsed_root", elapsedRoot), zap.Duration("elapsed_total", time.Since(now)))
+ logging.Logger.Info("Committing write done", zap.String("file_path", fileData.Path), zap.String("lookup_hash", fileData.LookupHash), zap.Duration("elapsed_total", time.Since(now)))
return true, nil
}
+func (fs *FileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error {
+ if oldFileLookupHash == newFileLookupHash {
+ return nil
+ }
+ var err error
+ oldObjectPath, err := fs.GetPathForFile(allocationID, oldFileLookupHash, VERSION)
+ if err != nil {
+ return common.NewError("get_file_path_error", err.Error())
+ }
+ oldFile, err := os.Open(oldObjectPath)
+ if err != nil {
+ return common.NewError("file_open_error", err.Error())
+ }
+ defer oldFile.Close()
+ stat, err := oldFile.Stat()
+ if err != nil {
+ return common.NewError("file_stat_error", err.Error())
+ }
+ size := stat.Size()
+
+ newObjectPath := fs.getPreCommitPathForFile(allocationID, newFileLookupHash, VERSION)
+ err = createDirs(filepath.Dir(newObjectPath))
+ if err != nil {
+ return common.NewError("blob_object_precommit_dir_creation_error", err.Error())
+ }
+ newFile, err := os.Create(newObjectPath)
+ if err != nil {
+ return common.NewError("file_create_error", err.Error())
+ }
+ defer func() {
+ newFile.Close()
+ if err != nil {
+ os.Remove(newObjectPath) //nolint:errcheck
+ }
+ }()
+ bufSize := BufferSize
+ if size < int64(bufSize) {
+ bufSize = int(size)
+ }
+ copyBuf := make([]byte, bufSize)
+ _, err = io.CopyBuffer(newFile, oldFile, copyBuf)
+ if err != nil {
+ return common.NewError("file_copy_error", err.Error())
+ }
+ // copy thumbnail if exists
+ oldThumbPath := fs.getPreCommitPathForFile(allocationID, oldFileLookupHash+ThumbnailSuffix, VERSION)
+ if _, err := os.Stat(oldThumbPath); err == nil {
+ newThumbPath := fs.getPreCommitPathForFile(allocationID, newFileLookupHash+ThumbnailSuffix, VERSION)
+ oldThumbFile, err := os.Open(oldThumbPath)
+ if err != nil {
+ return common.NewError("file_open_error", err.Error())
+ }
+ defer oldThumbFile.Close()
+ newThumbFile, err := os.Create(newThumbPath)
+ if err != nil {
+ return common.NewError("file_create_error", err.Error())
+ }
+ defer func() {
+ newThumbFile.Close()
+ if err != nil {
+ os.Remove(newThumbPath) //nolint:errcheck
+ }
+ }()
+ _, err = io.Copy(newThumbFile, oldThumbFile)
+ if err != nil {
+ return common.NewError("file_copy_error", err.Error())
+ }
+ }
+ return nil
+}
+
func (fs *FileStore) GetFilePathSize(allocID, filehash, thumbHash string, version int) (int64, int64, error) {
filePath, err := fs.GetPathForFile(allocID, filehash, version)
@@ -435,6 +505,7 @@ func (fs *FileStore) DeleteAllocation(allocID string) {
func (fs *FileStore) GetFileThumbnail(readBlockIn *ReadBlockInput) (*FileDownloadResponse, error) {
var fileObjectPath string
var err error
+ readBlockIn.Hash += ThumbnailSuffix
startBlock := readBlockIn.StartBlockNum
if startBlock < 0 {
return nil, common.NewError("invalid_block_number", "Invalid block number. Start block number cannot be negative")
@@ -515,7 +586,6 @@ func (fs *FileStore) GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadRes
}
startBlock := readBlockIn.StartBlockNum
- endBlock := readBlockIn.StartBlockNum + readBlockIn.NumBlocks - 1
if startBlock < 0 {
return nil, common.NewError("invalid_block_number", "Invalid block number. Start block number cannot be negative")
@@ -558,26 +628,6 @@ func (fs *FileStore) GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadRes
nodesSize := getNodesSize(filesize, util.MaxMerkleLeavesSize)
vmp := &FileDownloadResponse{}
- if readBlockIn.VerifyDownload {
- vpOffset := int64(FMTSize)
- if readBlockIn.FilestoreVersion == 1 {
- vpOffset += readBlockIn.FileSize
- }
- vp := validationTreeProof{
- dataSize: readBlockIn.FileSize,
- offset: vpOffset,
- }
-
- logging.Logger.Debug("calling GetMerkleProofOfMultipleIndexes", zap.Any("readBlockIn", readBlockIn), zap.Any("vmp", vmp))
- nodes, indexes, err := vp.GetMerkleProofOfMultipleIndexes(file, nodesSize, startBlock, endBlock)
- if err != nil {
- return nil, common.NewError("get_merkle_proof_error", err.Error())
- }
-
- vmp.Nodes = nodes
- vmp.Indexes = indexes
- }
- logging.Logger.Info("filestore_version", zap.Int("version", readBlockIn.FilestoreVersion))
fileOffset := int64(startBlock) * ChunkSize
if readBlockIn.FilestoreVersion == 1 {
_, err = file.Seek(fileOffset, io.SeekStart)
@@ -796,7 +846,7 @@ func (fs *FileStore) getAllocDir(allocID string) string {
}
func (fs *FileStore) GetPathForFile(allocID, hash string, version int) (string, error) {
- if len(allocID) != 64 || len(hash) != 64 {
+ if len(allocID) != 64 {
return "", errors.New("length of allocationID/hash must be 64")
}
var versionStr string
@@ -830,7 +880,7 @@ func (fs *FileStore) getPreCommitDir(allocationID string) string {
func (fs *FileStore) getTempPathForFile(allocId, fileName, pathHash, connectionID string) string {
fileName = sanitizeFileName(fileName)
- return filepath.Join(fs.getAllocTempDir(allocId), fileName+"."+pathHash+"."+connectionID)
+ return filepath.Join(fs.getAllocTempDir(allocId), getPartialPath(pathHash, getDirLevelsForFiles())+"."+connectionID)
}
func (fs *FileStore) getPreCommitPathForFile(allocID, hash string, version int) string {
diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go
index 228e8de05..3e39ae39d 100644
--- a/code/go/0chain.net/blobbercore/filestore/store.go
+++ b/code/go/0chain.net/blobbercore/filestore/store.go
@@ -10,12 +10,11 @@ const (
)
type FileInputData struct {
- Name string
- Path string
- ValidationRoot string
- FixedMerkleRoot string
- ThumbnailHash string
-
+ Name string
+ Path string
+ DataHash string
+ ThumbnailHash string
+ LookupHash string
// ChunkSize chunk size
ChunkSize int64
//UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer.
@@ -23,19 +22,16 @@ type FileInputData struct {
//Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer.
UploadOffset int64
//IsFinal the request is final chunk
- IsFinal bool
- IsThumbnail bool
- FilePathHash string
- Size int64
- Hasher *CommitHasher
+ IsFinal bool
+ IsThumbnail bool
+ Size int64
+ Hasher *CommitHasher
}
type FileOutputData struct {
- Name string
- Path string
- ValidationRoot string
- FixedMerkleRoot string
- ThumbnailHash string
+ Name string
+ Path string
+ ThumbnailHash string
// Size written size/chunk size
Size int64
// ChunkUploaded the chunk is uploaded or not.
@@ -57,6 +53,7 @@ type FileStorer interface {
DeleteFromFilestore(allocID, hash string, version int) error
DeletePreCommitDir(allocID string) error
DeleteAllocation(allocID string)
+ CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error
// GetFileBlock Get blocks of file starting from blockNum upto numBlocks. blockNum can't be less than 1.
GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadResponse, error)
GetBlocksMerkleTreeForChallenge(cri *ChallengeReadBlockInput) (*ChallengeResponse, error)
diff --git a/code/go/0chain.net/blobbercore/filestore/store_test.go b/code/go/0chain.net/blobbercore/filestore/store_test.go
index ad8a74dc1..21e4da514 100644
--- a/code/go/0chain.net/blobbercore/filestore/store_test.go
+++ b/code/go/0chain.net/blobbercore/filestore/store_test.go
@@ -21,7 +21,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/seqpriorityqueue"
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/core/util"
+ "github.com/0chain/gosdk_common/core/util"
"github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
@@ -251,16 +251,14 @@ func TestStoreStorageWriteAndCommit(t *testing.T) {
validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size))
require.Nil(t, err)
pathHash := encryption.Hash(test.remotePath)
- hasher := GetNewCommitHasher(0)
+ hasher := NewCommitHasher(0)
fid := &FileInputData{
- Name: test.fileName,
- Path: test.remotePath,
- ValidationRoot: validationRoot,
- FixedMerkleRoot: fixedMerkleRoot,
- ChunkSize: 64 * KB,
- FilePathHash: pathHash,
- Hasher: hasher,
- Size: int64(size),
+ Name: test.fileName,
+ Path: test.remotePath,
+ ChunkSize: 64 * KB,
+ FilePathHash: pathHash,
+ Hasher: hasher,
+ Size: int64(size),
}
f, err := os.Open(fPath)
@@ -338,7 +336,7 @@ func TestDeletePreCommitDir(t *testing.T) {
validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size))
require.Nil(t, err)
pathHash := encryption.Hash(remotePath)
- hasher := GetNewCommitHasher(int64(size))
+ hasher := NewCommitHasher(int64(size))
fid := &FileInputData{
Name: fileName,
Path: remotePath,
@@ -383,7 +381,7 @@ func TestDeletePreCommitDir(t *testing.T) {
fid.ValidationRoot = validationRoot
fid.FixedMerkleRoot = fixedMerkleRoot
- hasher = GetNewCommitHasher(int64(size))
+ hasher = NewCommitHasher(int64(size))
fid.Hasher = hasher
// Write file to temp location
@@ -446,7 +444,7 @@ func TestStorageUploadUpdate(t *testing.T) {
validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size))
require.Nil(t, err)
pathHash := encryption.Hash(remotePath)
- hasher := GetNewCommitHasher(int64(size))
+ hasher := NewCommitHasher(int64(size))
fid := &FileInputData{
Name: fileName,
Path: remotePath,
@@ -834,7 +832,7 @@ func TestValidationRoot(t *testing.T) {
fs, cleanUp := setupStorage(t)
defer cleanUp()
fPath := filepath.Join(fs.mp, randString(10)+".txt")
- cH := GetNewCommitHasher(size)
+ cH := NewCommitHasher(size)
_, err := cH.Write(thumbnailBytes)
require.Nil(t, err)
@@ -897,7 +895,7 @@ func generateRandomData(fPath string, size int64) (string, string, error) {
}
defer f.Close()
- cH := GetNewCommitHasher(size)
+ cH := NewCommitHasher(size)
_, err = cH.Write(p)
if err != nil {
return "", "", err
@@ -938,7 +936,7 @@ func generateRandomDataAndStoreNodes(fPath string, size int64) (string, string,
}
defer f.Close()
- cH := GetNewCommitHasher(size)
+ cH := NewCommitHasher(size)
_, err = cH.Write(p)
if err != nil {
return "", "", err
diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go
index 75856d485..b56a742ff 100644
--- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go
+++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go
@@ -5,9 +5,11 @@ package filestore
import (
"context"
+ "crypto/md5"
"encoding/hex"
"errors"
"fmt"
+ "hash"
"io"
"math"
"os"
@@ -15,7 +17,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/seqpriorityqueue"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/core/util"
+ "github.com/0chain/gosdk_common/core/util"
"github.com/minio/sha256-simd"
"go.uber.org/zap"
)
@@ -406,16 +408,24 @@ func getNewValidationTree(dataSize int64) *validationTree {
type CommitHasher struct {
fmt *fixedMerkleTree
vt *validationTree
+ md5hasher hash.Hash
isInitialized bool
doneChan chan struct{}
hashErr error
dataSize int64
}
-func GetNewCommitHasher(dataSize int64) *CommitHasher {
+var (
+ md5Pool = &sync.Pool{
+ New: func() interface{} {
+ return md5.New()
+ },
+ }
+)
+
+func NewCommitHasher(dataSize int64) *CommitHasher {
c := new(CommitHasher)
- c.fmt = getNewFixedMerkleTree()
- c.vt = getNewValidationTree(dataSize)
+ c.md5hasher = md5Pool.Get().(hash.Hash)
c.isInitialized = true
c.doneChan = make(chan struct{})
c.dataSize = dataSize
@@ -434,7 +444,7 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil
defer f.Close()
var toFinalize bool
var totalWritten int64
-
+ logging.Logger.Info("hasher_start", zap.String("fileHash", filePathHash), zap.String("fileName", fileName), zap.String("tempFilePath", tempFilePath))
for {
select {
case <-ctx.Done():
@@ -456,7 +466,6 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil
} else if pq.DataBytes == 0 {
continue
}
- logging.Logger.Info("hasher_pop", zap.Int64("offset", pq.Offset), zap.Int64("dataBytes", pq.DataBytes), zap.Any("toFinalize", toFinalize), zap.Int64("dataSize", c.dataSize), zap.String("filename", fileName), zap.Int64("totalWritten", totalWritten))
bufSize := 2 * BufferSize
if pq.DataBytes < int64(bufSize) {
bufSize = int(pq.DataBytes)
@@ -474,7 +483,7 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil
pq.DataBytes -= int64(n)
pq.Offset += int64(n)
totalWritten += int64(n)
- _, err = c.Write(buf[:n])
+ _, err = c.md5hasher.Write(buf[:n])
if err != nil {
logging.Logger.Error("hasher_write", zap.Error(err), zap.Int("n", n), zap.Int64("offset", pq.Offset), zap.Int64("dataBytes", pq.DataBytes), zap.Int64("dataSize", c.dataSize), zap.String("filename", fileName), zap.Int64("totalWritten", totalWritten))
c.hashErr = err
@@ -483,13 +492,12 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil
}
buf = nil
if toFinalize {
- c.hashErr = c.Finalize()
return
}
}
}
-func (c *CommitHasher) Wait(ctx context.Context, connID, allocID, fileName, filePathHash string) error {
+func (c *CommitHasher) Wait(ctx context.Context) error {
select {
case <-c.doneChan:
return c.hashErr
@@ -568,3 +576,10 @@ func (c *CommitHasher) GetFixedMerkleRoot() string {
func (c *CommitHasher) GetValidationMerkleRoot() string {
return hex.EncodeToString(c.vt.GetValidationRoot())
}
+
+func (c *CommitHasher) GetMd5Hash() string {
+ hash := hex.EncodeToString(c.md5hasher.Sum(nil))
+ c.md5hasher.Reset()
+ md5Pool.Put(c.md5hasher)
+ return hash
+}
diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go
index 4d12fcfff..17c218d0b 100644
--- a/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go
+++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go
@@ -7,7 +7,7 @@ import (
"os"
"testing"
- "github.com/0chain/gosdk/core/util"
+ "github.com/0chain/gosdk_common/core/util"
"github.com/minio/sha256-simd"
)
diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go b/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go
index 5f10ac11c..7f16119e5 100644
--- a/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go
+++ b/code/go/0chain.net/blobbercore/filestore/tree_validation_test.go
@@ -11,7 +11,7 @@ import (
"testing"
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
- "github.com/0chain/gosdk/core/util"
+ "github.com/0chain/gosdk_common/core/util"
"github.com/stretchr/testify/require"
)
diff --git a/code/go/0chain.net/blobbercore/handler/auth_ticket.go b/code/go/0chain.net/blobbercore/handler/auth_ticket.go
index 89032880c..faa119c08 100644
--- a/code/go/0chain.net/blobbercore/handler/auth_ticket.go
+++ b/code/go/0chain.net/blobbercore/handler/auth_ticket.go
@@ -2,9 +2,14 @@ package handler
import (
"context"
+ "fmt"
+ "net/http"
+
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
"github.com/0chain/common/core/common"
- "net/http"
+ "github.com/0chain/gosdk_common/core/encryption"
+ "go.uber.org/zap"
)
// swagger:model AuthTicketResponse
@@ -19,28 +24,38 @@ type AuthTicketResponse struct {
//
// parameters:
//
-// +name: Zbox-Signature
-// in: header
-// type: string
-// description: Digital signature to verify that the sender is 0box service.
-// +name: client_id
-// type: string
-// in: query
-// description: Client ID is used as a payload to the token generated. The token represents a signed version of this string by the blobber's private key.
+// +name: Zbox-Signature
+// in: header
+// type: string
+// description: Digital signature to verify that the sender is 0box service.
+// +name: client_id
+// type: string
+// in: query
+// description: Client ID is used as a payload to the token generated. The token represents a signed version of this string by the blobber's private key.
//
// responses:
-// 200: AuthTicketResponse
+//
+// 200: AuthTicketResponse
func GenerateAuthTicket(ctx context.Context, r *http.Request) (interface{}, error) {
+
clientID := r.URL.Query().Get("client_id")
if clientID == "" {
return nil, common.NewError("missing_client_id", "client_id is required")
}
- signature, err := node.Self.Sign(clientID)
+ round := r.URL.Query().Get("round")
+ if round == "" {
+ return nil, common.NewError("missing_round", "round is required")
+ }
+
+ payload := encryption.Hash(fmt.Sprintf("%s_%s", clientID, round))
+
+ logging.Logger.Info("GenerateAuthTicket", zap.String("payload", payload), zap.String("client_id", clientID), zap.String("round", round))
+
+ signature, err := node.Self.Sign(payload)
if err != nil {
return nil, common.NewError("signature_failed", "signature failed")
}
-
return &AuthTicketResponse{
AuthTicket: signature,
}, nil
diff --git a/code/go/0chain.net/blobbercore/handler/chunk_encoder.go b/code/go/0chain.net/blobbercore/handler/chunk_encoder.go
index 6665b9dac..97e0d8f7d 100644
--- a/code/go/0chain.net/blobbercore/handler/chunk_encoder.go
+++ b/code/go/0chain.net/blobbercore/handler/chunk_encoder.go
@@ -4,7 +4,7 @@ import (
"bytes"
"errors"
- zencryption "github.com/0chain/gosdk/zboxcore/encryption"
+ zencryption "github.com/0chain/gosdk_common/zboxcore/encryption"
)
// ChunkEncoder encode/decode chunk data
diff --git a/code/go/0chain.net/blobbercore/handler/client_quota.go b/code/go/0chain.net/blobbercore/handler/client_quota.go
index cee45a830..4335192d4 100644
--- a/code/go/0chain.net/blobbercore/handler/client_quota.go
+++ b/code/go/0chain.net/blobbercore/handler/client_quota.go
@@ -8,6 +8,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+ "go.uber.org/zap"
"gorm.io/gorm"
)
@@ -84,6 +86,7 @@ func AddWriteMarkerCount(clientID string, zeroSizeWM bool) {
cs.TotalZeroWM++
}
if cs.TotalZeroWM > config.Configuration.CommitZeroLimitDaily || cs.TotalWM > config.Configuration.CommitLimitDaily {
+ logging.Logger.Info("Client blacklisted", zap.String("client_id", clientID), zap.Int64("total_write_marker", cs.TotalWM), zap.Int64("total_zero_write_marker", cs.TotalZeroWM), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily), zap.Int64("commit_zero_limit_daily", config.Configuration.CommitZeroLimitDaily))
SetBlacklist(clientID)
}
}
@@ -136,6 +139,7 @@ func saveClientStats() {
}
func startBlackListWorker(ctx context.Context) {
+ logging.Logger.Info("Starting black list worker", zap.Int64("upload_limit", config.Configuration.UploadLimitMonthly), zap.Int64("download_limit", config.Configuration.BlockLimitMonthly), zap.Int64("commit_limit", config.Configuration.CommitLimitMonthly), zap.Int64("commit_zero_limit", config.Configuration.CommitZeroLimitDaily), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily))
BlackListWorkerTime := 24 * time.Hour
if config.Development() {
BlackListWorkerTime = 10 * time.Second
diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go
index 38497b283..2fcdebc96 100644
--- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go
+++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go
@@ -4,8 +4,9 @@ import (
"context"
"errors"
"net/http"
+ "path/filepath"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"gorm.io/gorm"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
@@ -41,6 +42,10 @@ func (cmd *DeleteFileCommand) IsValidated(ctx context.Context, req *http.Request
return common.NewError("invalid_parameters", "Invalid path")
}
+ if filepath.Clean(path) != path {
+ return common.NewError("invalid_parameters", "Invalid path")
+ }
+
cmd.path = path
connectionID, ok := common.GetField(req, "connection_id")
@@ -50,13 +55,23 @@ func (cmd *DeleteFileCommand) IsValidated(ctx context.Context, req *http.Request
cmd.connectionID = connectionID
var err error
lookUpHash := reference.GetReferenceLookup(allocationObj.ID, path)
- cmd.existingFileRef, err = reference.GetLimitedRefFieldsByLookupHashWith(ctx, allocationObj.ID, lookUpHash, []string{"path", "name", "size", "hash", "fixed_merkle_root"})
+ cmd.existingFileRef, err = reference.GetLimitedRefFieldsByLookupHashWith(ctx, allocationObj.ID, lookUpHash, []string{"path", "name", "type", "id", "size"})
if err != nil {
if errors.Is(gorm.ErrRecordNotFound, err) {
return common.ErrFileWasDeleted
}
return common.NewError("bad_db_operation", err.Error())
}
+ if cmd.existingFileRef.Type == reference.DIRECTORY {
+ // check if directory is empty
+ empty, err := reference.IsDirectoryEmpty(ctx, cmd.existingFileRef.ID)
+ if err != nil {
+ return common.NewError("bad_db_operation", err.Error())
+ }
+ if !empty {
+ return common.NewError("invalid_operation", "Directory is not empty")
+ }
+ }
cmd.existingFileRef.LookupHash = lookUpHash
return nil
}
@@ -82,12 +97,10 @@ func (cmd *DeleteFileCommand) ProcessContent(_ context.Context, allocationObj *a
connectionID := cmd.connectionID
cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionID,
AllocationID: allocationObj.ID, Name: cmd.existingFileRef.Name,
- Hash: cmd.existingFileRef.Hash, Path: cmd.existingFileRef.Path, Size: deleteSize}
+ LookupHash: cmd.existingFileRef.LookupHash, Path: cmd.existingFileRef.Path, Size: deleteSize, Type: cmd.existingFileRef.Type}
result := allocation.UploadResult{}
result.Filename = cmd.existingFileRef.Name
- result.ValidationRoot = cmd.existingFileRef.ValidationRoot
- result.FixedMerkleRoot = cmd.existingFileRef.FixedMerkleRoot
result.Size = cmd.existingFileRef.Size
result.UpdateChange = true
diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go
index 1b1daf5d4..1bfe4df13 100644
--- a/code/go/0chain.net/blobbercore/handler/file_command_update.go
+++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go
@@ -14,9 +14,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
- sdkConst "github.com/0chain/gosdk/constants"
- "github.com/0chain/gosdk/zboxcore/fileref"
+ sdkConst "github.com/0chain/gosdk_common/constants"
+ "github.com/0chain/gosdk_common/zboxcore/fileref"
)
const (
@@ -72,19 +71,19 @@ func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request
return common.NewError("invalid_connection", "Invalid connection id")
}
- cmd.fileChanger.PathHash = encryption.Hash(cmd.fileChanger.Path)
+ cmd.fileChanger.LookupHash = reference.GetReferenceLookup(allocationObj.ID, cmd.fileChanger.Path)
if cmd.fileChanger.ChunkSize <= 0 {
cmd.fileChanger.ChunkSize = fileref.CHUNK_SIZE
}
- cmd.existingFileRef = allocation.GetExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash)
+ cmd.existingFileRef = allocation.GetExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash)
if cmd.existingFileRef == nil {
cmd.existingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, cmd.fileChanger.Path)
if cmd.existingFileRef == nil {
return common.NewError("invalid_file_update", "File at path does not exist for update")
}
- allocation.SaveExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash, cmd.existingFileRef) //nolint:errcheck
+ allocation.SaveExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash, cmd.existingFileRef) //nolint:errcheck
}
thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile)
@@ -112,7 +111,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj
result.Filename = cmd.fileChanger.Filename
defer cmd.contentFile.Close()
- filePathHash := cmd.fileChanger.PathHash
+ filePathHash := cmd.fileChanger.LookupHash
connID := cmd.fileChanger.ConnectionID
fileInputData := &filestore.FileInputData{
@@ -120,7 +119,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj
Path: cmd.fileChanger.Path,
UploadOffset: cmd.fileChanger.UploadOffset,
IsFinal: cmd.fileChanger.IsFinal,
- FilePathHash: filePathHash,
+ LookupHash: filePathHash,
Size: cmd.fileChanger.Size,
}
fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connID, fileInputData, cmd.contentFile)
@@ -128,8 +127,6 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj
return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error())
}
- result.ValidationRoot = fileOutputData.ValidationRoot
- result.FixedMerkleRoot = fileOutputData.FixedMerkleRoot
result.Size = fileOutputData.Size
cmd.fileChanger.AllocationID = allocationObj.ID
@@ -156,7 +153,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj
}
}
- saveChange, err := allocation.SaveFileChange(ctx, connID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size)
+ saveChange, err := allocation.SaveFileChange(ctx, connID, cmd.fileChanger.LookupHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size)
if err != nil {
return result, err
}
@@ -182,7 +179,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat
connectionID := cmd.fileChanger.ConnectionID
if cmd.thumbHeader != nil {
defer cmd.thumbFile.Close()
- thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.PathHash}
+ thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, LookupHash: cmd.fileChanger.LookupHash}
thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile)
if err != nil {
return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error())
@@ -196,7 +193,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat
}
func (cmd *UpdateFileCommand) reloadChange() {
- changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash)
+ changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash)
if changer != nil && changer.ThumbnailHash != "" {
cmd.fileChanger.ThumbnailFilename = changer.ThumbnailFilename
cmd.fileChanger.ThumbnailSize = changer.ThumbnailSize
diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go
index 88fea77f9..4dbd00544 100644
--- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go
+++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go
@@ -17,10 +17,9 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/constants"
- "github.com/0chain/gosdk/zboxcore/fileref"
+ "github.com/0chain/gosdk_common/constants"
+ "github.com/0chain/gosdk_common/zboxcore/fileref"
)
const (
@@ -77,25 +76,15 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request
return common.NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fileChanger.Path))
}
+ if filepath.Clean(fileChanger.Path) != fileChanger.Path {
+ return common.NewError("invalid_path", fmt.Sprintf("%v is not a clean path", fileChanger.Path))
+ }
+
if fileChanger.ConnectionID == "" {
return common.NewError("invalid_connection", "Invalid connection id")
}
- fileChanger.PathHash = encryption.Hash(fileChanger.Path)
-
- if fileChanger.UploadOffset == 0 {
- isExist, err := reference.IsRefExist(ctx, allocationObj.ID, fileChanger.Path)
-
- if err != nil {
- logging.Logger.Error(err.Error())
- return common.NewError("database_error", "Got db error while getting ref")
- }
-
- if isExist {
- msg := fmt.Sprintf("File at path :%s: already exists", fileChanger.Path)
- return common.NewError("duplicate_file", msg)
- }
- }
+ fileChanger.LookupHash = reference.GetReferenceLookup(allocationObj.ID, fileChanger.Path)
thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile)
if thumbHeader != nil {
@@ -111,6 +100,9 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request
if fileChanger.ChunkSize <= 0 {
fileChanger.ChunkSize = fileref.CHUNK_SIZE
}
+ if allocationObj.CanUpdate() {
+ fileChanger.CanUpdate = true
+ }
origfile, _, err := req.FormFile(UploadFile)
if err != nil {
@@ -135,7 +127,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj
ChunkSize: cmd.fileChanger.ChunkSize,
UploadOffset: cmd.fileChanger.UploadOffset,
IsFinal: cmd.fileChanger.IsFinal,
- FilePathHash: cmd.fileChanger.PathHash,
+ LookupHash: cmd.fileChanger.LookupHash,
Size: cmd.fileChanger.Size,
}
fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, fileInputData, cmd.contentFile)
@@ -144,10 +136,8 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj
return result, common.NewError("upload_error", "Failed to write file. "+err.Error())
}
result.Filename = cmd.fileChanger.Filename
- result.ValidationRoot = fileOutputData.ValidationRoot
result.Size = fileOutputData.Size
- allocationSize := allocation.GetConnectionObjSize(connectionID) + cmd.fileChanger.Size
cmd.fileChanger.AllocationID = allocationObj.ID
cmd.allocationChange = &allocation.AllocationChange{}
@@ -171,7 +161,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj
}
}
- saveChange, err := allocation.SaveFileChange(ctx, connectionID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size)
+ saveChange, err := allocation.SaveFileChange(ctx, connectionID, cmd.fileChanger.LookupHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size)
if err != nil {
logging.Logger.Error("UploadFileCommand.ProcessContent", zap.Error(err))
return result, err
@@ -185,6 +175,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj
return result, err
}
}
+ allocationSize := allocation.GetConnectionObjSize(connectionID)
if allocationObj.BlobberSizeUsed+allocationSize > allocationObj.BlobberSize {
return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber")
@@ -195,12 +186,12 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj
// ProcessThumbnail flush thumbnail file to FileStorage if it has.
func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocation) error {
- logging.Logger.Info("ProcessThumbnail: ", zap.String("allocationID: ", cmd.fileChanger.AllocationID))
+
connectionID := cmd.fileChanger.ConnectionID
if cmd.thumbHeader != nil {
defer cmd.thumbFile.Close()
- thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.PathHash}
+ thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, LookupHash: cmd.fileChanger.LookupHash}
thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile)
if err != nil {
return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error())
@@ -214,7 +205,7 @@ func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat
}
func (cmd *UploadFileCommand) reloadChange() {
- changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash)
+ changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash)
if changer != nil && changer.ThumbnailHash != "" {
cmd.fileChanger.ThumbnailFilename = changer.ThumbnailFilename
cmd.fileChanger.ThumbnailSize = changer.ThumbnailSize
@@ -234,6 +225,7 @@ func (cmd *UploadFileCommand) AddChange(ctx context.Context) error {
connectionInput, _ := cmd.fileChanger.Marshal()
cmd.allocationChange.LookupHash = reference.GetReferenceLookup(cmd.fileChanger.AllocationID, cmd.fileChanger.Path)
cmd.allocationChange.Input = connectionInput
+ logging.Logger.Info("AddChange: ", zap.String("connectionID", cmd.allocationChange.ConnectionID), zap.String("lookupHash", cmd.allocationChange.LookupHash))
return cmd.allocationChange.Create(ctx)
}
diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go
index cc110b268..9f478547e 100644
--- a/code/go/0chain.net/blobbercore/handler/handler.go
+++ b/code/go/0chain.net/blobbercore/handler/handler.go
@@ -14,14 +14,14 @@ import (
"strings"
"time"
- "github.com/0chain/gosdk/core/zcncrypto"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
"github.com/go-openapi/runtime/middleware"
- "github.com/0chain/gosdk/constants"
- "github.com/0chain/gosdk/zboxcore/fileref"
+ "github.com/0chain/gosdk_common/constants"
+ "github.com/0chain/gosdk_common/zboxcore/fileref"
"github.com/didip/tollbooth/v6/limiter"
"github.com/gorilla/mux"
"github.com/spf13/viper"
@@ -285,7 +285,6 @@ func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF
defer func() {
tx.Rollback()
}()
-
res, err := handler(ctx, r)
return res, err
}
@@ -312,21 +311,21 @@ func Authenticate0Box(handler common.ReqRespHandlerf) common.ReqRespHandlerf {
signature := r.Header.Get("Zbox-Signature")
if signature == "" {
w.WriteHeader(http.StatusForbidden)
- w.Write([]byte("Invalid signature")) // nolint
+ w.Write([]byte("Invalid signature " + signature)) // nolint
return
}
signatureScheme := zcncrypto.NewSignatureScheme(config.Configuration.SignatureScheme)
if err := signatureScheme.SetPublicKey(common.PublicKey0box); err != nil {
w.WriteHeader(http.StatusForbidden)
- w.Write([]byte("Invalid signature")) // nolint
+ w.Write([]byte("Invalid signature 2")) // nolint
return
}
success, err := signatureScheme.Verify(signature, hex.EncodeToString([]byte(common.PublicKey0box)))
if err != nil || !success {
w.WriteHeader(http.StatusForbidden)
- w.Write([]byte("Invalid signature")) // nolint
+ w.Write([]byte("Invalid signature 3" + common.PublicKey0box)) // nolint
return
}
diff --git a/code/go/0chain.net/blobbercore/handler/handler_common.go b/code/go/0chain.net/blobbercore/handler/handler_common.go
index ebd6a9b64..340a82378 100644
--- a/code/go/0chain.net/blobbercore/handler/handler_common.go
+++ b/code/go/0chain.net/blobbercore/handler/handler_common.go
@@ -8,14 +8,12 @@ import (
"time"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
- "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp"
- "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker"
"github.com/0chain/blobber/code/go/0chain.net/core/build"
"github.com/0chain/blobber/code/go/0chain.net/core/chain"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/lock"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk_common/core/client"
"go.uber.org/zap"
. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
@@ -79,16 +77,19 @@ func HomepageHandler(w http.ResponseWriter, r *http.Request) {
)
fmt.Fprintf(w, "Miners ...\n")
- network := zcncore.GetNetwork()
- for _, miner := range network.Miners {
- fmt.Fprintf(w, "%v\n", miner)
- }
- fmt.Fprintf(w, "
\n")
- fmt.Fprintf(w, "Sharders ...\n")
- for _, sharder := range network.Sharders {
- fmt.Fprintf(w, "%v\n", sharder)
+ network, err := client.GetNetwork(context.Background())
+ if err == nil {
+ fmt.Fprintf(w, "
Miners ...\n")
+ for _, miner := range network.Miners {
+ fmt.Fprintf(w, "%v\n", miner)
+ }
+ fmt.Fprintf(w, "
\n")
+ fmt.Fprintf(w, "
Sharders ...\n")
+ for _, sharder := range network.Sharders {
+ fmt.Fprintf(w, "%v\n", sharder)
+ }
+ fmt.Fprintf(w, "
\n")
}
- fmt.Fprintf(w, "
Running since %v (Total elapsed time: %v)
\n", StartTime.Format(common.DateTimeFormat), time.Since(StartTime))
fmt.Fprintf(w, "")
@@ -127,10 +128,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu
mutex := lock.GetMutex(allocation.Allocation{}.TableName(), allocationID)
Logger.Info("Locking allocation", zap.String("allocation_id", allocationID))
- wmSet := writemarker.SetCommittingMarker(allocationID, true)
- if !wmSet {
- return nil, http.StatusBadRequest, common.NewError("pending_markers", "Committing marker set failed")
- }
mutex.Lock()
defer mutex.Unlock()
ctx = GetMetaDataStore().CreateTransaction(ctx)
@@ -144,7 +141,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu
if rollErr != nil {
Logger.Error("couldn't rollback", zap.Error(err))
}
- writemarker.SetCommittingMarker(allocationID, false)
}
}()
@@ -160,14 +156,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu
}
Logger.Info("commit_success", zap.String("allocation_id", allocationID), zap.Any("response", resp))
-
- if blobberRes, ok := resp.(*blobberhttp.CommitResult); ok {
- // Save the write marker data
- writemarker.SaveMarkerData(allocationID, blobberRes.WriteMarker.WM.Timestamp, blobberRes.WriteMarker.WM.ChainLength)
- } else {
- Logger.Error("Invalid response type for commit handler")
- return resp, http.StatusInternalServerError, common.NewError("invalid_response_type", "Invalid response type for commit handler")
- }
return
}
}
diff --git a/code/go/0chain.net/blobbercore/handler/handler_download_test.go b/code/go/0chain.net/blobbercore/handler/handler_download_test.go
index 70a0e5dfe..e02abab31 100644
--- a/code/go/0chain.net/blobbercore/handler/handler_download_test.go
+++ b/code/go/0chain.net/blobbercore/handler/handler_download_test.go
@@ -14,11 +14,11 @@ import (
"testing"
"time"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
- zencryption "github.com/0chain/gosdk/zboxcore/encryption"
- "github.com/0chain/gosdk/zboxcore/fileref"
- "github.com/0chain/gosdk/zboxcore/zboxutil"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
+ zencryption "github.com/0chain/gosdk_common/zboxcore/encryption"
+ "github.com/0chain/gosdk_common/zboxcore/fileref"
+ "github.com/0chain/gosdk_common/zboxcore/zboxutil"
"github.com/DATA-DOG/go-sqlmock"
"github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
@@ -60,11 +60,8 @@ func TestHandlers_Download(t *testing.T) {
clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}`
guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}`
- require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain"))
- clients := client.GetClients()
-
- ownerClient, guestClient := clients[0], clients[1]
-
+ ownerClient, _ := client.PopulateClient(clientJson, "bls0chain")
+ guestClient, _ := client.PopulateClient(guestClientJson, "bls0chain")
ownerScheme, err := getEncryptionScheme(ownerClient.Mnemonic)
if err != nil {
t.Fatal(err)
diff --git a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go
index 424b4c99f..bd073d88e 100644
--- a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go
+++ b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go
@@ -41,7 +41,7 @@ func UseRecovery(h http.Handler) http.Handler {
defer func() {
if err := recover(); err != nil {
escapedUrl := sanitizeString(r.URL.String())
- logging.Logger.Error("[recover]http", zap.String("url", escapedUrl), zap.Any("err", err))
+ logging.Logger.Error("[recover]http", zap.String("url", escapedUrl), zap.Any("err", err), zap.Stack("recover_stack"))
}
}()
diff --git a/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go b/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go
index 79ea68676..3217712c5 100644
--- a/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go
+++ b/code/go/0chain.net/blobbercore/handler/handler_objecttree_test.go
@@ -4,6 +4,7 @@
package handler
import (
+ "encoding/json"
"net/http"
"net/http/httptest"
"os"
@@ -11,8 +12,8 @@ import (
"testing"
"time"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
"github.com/DATA-DOG/go-sqlmock"
"github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
@@ -42,18 +43,18 @@ func TestHandlers_ObjectTree(t *testing.T) {
setup(t)
clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}`
- guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}`
- require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain"))
- clients := client.GetClients()
-
- ownerClient := clients[0]
+ ownerClient := zcncrypto.Wallet{}
+ err := json.Unmarshal([]byte(clientJson), &ownerClient)
+ require.NoError(t, err)
+ client.SetWallet(ownerClient)
+ client.SetSignatureScheme("bls0chain")
router, handlers := setupObjectTreeHandlers()
sch := zcncrypto.NewSignatureScheme("bls0chain")
//sch.Mnemonic = "expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe"
- _, err := sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe")
+ _, err = sch.RecoverKeys("expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe")
if err != nil {
t.Fatal(err)
}
diff --git a/code/go/0chain.net/blobbercore/handler/handler_refpath_test.go b/code/go/0chain.net/blobbercore/handler/handler_refpath_test.go
index 04b47d2a5..5926bfcdf 100644
--- a/code/go/0chain.net/blobbercore/handler/handler_refpath_test.go
+++ b/code/go/0chain.net/blobbercore/handler/handler_refpath_test.go
@@ -13,12 +13,11 @@ import (
"testing"
"time"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
"github.com/DATA-DOG/go-sqlmock"
"github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
@@ -44,12 +43,8 @@ func TestHandlers_ReferencePath(t *testing.T) {
setup(t)
clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}`
- guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}`
- require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain"))
- clients := client.GetClients()
-
- ownerClient := clients[0]
+ ownerClient, _ := client.PopulateClient(clientJson, "bls0chain")
router, handlers := setupReferencePathHandlers()
diff --git a/code/go/0chain.net/blobbercore/handler/handler_share_test.go b/code/go/0chain.net/blobbercore/handler/handler_share_test.go
index 9a288997a..cb5fe690d 100644
--- a/code/go/0chain.net/blobbercore/handler/handler_share_test.go
+++ b/code/go/0chain.net/blobbercore/handler/handler_share_test.go
@@ -15,9 +15,9 @@ import (
"testing"
"time"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
- "github.com/0chain/gosdk/zboxcore/fileref"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
+ "github.com/0chain/gosdk_common/zboxcore/fileref"
"github.com/DATA-DOG/go-sqlmock"
"github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
@@ -61,13 +61,8 @@ func TestHandlers_Share(t *testing.T) {
setup(t)
clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}`
- guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}`
-
- require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain"))
- clients := client.GetClients()
-
- ownerClient := clients[0]
+ ownerClient, _ := client.PopulateClient(clientJson, "bls0chain")
router, handlers := setupShareHandlers()
sch := zcncrypto.NewSignatureScheme("bls0chain")
diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go
index a7ed52633..ffba186ce 100644
--- a/code/go/0chain.net/blobbercore/handler/handler_test.go
+++ b/code/go/0chain.net/blobbercore/handler/handler_test.go
@@ -18,13 +18,12 @@ import (
"github.com/DATA-DOG/go-sqlmock"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zboxcore/client"
- "github.com/0chain/gosdk/zboxcore/fileref"
- "github.com/0chain/gosdk/zboxcore/marker"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
+ "github.com/0chain/gosdk_common/zboxcore/fileref"
+ "github.com/0chain/gosdk_common/zboxcore/marker"
"github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
"go.uber.org/zap"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
@@ -49,7 +48,7 @@ func resetMockFileBlock() {
mockFileBlock = []byte("mock")
}
-func signHash(client *client.Client, hash string) (string, error) {
+func signHash(client zcncrypto.Wallet, hash string) (string, error) {
retSignature := ""
for _, kv := range client.Keys {
ss := zcncrypto.NewSignatureScheme("bls0chain")
@@ -155,7 +154,7 @@ func isEndpointAllowGetReq(name string) bool {
}
}
-func GetAuthTicketForEncryptedFile(ownerClient *client.Client, allocationID, remotePath, fileHash, clientID, encPublicKey string) (string, error) {
+func GetAuthTicketForEncryptedFile(ownerClient zcncrypto.Wallet, allocationID, remotePath, fileHash, clientID, encPublicKey string) (string, error) {
at := &marker.AuthTicket{}
at.AllocationID = allocationID
at.OwnerID = ownerClient.ClientID
@@ -190,13 +189,7 @@ func TestHandlers_Requiring_Signature(t *testing.T) {
setup(t)
clientJson := `{"client_id":"2f34516ed8c567089b7b5572b12950db34a62a07e16770da14b15b170d0d60a9","client_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","keys":[{"public_key":"bc94452950dd733de3b4498afdab30ff72741beae0b82de12b80a14430018a09ba119ff0bfe69b2a872bded33d560b58c89e071cef6ec8388268d4c3e2865083","private_key":"9fef6ff5edc39a79c1d8e5eb7ca7e5ac14d34615ee49e6d8ca12ecec136f5907"}],"mnemonics":"expose culture dignity plastic digital couple promote best pool error brush upgrade correct art become lobster nature moment obtain trial multiply arch miss toe","version":"1.0","date_created":"2021-05-30 17:45:06.492093 +0545 +0545 m=+0.139083805"}`
- guestClientJson := `{"client_id":"213297e22c8282ff85d1d5c99f4967636fe68f842c1351b24bd497246cbd26d9","client_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","keys":[{"public_key":"7710b547897e0bddf93a28903875b244db4d320e4170172b19a5d51280c73522e9bb381b184fa3d24d6e1464882bf7f89d24ac4e8d05616d55eb857a6e235383","private_key":"19ca446f814dcd56e28e11d4147f73590a07c7f1a9a6012087808a8602024a08"}],"mnemonics":"crazy dutch object arrest jump fragile oak amateur taxi trigger gap aspect marriage hat slice wool island spike unlock alter include easily say ramp","version":"1.0","date_created":"2022-01-26T07:26:41+05:45"}`
-
- require.NoError(t, client.PopulateClients([]string{clientJson, guestClientJson}, "bls0chain"))
- clients := client.GetClients()
-
- ownerClient := clients[0]
-
+ ownerClient, _ := client.PopulateClient(clientJson, "bls0chain")
router, handlers := setupTestHandlers()
sch := zcncrypto.NewSignatureScheme("bls0chain")
diff --git a/code/go/0chain.net/blobbercore/handler/health.go b/code/go/0chain.net/blobbercore/handler/health.go
index e4da97bf7..b0e406c44 100644
--- a/code/go/0chain.net/blobbercore/handler/health.go
+++ b/code/go/0chain.net/blobbercore/handler/health.go
@@ -7,6 +7,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
+ coreTxn "github.com/0chain/gosdk_common/core/transaction"
"go.uber.org/zap"
)
@@ -29,48 +30,41 @@ func getBlobberHealthCheckError() error {
return err
}
-func BlobberHealthCheck() (*transaction.Transaction, error) {
+func BlobberHealthCheck() (string, error) {
if config.Configuration.Capacity == 0 {
setBlobberHealthCheckError(ErrBlobberHasRemoved)
- return nil, ErrBlobberHasRemoved
+ return "", ErrBlobberHasRemoved
}
- txn, err := transaction.NewTransactionEntity()
- if err != nil {
- setBlobberHealthCheckError(err)
- return nil, err
- }
-
- err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS,
- transaction.BLOBBER_HEALTH_CHECK, common.Now(), 0)
- if err != nil {
+ _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{
+ Name: transaction.BLOBBER_HEALTH_CHECK,
+ InputArgs: common.Now(),
+ }, true)
+ if err != nil || txn == nil {
logging.Logger.Error("Failed to health check blobber on the blockchain",
zap.Error(err))
setBlobberHealthCheckError(err)
- return nil, err
+ return "", err
}
setBlobberHealthCheckError(nil)
- return txn, nil
+ return txn.Hash, nil
}
-func ValidatorHealthCheck() (*transaction.Transaction, error) {
-
- txn, err := transaction.NewTransactionEntity()
-
- if err != nil {
-
- return nil, err
- }
+func ValidatorHealthCheck() (string, error) {
+ _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{
+ Name: transaction.VALIDATOR_HEALTH_CHECK,
+ InputArgs: common.Now(),
+ }, true)
- if err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.VALIDATOR_HEALTH_CHECK, common.Now(), 0); err != nil {
+ if err != nil || txn == nil {
logging.Logger.Error("Failed to health check validator on the blockchain",
zap.Error(err))
- return nil, err
+ return "", err
}
- return txn, err
+ return txn.Hash, err
}
diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go
index 968eaa2a1..b73e45ea4 100644
--- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go
+++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go
@@ -16,7 +16,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
@@ -88,14 +88,6 @@ func readPreRedeem(
return
}
-func checkPendingMarkers(ctx context.Context, allocationID string) error {
- pending := writemarker.CheckProcessingMarker(allocationID)
- if pending {
- return common.NewError("pending_markers", "previous marker is still pending to be redeemed")
- }
- return nil
-}
-
func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, writeMarker *writemarker.WriteMarker, payerID string) (err error) {
// check out read pool tokens if read_price > 0
var (
@@ -462,22 +454,17 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i
return nil, common.NewErrorf("download_file", "BlockNum or NumBlocks is too large to convert to int")
}
- fromPreCommit := false
+ fromPreCommit := alloc.AllocationVersion == fileref.AllocationVersion
if downloadMode == DownloadContentThumb {
-
- if fileref.IsPrecommit {
- fromPreCommit = fileref.ThumbnailHash != fileref.PrevThumbnailHash
- }
-
rbi := &filestore.ReadBlockInput{
AllocationID: alloc.ID,
FileSize: fileref.ThumbnailSize,
- Hash: fileref.ThumbnailHash,
+ Hash: fileref.LookupHash,
StartBlockNum: int(dr.BlockNum),
NumBlocks: int(dr.NumBlocks),
IsThumbnail: true,
IsPrecommit: fromPreCommit,
- FilestoreVersion: fileref.FilestoreVersion,
+ FilestoreVersion: filestore.VERSION,
}
logging.Logger.Info("calling GetFileBlock for thumb", zap.Any("rbi", rbi))
@@ -486,20 +473,15 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i
return nil, common.NewErrorf("download_file", "couldn't get thumbnail block: %v", err)
}
} else {
-
- if fileref.IsPrecommit {
- fromPreCommit = fileref.ValidationRoot != fileref.PrevValidationRoot
- }
-
rbi := &filestore.ReadBlockInput{
AllocationID: alloc.ID,
FileSize: fileref.Size,
- Hash: fileref.ValidationRoot,
+ Hash: fileref.LookupHash,
StartBlockNum: int(dr.BlockNum),
NumBlocks: int(dr.NumBlocks),
VerifyDownload: dr.VerifyDownload,
IsPrecommit: fromPreCommit,
- FilestoreVersion: fileref.FilestoreVersion,
+ FilestoreVersion: filestore.VERSION,
}
logging.Logger.Info("calling GetFileBlock", zap.Any("rbi", rbi))
fileDownloadResponse, err = filestore.GetFileStore().GetFileBlock(rbi)
@@ -588,7 +570,6 @@ func (fsh *StorageHandler) CreateConnection(ctx context.Context, r *http.Request
}
func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*blobberhttp.CommitResult, error) {
- var prevChainHash string
startTime := time.Now()
if r.Method == "GET" {
return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use POST instead")
@@ -629,12 +610,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
elapsedGetLock := time.Since(startTime) - elapsedAllocation
- err = checkPendingMarkers(ctx, allocationObj.ID)
- if err != nil {
- Logger.Error("Error checking pending markers", zap.Error(err))
- return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed")
- }
-
connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID)
if err != nil {
// might be good to check if blobber already has stored writemarker
@@ -642,12 +617,12 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
"Invalid connection id. Connection id was not found: %v", err)
}
if len(connectionObj.Changes) == 0 {
- if connectionObj.Status == allocation.NewConnection {
- return nil, common.NewError("invalid_parameters",
- "Invalid connection id. Connection not found.")
- }
- return nil, common.NewError("invalid_parameters",
- "Invalid connection id. Connection does not have any changes.")
+ logging.Logger.Info("commit_write_empty", zap.String("connection_id", connectionID))
+ }
+
+ if len(connectionObj.Changes) > config.Configuration.MaxConnectionChanges {
+ return nil, common.NewError("max_connection_changes",
+ "Max connection changes reached. A connection can only have "+fmt.Sprintf("%v", config.Configuration.MaxConnectionChanges)+" changes")
}
elapsedGetConnObj := time.Since(startTime) - elapsedAllocation - elapsedGetLock
@@ -656,184 +631,87 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation")
}
- writeMarkerString := r.FormValue("write_marker")
- if writeMarkerString == "" {
- return nil, common.NewError("invalid_parameters", "Invalid write marker passed")
- }
- writeMarker := writemarker.WriteMarker{}
- err = json.Unmarshal([]byte(writeMarkerString), &writeMarker)
- if err != nil {
- return nil, common.NewErrorf("invalid_parameters",
- "Invalid parameters. Error parsing the writemarker for commit: %v",
- err)
+ if allocationObj.BlobberSizeUsed+connectionObj.Size > allocationObj.BlobberSize {
+ return nil, common.NewError("max_allocation_size",
+ "Max size reached for the allocation with this blobber")
}
var result blobberhttp.CommitResult
- var latestWriteMarkerEntity *writemarker.WriteMarkerEntity
- if allocationObj.AllocationRoot == "" {
- latestWriteMarkerEntity = nil
- } else {
- latestWriteMarkerEntity, err = writemarker.GetWriteMarkerEntity(ctx,
- allocationObj.AllocationRoot)
- if err != nil {
- return nil, common.NewErrorf("latest_write_marker_read_error",
- "Error reading the latest write marker for allocation: %v", err)
- }
- if latestWriteMarkerEntity.Status == writemarker.Failed {
- return nil, common.NewError("latest_write_marker_failed",
- "Latest write marker is in failed state")
- }
-
- if latestWriteMarkerEntity.Status != writemarker.Committed {
- writeMarker.ChainLength = latestWriteMarkerEntity.WM.ChainLength
- }
- prevChainHash = latestWriteMarkerEntity.WM.ChainHash
- }
-
- writemarkerEntity := &writemarker.WriteMarkerEntity{}
- writemarkerEntity.WM = writeMarker
- writemarkerEntity.WM.ChainLength += 1
- if writemarkerEntity.WM.ChainLength > config.Configuration.MaxChainLength {
- return nil, common.NewError("chain_length_exceeded", "Chain length exceeded")
- }
-
- err = writemarkerEntity.VerifyMarker(ctx, allocationObj, connectionObj, latestWriteMarkerEntity)
- if err != nil {
- result.AllocationRoot = allocationObj.AllocationRoot
- result.ErrorMessage = "Verification of write marker failed: " + err.Error()
- result.Success = false
- if latestWriteMarkerEntity != nil {
- result.WriteMarker = latestWriteMarkerEntity
- }
- Logger.Error("verify_writemarker_failed", zap.Error(err))
- return &result, common.NewError("write_marker_verification_failed", result.ErrorMessage)
- }
-
- elapsedVerifyWM := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj
-
- var clientIDForWriteRedeem = writeMarker.ClientID
-
- if err := writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil {
- return nil, err
+ versionMarkerStr := r.FormValue("version_marker")
+ if versionMarkerStr == "" {
+ return nil, common.NewError("invalid_parameters", "Invalid version marker passed")
}
-
- elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock -
- elapsedGetConnObj - elapsedVerifyWM
-
- fileIDMetaStr := r.FormValue("file_id_meta")
- if fileIDMetaStr == "" {
- return nil, common.NewError("invalid_parameters", "Invalid file ID meta passed")
- }
- fileIDMeta := make(map[string]string, 0)
- err = json.Unmarshal([]byte(fileIDMetaStr), &fileIDMeta)
- if err != nil {
- return nil, common.NewError("unmarshall_error",
- fmt.Sprintf("Error while unmarshalling file ID meta data: %s", err.Error()))
- }
-
- // Move preCommitDir to finalDir
- err = connectionObj.MoveToFilestore(ctx)
+ versionMarker := writemarker.VersionMarker{}
+ err = json.Unmarshal([]byte(versionMarkerStr), &versionMarker)
if err != nil {
- return nil, common.NewError("move_to_filestore_error", fmt.Sprintf("Error while moving to filestore: %s", err.Error()))
+ return nil, common.NewError("unmarshall_error", fmt.Sprintf("Error while unmarshalling version marker: %s", err.Error()))
}
- elapsedMoveToFilestore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem
-
- rootRef, err := connectionObj.ApplyChanges(
- ctx, writeMarker.AllocationRoot, writeMarker.PreviousAllocationRoot, writeMarker.Timestamp, fileIDMeta)
+ err = versionMarker.Verify(allocationID, allocationObj.OwnerPublicKey)
if err != nil {
- Logger.Error("Error applying changes", zap.Error(err))
return nil, err
}
- if !rootRef.IsPrecommit {
- return nil, common.NewError("no_root_change", "No change in root ref")
- }
- connectionObj.Size = rootRef.Size - allocationObj.BlobberSizeUsed
-
- if writemarkerEntity.WM.Size != connectionObj.Size {
- return nil, common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker size %v does not match the connection size %v", writemarkerEntity.WM.Size, connectionObj.Size))
- }
-
- if allocationObj.BlobberSizeUsed+connectionObj.Size > allocationObj.BlobberSize {
- return nil, common.NewError("max_allocation_size",
- "Max size reached for the allocation with this blobber")
- }
- if latestWriteMarkerEntity != nil && latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size != writeMarker.ChainSize {
- return nil, common.NewErrorf("invalid_chain_size",
- "Invalid chain size. expected:%v got %v", latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size, writeMarker.ChainSize)
- } else if latestWriteMarkerEntity == nil && connectionObj.Size != writeMarker.ChainSize {
- return nil, common.NewErrorf("invalid_chain_size",
- "Invalid chain size. expected:%v got %v", connectionObj.Size, writeMarker.ChainSize)
- }
-
- elapsedApplyChanges := time.Since(startTime) - elapsedAllocation - elapsedGetLock -
- elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem
-
- allocationRoot := rootRef.Hash
- fileMetaRoot := rootRef.FileMetaHash
- if allocationRoot != writeMarker.AllocationRoot {
- result.AllocationRoot = allocationObj.AllocationRoot
- if latestWriteMarkerEntity != nil {
- result.WriteMarker = latestWriteMarkerEntity
+ // Move preCommitDir to finalDir
+ if allocationObj.IsRedeemRequired {
+ err = connectionObj.MoveToFilestore(ctx, allocationObj.AllocationVersion)
+ if err != nil {
+ return nil, common.NewError("move_to_filestore_error", fmt.Sprintf("Error while moving to filestore: %s", err.Error()))
}
- result.Success = false
- result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root." +
- " Expected hash: " + allocationRoot
- return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage)
- }
-
- chainHash := writemarker.CalculateChainHash(prevChainHash, allocationRoot)
- if chainHash != writeMarker.ChainHash {
- return nil, common.NewError("chain_hash_mismatch", "Chain hash in the write marker does not match the calculated chain hash")
}
- if fileMetaRoot != writeMarker.FileMetaRoot {
- // result.AllocationRoot = allocationObj.AllocationRoot
- if latestWriteMarkerEntity != nil {
- result.WriteMarker = latestWriteMarkerEntity
+ elapsedMoveToFilestore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj
+ if len(connectionObj.Changes) > 0 {
+ err = connectionObj.ApplyChanges(
+ ctx, common.Now(), versionMarker.Version)
+ if err != nil {
+ Logger.Error("Error applying changes", zap.Error(err))
+ return nil, err
}
- result.Success = false
- result.ErrorMessage = "File meta root in the write marker does not match the calculated file meta root." +
- " Expected hash: " + fileMetaRoot + "; Got: " + writeMarker.FileMetaRoot
- return &result, common.NewError("file_meta_root_mismatch", result.ErrorMessage)
}
- writemarkerEntity.ConnectionID = connectionObj.ID
- writemarkerEntity.ClientPublicKey = clientKey
+ elapsedApplyChanges := time.Since(startTime) - elapsedAllocation - elapsedGetLock -
+ elapsedGetConnObj - elapsedMoveToFilestore
db := datastore.GetStore().GetTransaction(ctx)
- writemarkerEntity.Latest = true
- if err = db.Create(writemarkerEntity).Error; err != nil {
- return nil, common.NewError("write_marker_error", "Error persisting the write marker")
+ err = db.Create(&versionMarker).Error
+ if err != nil {
+ return nil, common.NewError("db_error", fmt.Sprintf("Error while saving version marker: %s", err.Error()))
}
- allocationObj.AllocationRoot = allocationRoot
- allocationObj.FileMetaRoot = fileMetaRoot
- allocationObj.IsRedeemRequired = true
+ allocationObj.PrevBlobberSizeUsed = allocationObj.BlobberSizeUsed
+ allocationObj.PrevUsedSize = allocationObj.UsedSize
allocationObj.BlobberSizeUsed += connectionObj.Size
allocationObj.UsedSize += connectionObj.Size
+ allocationObj.AllocationVersion = versionMarker.Version
+ if len(connectionObj.Changes) == 0 {
+ allocationObj.IsRedeemRequired = false
+ } else {
+ allocationObj.IsRedeemRequired = true
+ }
updateMap := map[string]interface{}{
- "allocation_root": allocationRoot,
- "file_meta_root": fileMetaRoot,
- "used_size": allocationObj.UsedSize,
- "blobber_size_used": allocationObj.BlobberSizeUsed,
- "is_redeem_required": true,
+ "used_size": allocationObj.UsedSize,
+ "blobber_size_used": allocationObj.BlobberSizeUsed,
+ "is_redeem_required": allocationObj.IsRedeemRequired,
+ "allocation_version": versionMarker.Version,
+ "prev_used_size": allocationObj.PrevUsedSize,
+ "prev_blobber_size_used": allocationObj.PrevBlobberSizeUsed,
}
updateOption := func(a *allocation.Allocation) {
- a.AllocationRoot = allocationRoot
- a.FileMetaRoot = fileMetaRoot
- a.IsRedeemRequired = true
+ a.IsRedeemRequired = allocationObj.IsRedeemRequired
a.BlobberSizeUsed = allocationObj.BlobberSizeUsed
a.UsedSize = allocationObj.UsedSize
+ a.AllocationVersion = allocationObj.AllocationVersion
+ a.PrevUsedSize = allocationObj.PrevUsedSize
+ a.PrevBlobberSizeUsed = allocationObj.PrevBlobberSizeUsed
}
if err = allocation.Repo.UpdateAllocation(ctx, allocationObj, updateMap, updateOption); err != nil {
- return nil, common.NewError("allocation_write_error", "Error persisting the allocation object")
+ return nil, common.NewError("allocation_write_error", "Error persisting the allocation object "+err.Error())
}
elapsedSaveAllocation := time.Since(startTime) - elapsedAllocation - elapsedGetLock -
- elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - elapsedApplyChanges
+ elapsedGetConnObj - elapsedApplyChanges
err = connectionObj.CommitToFileStore(ctx)
if err != nil {
@@ -841,17 +719,25 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
return nil, common.NewError("file_store_error", "Error committing to file store. "+err.Error())
}
}
- elapsedCommitStore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - elapsedApplyChanges - elapsedSaveAllocation
- logging.Logger.Info("commit_filestore", zap.String("allocation_id", allocationId), zap.String("allocation_root", allocationRoot))
+ elapsedCommitStore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedApplyChanges - elapsedSaveAllocation
+ logging.Logger.Info("commit_filestore", zap.String("allocation_id", allocationId))
connectionObj.DeleteChanges(ctx)
db.Model(connectionObj).Updates(allocation.AllocationChangeCollector{Status: allocation.CommittedConnection})
result.AllocationRoot = allocationObj.AllocationRoot
- result.WriteMarker = writemarkerEntity
result.Success = true
result.ErrorMessage = ""
- commitOperation := connectionObj.Changes[0].Operation
- input := connectionObj.Changes[0].Input
+ var (
+ commitOperation string
+ input string
+ )
+ if len(connectionObj.Changes) > 0 {
+ commitOperation = connectionObj.Changes[0].Operation
+ input = connectionObj.Changes[0].Input
+ } else {
+ commitOperation = "[commit]empty"
+ input = "[commit]empty"
+ }
//Delete connection object and its changes
@@ -861,13 +747,10 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
Logger.Info("[commit]"+commitOperation,
zap.String("alloc_id", allocationID),
- zap.String("allocation_root", writeMarker.AllocationRoot),
zap.String("input", input),
zap.Duration("get_alloc", elapsedAllocation),
zap.Duration("get-lock", elapsedGetLock),
zap.Duration("get-conn-obj", elapsedGetConnObj),
- zap.Duration("verify-wm", elapsedVerifyWM),
- zap.Duration("write-pre-redeem", elapsedWritePreRedeem),
zap.Duration("move-to-filestore", elapsedMoveToFilestore),
zap.Duration("apply-changes", elapsedApplyChanges),
zap.Duration("save-allocation", elapsedSaveAllocation),
@@ -906,7 +789,10 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i
if new_name == "" {
return nil, common.NewError("invalid_parameters", "Invalid name")
}
-
+ if filepath.Base(new_name) != new_name {
+ logging.Logger.Error("invalid_parameters", zap.String("new_name", new_name), zap.String("base", filepath.Base(new_name)))
+ return nil, common.NewError("invalid_parameters", "Invalid name")
+ }
pathHash, err := pathHashFromReq(r, allocationID)
if err != nil {
return nil, err
@@ -926,7 +812,7 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i
return nil, common.NewError("meta_error", "Error reading metadata for connection")
}
- objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root", "type"})
+ objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"})
if err != nil {
return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error())
@@ -936,6 +822,16 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i
return nil, common.NewError("invalid_operation", "cannot rename root path")
}
+ if objectRef.Type != reference.FILE {
+ isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID)
+ if err != nil {
+ return nil, common.NewError("invalid_operation", "Error checking if directory is empty")
+ }
+ if !isEmpty {
+ return nil, common.NewError("invalid_operation", "Directory is not empty")
+ }
+ }
+
allocationChange := &allocation.AllocationChange{}
allocationChange.ConnectionID = connectionObj.ID
allocationChange.Size = 0
@@ -954,9 +850,6 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i
result := &allocation.UploadResult{}
result.Filename = new_name
- result.Hash = objectRef.Hash
- result.ValidationRoot = objectRef.ValidationRoot
- result.FixedMerkleRoot = objectRef.FixedMerkleRoot
result.Size = objectRef.Size
return result, nil
@@ -993,7 +886,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int
if destPath == "" {
return nil, common.NewError("invalid_parameters", "Invalid destination for operation")
}
-
+ destPath = filepath.Clean(destPath)
pathHash, err := pathHashFromReq(r, allocationID)
if err != nil {
return nil, err
@@ -1013,7 +906,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int
return nil, common.NewError("meta_error", "Error reading metadata for connection")
}
- objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"})
+ objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"})
if err != nil {
return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error())
@@ -1021,7 +914,17 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int
if objectRef.ParentPath == destPath || objectRef.Path == destPath {
return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot copy to the same parent directory.")
}
+ if objectRef.Type == reference.DIRECTORY {
+ isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID)
+ if err != nil {
+ return nil, common.NewError("invalid_operation", "Error checking if directory is empty")
+ }
+ if !isEmpty {
+ return nil, common.NewError("invalid_operation", "Directory is not empty")
+ }
+ }
newPath := filepath.Join(destPath, objectRef.Name)
+ newPath = filepath.Clean(newPath)
paths, err := common.GetParentPaths(newPath)
if err != nil {
return nil, err
@@ -1052,8 +955,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int
allocationChange.LookupHash = pathHash
allocationChange.Operation = constants.FileOperationCopy
dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ID,
- AllocationID: connectionObj.AllocationID, DestPath: destPath}
- dfc.SrcPath = objectRef.Path
+ AllocationID: connectionObj.AllocationID, DestPath: newPath, Type: objectRef.Type, SrcPath: objectRef.Path}
allocation.UpdateConnectionObjSize(connectionID, allocationChange.Size)
connectionObj.AddChange(allocationChange, dfc)
@@ -1065,9 +967,6 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int
result := &allocation.UploadResult{}
result.Filename = objectRef.Name
- result.Hash = objectRef.Hash
- result.ValidationRoot = objectRef.ValidationRoot
- result.FixedMerkleRoot = objectRef.FixedMerkleRoot
result.Size = objectRef.Size
return result, nil
}
@@ -1103,6 +1002,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int
if destPath == "" {
return nil, common.NewError("invalid_parameters", "Invalid destination for operation")
}
+ destPath = filepath.Clean(destPath)
pathHash, err := pathHashFromReq(r, allocationID)
if err != nil {
@@ -1124,7 +1024,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int
}
objectRef, err := reference.GetLimitedRefFieldsByLookupHash(
- ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"})
+ ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"})
if err != nil {
return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error())
@@ -1133,7 +1033,17 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int
if objectRef.ParentPath == destPath {
return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot move to the same parent directory.")
}
+ if objectRef.Type == reference.DIRECTORY {
+ isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID)
+ if err != nil {
+ return nil, common.NewError("invalid_operation", "Error checking if directory is empty")
+ }
+ if !isEmpty {
+ return nil, common.NewError("invalid_operation", "Directory is not empty")
+ }
+ }
newPath := filepath.Join(destPath, objectRef.Name)
+ newPath = filepath.Clean(newPath)
paths, err := common.GetParentPaths(newPath)
if err != nil {
return nil, err
@@ -1167,7 +1077,8 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int
ConnectionID: connectionObj.ID,
AllocationID: connectionObj.AllocationID,
SrcPath: objectRef.Path,
- DestPath: destPath,
+ DestPath: newPath,
+ Type: objectRef.Type,
}
dfc.SrcPath = objectRef.Path
connectionObj.AddChange(allocationChange, dfc)
@@ -1180,9 +1091,6 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int
result := &allocation.UploadResult{}
result.Filename = objectRef.Name
- result.Hash = objectRef.Hash
- result.ValidationRoot = objectRef.ValidationRoot
- result.FixedMerkleRoot = objectRef.FixedMerkleRoot
result.Size = objectRef.Size
return result, nil
}
@@ -1194,7 +1102,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn
return nil, common.NewError("invalid_parameters", "Invalid path")
}
fileRef, err := reference.GetLimitedRefFieldsByPath(ctx, connectionObj.AllocationID, path,
- []string{"path", "name", "size", "hash", "validation_root", "fixed_merkle_root"})
+ []string{"path", "name", "size"})
if err != nil {
Logger.Error("invalid_file", zap.Error(err))
@@ -1209,7 +1117,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn
allocationChange.Operation = constants.FileOperationDelete
dfc := &allocation.DeleteFileChange{ConnectionID: connectionObj.ID,
AllocationID: connectionObj.AllocationID, Name: fileRef.Name,
- Hash: fileRef.Hash, Path: fileRef.Path, Size: deleteSize}
+ LookupHash: fileRef.LookupHash, Path: fileRef.Path, Size: deleteSize}
allocation.UpdateConnectionObjSize(connectionObj.ID, allocationChange.Size)
@@ -1217,9 +1125,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn
result := &allocation.UploadResult{}
result.Filename = fileRef.Name
- result.Hash = fileRef.Hash
- result.ValidationRoot = fileRef.ValidationRoot
- result.FixedMerkleRoot = fileRef.FixedMerkleRoot
+ result.Hash = fileRef.LookupHash
result.Size = fileRef.Size
return result, nil
@@ -1335,6 +1241,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*all
allocationID := ctx.Value(constants.ContextKeyAllocationID).(string)
allocationTx := ctx.Value(constants.ContextKeyAllocation).(string)
clientID := ctx.Value(constants.ContextKeyClient).(string)
+ logging.Logger.Info("writeFile", zap.String("allocation_id", allocationID))
connectionID, ok := common.GetField(r, "connection_id")
if !ok {
logging.Logger.Error("no_connection_id", zap.String("alloc_id", allocationID))
@@ -1359,7 +1266,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*all
if err != nil {
return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error())
}
-
+ logging.Logger.Info("writeFileAllocation")
if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID {
return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation")
}
@@ -1463,19 +1370,14 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob
return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error())
}
- if allocationObj.AllocationRoot == "" {
- Logger.Error("Allocation root is not set", zap.String("allocation_id", allocationObj.ID))
- return nil, common.NewError("invalid_parameters", "Allocation root is not set")
+ if allocationObj.AllocationVersion == 0 {
+ Logger.Error("Allocation version is 0", zap.String("allocation_id", allocationObj.ID))
+ return nil, common.NewError("invalid_parameters", "Allocation version is not set")
}
elapsedAllocation := time.Since(startTime)
allocationID := allocationObj.ID
- connectionID, ok := common.GetField(r, "connection_id")
- if !ok {
- return nil, common.NewError("invalid_parameters", "Invalid connection id passed")
- }
-
elapsedGetLock := time.Since(startTime) - elapsedAllocation
if clientID == "" || clientKey == "" {
@@ -1486,105 +1388,55 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob
return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation")
}
- writeMarkerString := r.FormValue("write_marker")
- writeMarker := writemarker.WriteMarker{}
- err = json.Unmarshal([]byte(writeMarkerString), &writeMarker)
+ if !allocationObj.IsRedeemRequired {
+ return nil, common.NewError("invalid_operation", "Last commit is rollback")
+ }
+
+ versionMarkerString := r.FormValue("version_marker")
+ versionMarker := writemarker.VersionMarker{}
+ err = json.Unmarshal([]byte(versionMarkerString), &versionMarker)
if err != nil {
return nil, common.NewErrorf("invalid_parameters",
"Invalid parameters. Error parsing the writemarker for commit: %v",
err)
}
- var result blobberhttp.CommitResult
+ if versionMarker.IsRepair {
+ return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Rollback marker cannot be a repair marker")
+ }
- var latestWriteMarkerEntity *writemarker.WriteMarkerEntity
- latestWriteMarkerEntity, err = writemarker.GetWriteMarkerEntity(ctx,
- allocationObj.AllocationRoot)
+ var result blobberhttp.CommitResult
+ err = versionMarker.Verify(allocationID, allocationObj.OwnerPublicKey)
if err != nil {
- return nil, common.NewErrorf("latest_write_marker_read_error",
- "Error reading the latest write marker for allocation: %v", err)
- }
- if latestWriteMarkerEntity == nil {
- return nil, common.NewError("latest_write_marker_not_found",
- "Latest write marker not found for allocation")
+ return nil, common.NewError("invalid_parameters", "Invalid version marker passed: "+err.Error())
}
- writemarkerEntity := &writemarker.WriteMarkerEntity{}
- writemarkerEntity.WM = writeMarker
+ if versionMarker.Version == allocationObj.AllocationVersion {
+ return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Version marker is same as the current version")
+ }
- err = writemarkerEntity.VerifyRollbackMarker(ctx, allocationObj, latestWriteMarkerEntity)
+ currentVersionMarker, err := writemarker.GetCurrentVersion(ctx, allocationID)
if err != nil {
- return nil, common.NewError("write_marker_verification_failed", "Verification of the write marker failed: "+err.Error())
+ return nil, common.NewError("invalid_parameters", "Error getting the current version marker")
}
-
- if writemarkerEntity.WM.ChainLength > config.Configuration.MaxChainLength {
- return nil, common.NewError("chain_length_exceeded", "Chain length exceeded")
+ if currentVersionMarker.IsRepair {
+ return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Allocation is in repair mode")
}
-
- elapsedVerifyWM := time.Since(startTime) - elapsedAllocation - elapsedGetLock
-
- var clientIDForWriteRedeem = writeMarker.ClientID
-
- if err := writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil {
- return nil, err
+ if versionMarker.Version != currentVersionMarker.Version-1 {
+ return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Version marker is not the previous version")
}
- elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM
+ elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock
timeoutCtx, cancel := context.WithTimeout(ctx, 45*time.Second)
defer cancel()
c := datastore.GetStore().CreateTransaction(timeoutCtx)
txn := datastore.GetStore().GetTransaction(c)
- err = allocation.ApplyRollback(c, allocationID)
+ err = allocation.ApplyRollback(c, allocationID, allocationObj.AllocationVersion)
if err != nil {
txn.Rollback()
return nil, common.NewError("allocation_rollback_error", "Error applying the rollback for allocation: "+err.Error())
}
- elapsedApplyRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM - elapsedWritePreRedeem
-
- //get allocation root and ref
- rootRef, err := reference.GetLimitedRefFieldsByPath(c, allocationID, "/", []string{"hash", "file_meta_hash", "is_precommit"})
- if err != nil && err != gorm.ErrRecordNotFound {
- txn.Rollback()
- return nil, common.NewError("root_ref_read_error", "Error reading the root reference: "+err.Error())
- }
- if err == gorm.ErrRecordNotFound {
- rootRef = &reference.Ref{}
- }
-
- Logger.Info("rollback_root_ref", zap.Any("root_ref", rootRef))
- allocationRoot := rootRef.Hash
- fileMetaRoot := rootRef.FileMetaHash
-
- if allocationRoot != writeMarker.AllocationRoot {
- result.AllocationRoot = allocationObj.AllocationRoot
- result.WriteMarker = latestWriteMarkerEntity
- result.Success = false
- result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root." +
- " Expected hash: " + allocationRoot
- txn.Rollback()
- return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage)
- }
-
- chainHash := writemarker.CalculateChainHash(latestWriteMarkerEntity.WM.ChainHash, allocationRoot)
- if chainHash != writeMarker.ChainHash {
- txn.Rollback()
- return nil, common.NewError("chain_hash_mismatch", "Chain hash in the write marker does not match the calculated chain hash")
- }
-
- if fileMetaRoot != writeMarker.FileMetaRoot {
- if latestWriteMarkerEntity != nil {
- result.WriteMarker = latestWriteMarkerEntity
- }
- result.Success = false
- result.ErrorMessage = "File meta root in the write marker does not match the calculated file meta root." +
- " Expected hash: " + fileMetaRoot + "; Got: " + writeMarker.FileMetaRoot
- txn.Rollback()
- return &result, common.NewError("file_meta_root_mismatch", result.ErrorMessage)
- }
-
- writemarkerEntity.ConnectionID = connectionID
- writemarkerEntity.ClientPublicKey = clientKey
- Logger.Info("rollback_writemarker", zap.Any("writemarker", writemarkerEntity.WM))
+ elapsedApplyRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedWritePreRedeem
alloc, err := allocation.Repo.GetByIdAndLock(c, allocationID)
Logger.Info("[rollback]Lock Allocation", zap.Bool("is_redeem_required", alloc.IsRedeemRequired), zap.String("allocation_root", alloc.AllocationRoot), zap.String("latest_wm_redeemed", alloc.LatestRedeemedWM))
@@ -1593,28 +1445,25 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob
return &result, common.NewError("allocation_read_error", "Error reading the allocation object")
}
- alloc.BlobberSizeUsed -= latestWriteMarkerEntity.WM.Size
- alloc.UsedSize -= latestWriteMarkerEntity.WM.Size
- alloc.AllocationRoot = allocationRoot
- alloc.FileMetaRoot = fileMetaRoot
- alloc.IsRedeemRequired = true
+ alloc.BlobberSizeUsed = alloc.PrevBlobberSizeUsed
+ alloc.UsedSize = alloc.PrevUsedSize
+ alloc.IsRedeemRequired = false
+ alloc.AllocationVersion = versionMarker.Version
updateMap := map[string]interface{}{
"blobber_size_used": alloc.BlobberSizeUsed,
"used_size": alloc.UsedSize,
- "allocation_root": alloc.AllocationRoot,
- "file_meta_root": alloc.FileMetaRoot,
- "is_redeem_required": true,
+ "is_redeem_required": false,
+ "allocation_version": versionMarker.Version,
}
updateOption := func(a *allocation.Allocation) {
a.BlobberSizeUsed = alloc.BlobberSizeUsed
a.UsedSize = alloc.UsedSize
- a.AllocationRoot = alloc.AllocationRoot
+ a.AllocationVersion = alloc.AllocationVersion
a.FileMetaRoot = alloc.FileMetaRoot
a.IsRedeemRequired = alloc.IsRedeemRequired
}
- writemarkerEntity.Latest = true
- err = txn.Create(writemarkerEntity).Error
+ err = txn.Create(&versionMarker).Error
if err != nil {
txn.Rollback()
return &result, common.NewError("write_marker_error", "Error persisting the write marker "+err.Error())
@@ -1633,9 +1482,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob
Logger.Error("Error committing the rollback for allocation", zap.Error(err))
}
- elapsedCommitRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM - elapsedWritePreRedeem
- result.AllocationRoot = allocationObj.AllocationRoot
- result.WriteMarker = writemarkerEntity
+ elapsedCommitRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedWritePreRedeem
result.Success = true
result.ErrorMessage = ""
commitOperation := "rollback"
@@ -1644,7 +1491,6 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob
zap.String("alloc_id", allocationID),
zap.Duration("get_alloc", elapsedAllocation),
zap.Duration("get-lock", elapsedGetLock),
- zap.Duration("verify-wm", elapsedVerifyWM),
zap.Duration("write-pre-redeem", elapsedWritePreRedeem),
zap.Duration("apply-rollback", elapsedApplyRollback),
zap.Duration("total", time.Since(startTime)),
diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go
index adcb74109..0d0094022 100644
--- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go
+++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_bench_test.go
@@ -11,7 +11,7 @@ package handler
// "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
// "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
// "github.com/0chain/blobber/code/go/0chain.net/blobbercore/mock"
-// "github.com/0chain/gosdk/zboxcore/sdk"
+// "github.com/0chain/gosdk_common/zboxcore/sdk"
// )
// func BenchmarkUploadFileWithDisk(b *testing.B) {
diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go
index ccc627dc9..b1f33365b 100644
--- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go
+++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go
@@ -16,23 +16,24 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
mocket "github.com/selvatico/go-mocket"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
- "github.com/0chain/gosdk/zboxcore/fileref"
+ "github.com/0chain/gosdk_common/zboxcore/fileref"
- "github.com/0chain/gosdk/zboxcore/blockchain"
+ "github.com/0chain/gosdk_common/zboxcore/blockchain"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- c_common "github.com/0chain/gosdk/core/common"
- "github.com/0chain/gosdk/zboxcore/marker"
+ c_common "github.com/0chain/gosdk_common/core/common"
+ "github.com/0chain/gosdk_common/zboxcore/marker"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"github.com/stretchr/testify/require"
"testing"
@@ -56,10 +57,8 @@ func TestDownloadFile(t *testing.T) {
)
ts := time.Now().Add(time.Hour)
var mockLongTimeInFuture = common.Timestamp(ts.Unix()) + common.Timestamp(time.Second*1000)
- var mockClient client.Client
- require.NoError(t, json.Unmarshal([]byte(mockClientWallet), &mockClient))
- var mockOwner client.Client
- require.NoError(t, json.Unmarshal([]byte(mockOwnerWallet), &mockOwner))
+ mockClient, _ := client.PopulateClient(mockClientWallet, "bls0chain")
+ mockOwner, _ := client.PopulateClient(mockOwnerWallet, "bls0chain")
var (
now = c_common.Timestamp(time.Now().Unix())
)
@@ -84,7 +83,7 @@ func TestDownloadFile(t *testing.T) {
isRevoked bool
isFundedBlobber bool
isFunded0Chain bool
- payerId client.Client
+ payerId zcncrypto.Wallet
// client input from gosdk's BlockDownloadRequest,
inData blockDownloadRequest
@@ -121,15 +120,17 @@ func TestDownloadFile(t *testing.T) {
if p.useAuthTicket {
authTicket := &marker.AuthTicket{
AllocationID: p.inData.allocationID,
- ClientID: client.GetClientID(),
+ ClientID: client.Wallet().ClientID,
Expiration: int64(time.Duration(now) + 10000*time.Second),
OwnerID: mockOwner.ClientID,
Timestamp: int64(common.Now()),
FilePathHash: p.inData.pathHash,
}
- require.NoError(t, client.PopulateClient(mockOwnerWallet, "bls0chain"))
+ _, err := client.PopulateClient(mockOwnerWallet, "bls0chain")
+ require.NoError(t, err)
require.NoError(t, authTicket.Sign())
- require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain"))
+ _, err = client.PopulateClient(mockClientWallet, "bls0chain")
+ require.NoError(t, err)
authTicketBytes, _ := json.Marshal(authTicket)
auth := base64.StdEncoding.EncodeToString(authTicketBytes)
req.Header.Set("X-Auth-Token", auth)
@@ -139,8 +140,8 @@ func TestDownloadFile(t *testing.T) {
}
}
- makeMockMakeSCRestAPICall := func(t *testing.T, p parameters) func(scAddress string, relativePath string, params map[string]string) ([]byte, error) {
- return func(scAddress string, relativePath string, params map[string]string) ([]byte, error) {
+ makeMockMakeSCRestAPICall := func(t *testing.T, p parameters) func(scAddress string, relativePath string, params map[string]string, options ...string) ([]byte, error) {
+ return func(scAddress string, relativePath string, params map[string]string, options ...string) ([]byte, error) {
require.New(t)
require.EqualValues(t, scAddress, transaction.STORAGE_CONTRACT_ADDRESS)
switch relativePath {
@@ -290,9 +291,9 @@ func TestDownloadFile(t *testing.T) {
setupCtx := func(p parameters) context.Context {
ctx := context.TODO()
- ctx = context.WithValue(ctx, constants.ContextKeyClient, client.GetClientID())
+ ctx = context.WithValue(ctx, constants.ContextKeyClient, client.Id())
ctx = context.WithValue(ctx, constants.ContextKeyAllocation, p.inData.allocationTx)
- ctx = context.WithValue(ctx, constants.ContextKeyClientKey, client.GetClientPublicKey())
+ ctx = context.WithValue(ctx, constants.ContextKeyClientKey, client.PublicKey())
ctx = datastore.GetStore().CreateTransaction(ctx)
@@ -455,9 +456,11 @@ func TestDownloadFile(t *testing.T) {
func(t *testing.T) {
setupParams(&test.parameters)
if test.parameters.isOwner {
- require.NoError(t, client.PopulateClient(mockOwnerWallet, "bls0chain"))
+ _, err := client.PopulateClient(mockOwnerWallet, "bls0chain")
+ require.NoError(t, err)
} else {
- require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain"))
+ _, err := client.PopulateClient(mockClientWallet, "bls0chain")
+ require.NoError(t, err)
}
transaction.MakeSCRestAPICall = makeMockMakeSCRestAPICall(t, test.parameters)
request := setupRequest(test.parameters)
diff --git a/code/go/0chain.net/blobbercore/handler/protocol.go b/code/go/0chain.net/blobbercore/handler/protocol.go
index 976828558..9a2141db6 100644
--- a/code/go/0chain.net/blobbercore/handler/protocol.go
+++ b/code/go/0chain.net/blobbercore/handler/protocol.go
@@ -3,19 +3,19 @@ package handler
import (
"context"
"errors"
- "fmt"
"sync"
- "time"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
+ "github.com/0chain/gosdk_common/core/client"
+ coreTxn "github.com/0chain/gosdk_common/core/transaction"
+
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
- "github.com/0chain/blobber/code/go/0chain.net/core/util"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk_common/zcncore"
"go.uber.org/zap"
)
@@ -67,6 +67,7 @@ func getStorageNode() (*transaction.StorageNode, error) {
sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge
sn.IsEnterprise = config.Configuration.IsEnterprise
+ sn.StorageVersion = 1
return sn, nil
}
@@ -79,19 +80,13 @@ func RegisterBlobber(ctx context.Context) error {
})
if err != nil { // blobber is not registered yet
- txn, err := sendSmartContractBlobberAdd(ctx)
- if err != nil {
- logging.Logger.Error("Error when sending add request to blockchain", zap.Any("err", err))
- return err
- }
-
- t, err := TransactionVerify(txn)
+ txn, err := sendSmartContractBlobberAdd()
if err != nil {
- logging.Logger.Error("Failed to verify blobber register transaction", zap.Any("err", err), zap.String("txn.Hash", txn.Hash))
+ logging.Logger.Error("Error in add blobber", zap.Any("err", err))
return err
}
- logging.Logger.Info("Verified blobber register transaction", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput))
+ logging.Logger.Info("Verified blobber register transaction", zap.String("txn_hash", txn.Hash), zap.Any("txn_output", txn.TransactionOutput))
return nil
}
@@ -105,38 +100,31 @@ func RegisterBlobber(ctx context.Context) error {
}
func RefreshPriceOnChain(ctx context.Context) error {
- txn, err := sendSmartContractBlobberAdd(ctx)
+ txn, err := sendSmartContractBlobberAdd()
if err != nil {
- return err
- }
-
- if t, err := TransactionVerify(txn); err != nil {
logging.Logger.Error("Failed to verify price refresh transaction", zap.Any("err", err), zap.String("txn.Hash", txn.Hash))
- } else {
- logging.Logger.Info("Verified price refresh transaction", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput))
+ return err
}
- return err
+ logging.Logger.Info("Verified price refresh transaction", zap.String("txn_hash", txn.Hash), zap.Any("txn_output", txn.TransactionOutput))
+ return nil
}
// sendSmartContractBlobberAdd Add or update blobber on blockchain
-func sendSmartContractBlobberAdd(ctx context.Context) (*transaction.Transaction, error) {
+func sendSmartContractBlobberAdd() (*coreTxn.Transaction, error) {
// initialize storage node (ie blobber)
- txn, err := transaction.NewTransactionEntity()
- if err != nil {
- return nil, err
- }
-
sn, err := getStorageNode()
if err != nil {
return nil, err
}
- err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS,
- transaction.ADD_BLOBBER_SC_NAME, sn, 0)
+ _, _, _, txn, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{
+ Name: transaction.ADD_BLOBBER_SC_NAME,
+ InputArgs: sn,
+ }, true)
if err != nil {
logging.Logger.Error("Failed to set blobber on the blockchain",
- zap.String("err:", err.Error()))
+ zap.String("err:", err.Error()), zap.Any("Txn", txn), zap.Any("ClientFee", client.TxnFee()))
return nil, err
}
@@ -161,39 +149,20 @@ var ErrValidatorHasRemoved = errors.New("validator has been removed")
// ErrValidatorNotFound it is not registered on chain
var ErrValidatorNotFound = errors.New("validator is not found")
-func TransactionVerify(txn *transaction.Transaction) (t *transaction.Transaction, err error) {
- msg := fmt.Sprintf("Verifying transaction: max_retries: %d", util.MAX_RETRIES)
- logging.Logger.Info(msg)
- for i := 0; i < util.MAX_RETRIES; i++ {
- time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second)
- if t, err = transaction.VerifyTransactionWithNonce(txn.Hash, txn.GetTransaction().GetTransactionNonce()); err == nil {
- return t, nil
- }
- }
-
- return nil, errors.New("[txn]max retries exceeded with " + txn.Hash)
-}
-
// SendHealthCheck send heartbeat to blockchain
func SendHealthCheck(provider common.ProviderType) (string, error) {
- var txn *transaction.Transaction
+ var hash string
var err error
switch provider {
case common.ProviderTypeBlobber:
- txn, err = BlobberHealthCheck()
+ hash, err = BlobberHealthCheck()
case common.ProviderTypeValidator:
- txn, err = ValidatorHealthCheck()
+ hash, err = ValidatorHealthCheck()
default:
return "", errors.New("unknown provider type")
}
- if err != nil {
- return "", err
- }
-
- _, err = TransactionVerify(txn)
-
- return txn.Hash, err
+ return hash, err
}
diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go
index ebe240393..36374892e 100644
--- a/code/go/0chain.net/blobbercore/handler/storage_handler.go
+++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go
@@ -12,7 +12,7 @@ import (
"gorm.io/gorm"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"go.uber.org/zap"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
@@ -21,6 +21,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
)
@@ -142,8 +143,15 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in
return nil, common.NewError("invalid_signature", "Invalid signature")
}
}
-
+ if fileref.Type == reference.DIRECTORY {
+ fileref.IsEmpty, err = reference.IsDirectoryEmpty(ctx, fileref.ID)
+ if err != nil {
+ return nil, common.NewError("bad_db_operation", "Error checking if directory is empty. "+err.Error())
+ }
+ }
+ fileref.AllocationVersion = alloc.AllocationVersion
result := fileref.GetListingData(ctx)
+ Logger.Info("GetFileMeta", zap.Any("allocationResultVersion", result["allocation_version"]), zap.Int64("allocationVersion", alloc.AllocationVersion), zap.Any("path", result["path"]))
if !isOwner && !isRepairer {
var authTokenString = r.FormValue("auth_token")
@@ -255,11 +263,6 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i
if err != nil {
return nil, common.NewError("bad_db_operation", "Error retrieving file stats. "+err.Error())
}
- wm, _ := writemarker.GetWriteMarkerEntity(ctx, fileref.AllocationRoot)
- if wm != nil && fileStats != nil {
- fileStats.WriteMarkerRedeemTxn = wm.CloseTxnID
- fileStats.OnChain = wm.OnChain()
- }
statsMap := make(map[string]interface{})
statsBytes, err := json.Marshal(fileStats)
if err != nil {
@@ -391,7 +394,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (*
if !ok {
var listResult blobberhttp.ListResult
- listResult.AllocationRoot = allocationObj.AllocationRoot
+ listResult.AllocationVersion = allocationObj.AllocationVersion
if fileref == nil {
fileref = &reference.Ref{Type: reference.DIRECTORY, Path: path, AllocationID: allocationID}
}
@@ -402,6 +405,9 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (*
}
fileref = parent
}
+ if path == "/" {
+ fileref.Size = allocationObj.BlobberSizeUsed
+ }
listResult.Meta = fileref.GetListingData(ctx)
if clientID != allocationObj.OwnerID {
delete(listResult.Meta, "path")
@@ -461,16 +467,22 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (*
dirref = parent
} else {
+ if fileref == nil {
+ fileref = &reference.Ref{Type: reference.DIRECTORY, Path: path, AllocationID: allocationID}
+ }
r, err := reference.GetRefWithChildren(ctx, fileref, allocationID, filePath, offset, pageLimit)
if err != nil {
return nil, common.NewError("invalid_parameters", "Invalid path. "+err.Error())
}
dirref = r
+ if path == "/" {
+ dirref.Size = allocationObj.BlobberSizeUsed
+ }
}
var result blobberhttp.ListResult
- result.AllocationRoot = allocationObj.AllocationRoot
+ result.AllocationVersion = allocationObj.AllocationVersion
result.Meta = dirref.GetListingData(ctx)
if clientID != allocationObj.OwnerID {
delete(result.Meta, "path")
@@ -490,7 +502,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (*
return &result, nil
}
-func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Request) (*blobberhttp.LatestWriteMarkerResult, error) {
+func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Request) (*blobberhttp.LatestVersionMarkerResult, error) {
clientID := ctx.Value(constants.ContextKeyClient).(string)
if clientID == "" {
return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation")
@@ -508,44 +520,26 @@ func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Req
clientSignV2 := ctx.Value(constants.ContextKeyClientSignatureHeaderV2Key).(string)
valid, err := verifySignatureFromRequest(allocationTx, clientSign, clientSignV2, publicKey)
if !valid || err != nil {
- return nil, common.NewError("invalid_signature", "could not verify the allocation owner")
- }
-
- var latestWM *writemarker.WriteMarkerEntity
- var prevWM *writemarker.WriteMarkerEntity
- if allocationObj.AllocationRoot == "" {
- latestWM = nil
- } else {
- latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot)
if err != nil {
- Logger.Error("[latest_write_marker]", zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("allocation_id", allocationObj.ID))
- return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation. "+err.Error())
- }
- if latestWM == nil {
- Logger.Info("[latest_write_marker]", zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("allocation_id", allocationObj.ID))
- return nil, common.NewError("latest_write_marker_read_error", "Latest write marker not found for allocation.")
- }
- if latestWM.WM.PreviousAllocationRoot != "" {
- prevWM, err = writemarker.GetWriteMarkerEntity(ctx, latestWM.WM.PreviousAllocationRoot)
- if err != nil {
- return nil, common.NewError("latest_write_marker_read_error", "Error reading the previous write marker for allocation."+err.Error())
- }
+ return nil, common.NewError("invalid_signature", "could not verify the allocation owner"+err.Error())
}
+ return nil, common.NewError("invalid_signature", "could not verify the allocation owner"+": "+publicKey+": "+allocationId+": "+allocationTx+": "+clientSignV2)
}
- var result blobberhttp.LatestWriteMarkerResult
- result.Version = writemarker.MARKER_VERSION
- if latestWM != nil {
- if latestWM.Status == writemarker.Committed {
- latestWM.WM.ChainLength = 0 // start a new chain
+ var vm *writemarker.VersionMarker
+ if allocationObj.AllocationVersion != 0 {
+ vm, err = writemarker.GetVersionMarker(ctx, allocationId, allocationObj.AllocationVersion)
+ if err != nil {
+ return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error())
}
- result.LatestWM = &latestWM.WM
- }
- if prevWM != nil {
- result.PrevWM = &prevWM.WM
+ } else {
+ vm = &writemarker.VersionMarker{}
}
- return &result, nil
+ result := &blobberhttp.LatestVersionMarkerResult{
+ VersionMarker: vm,
+ }
+ return result, nil
}
func (fsh *StorageHandler) GetReferencePath(ctx context.Context, r *http.Request) (*blobberhttp.ReferencePathResult, error) {
@@ -626,7 +620,8 @@ func (fsh *StorageHandler) getReferencePath(ctx context.Context, r *http.Request
if allocationObj.AllocationRoot == "" {
latestWM = nil
} else {
- latestWM, err = writemarker.GetWriteMarkerEntity(ctx, rootRef.Hash)
+ //TODO: remove latestWM
+ latestWM, err = writemarker.GetWriteMarkerEntity(ctx, rootRef.FileMetaHash)
if err != nil {
errCh <- common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error())
return
@@ -834,15 +829,16 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb
return nil, common.NewError("invalid_parameters", "empty path and authtoken")
}
- var pathRef *reference.Ref
+ var pathRef *reference.PaginatedRef
switch {
case path != "":
pathHash = reference.GetReferenceLookup(allocationID, path)
fallthrough
case pathHash != "":
- pathRef, err = reference.GetReferenceByLookupHash(ctx, allocationID, pathHash)
+ pathRef, err = reference.GetPaginatedRefByLookupHash(ctx, pathHash)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
+ logging.Logger.Error("GetRefs: GetPaginatedRefByLookupHash", zap.Error(err), zap.String("path", path), zap.String("pathHash", pathHash))
return nil, common.NewError("invalid_path", "")
}
return nil, err
@@ -874,7 +870,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb
}
if pathRef == nil {
- pathRef, err = reference.GetReferenceByLookupHash(ctx, allocationID, authToken.FilePathHash)
+ pathRef, err = reference.GetPaginatedRefByLookupHash(ctx, authToken.FilePathHash)
if err != nil {
return nil, fsh.convertGormError(err)
}
@@ -910,13 +906,16 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb
pageLimit = o
}
}
-
+ var offsetTime int
offsetPath := r.FormValue("offsetPath")
offsetDate := r.FormValue("offsetDate")
updatedDate := r.FormValue("updatedDate")
err = checkValidDate(offsetDate, OffsetDateLayout)
if err != nil {
- return nil, err
+ offsetTime, err = strconv.Atoi(offsetDate)
+ if err != nil {
+ return nil, err
+ }
}
err = checkValidDate(updatedDate, OffsetDateLayout)
if err != nil {
@@ -945,8 +944,11 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb
switch {
case refType == "regular":
refs, totalPages, newOffsetPath, err = reference.GetRefs(
- ctx, allocationID, path, offsetPath, fileType, level, pageLimit,
+ ctx, allocationID, path, offsetPath, fileType, level, pageLimit, offsetTime, pathRef,
)
+ if refs != nil {
+ logging.Logger.Info("GetRefs: regular", zap.Int("refs", len(*refs)), zap.Int("totalPages", totalPages), zap.String("newOffsetPath", newOffsetPath), zap.Error(err))
+ }
case refType == "updated":
refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetUpdatedRefs(
@@ -961,24 +963,11 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb
if err != nil {
return nil, err
}
- var latestWM *writemarker.WriteMarkerEntity
- if allocationObj.AllocationRoot == "" {
- latestWM = nil
- } else {
- latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot)
- if err != nil {
- return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error())
- }
- }
-
var refResult blobberhttp.RefResult
refResult.Refs = refs
refResult.TotalPages = totalPages
refResult.OffsetPath = newOffsetPath
refResult.OffsetDate = newOffsetDate
- if latestWM != nil {
- refResult.LatestWM = &latestWM.WM
- }
// Refs will be returned as it is and object tree will be build in client side
return &refResult, nil
}
@@ -1000,7 +989,7 @@ func verifySignatureFromRequest(alloc, signV1, signV2, pbK string) (bool, error)
hash = encryption.Hash(hashData)
}
if len(sign) < 64 {
- return false, nil
+ return false, common.NewError("500", "len shorter than 64")
}
return encryption.Verify(pbK, sign, hash)
}
diff --git a/code/go/0chain.net/blobbercore/handler/tests_common_test.go b/code/go/0chain.net/blobbercore/handler/tests_common_test.go
index 78fa2dcb7..461a042e9 100644
--- a/code/go/0chain.net/blobbercore/handler/tests_common_test.go
+++ b/code/go/0chain.net/blobbercore/handler/tests_common_test.go
@@ -15,9 +15,11 @@ import (
"strings"
"testing"
+ coreNetwork "github.com/0chain/gosdk_common/core/conf"
+
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
+ "github.com/0chain/gosdk_common/zcncore"
)
func setup(t *testing.T) {
@@ -30,7 +32,7 @@ func setup(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if err := zcncore.SetWalletInfo(string(wBlob), true); err != nil {
+ if err := zcncore.SetWalletInfo(string(wBlob), "bls0chain", true); err != nil {
t.Fatal(err)
}
@@ -41,10 +43,10 @@ func setup(t *testing.T) {
},
),
)
- server := httptest.NewServer(
+ _ = httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
- n := zcncore.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}}
+ n := coreNetwork.Network{Miners: []string{"miner 1"}, Sharders: []string{sharderServ.URL}}
blob, err := json.Marshal(n)
if err != nil {
t.Fatal(err)
@@ -57,9 +59,9 @@ func setup(t *testing.T) {
),
)
- if err := zcncore.InitZCNSDK(server.URL, "ed25519"); err != nil {
- t.Fatal(err)
- }
+ // if err := zcncore.InitZCNSDK(server.URL, "ed25519"); err != nil {
+ // t.Fatal(err)
+ // }
}
type MockFileStore struct {
@@ -76,11 +78,9 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string,
b := bytes.NewBuffer(make([]byte, 0))
n, _ := io.Copy(b, infile)
return &filestore.FileOutputData{
- Name: fileData.Name,
- Path: fileData.Path,
- FixedMerkleRoot: "",
- ValidationRoot: fileData.ValidationRoot,
- Size: n,
+ Name: fileData.Name,
+ Path: fileData.Path,
+ Size: n,
}, nil
}
@@ -119,6 +119,10 @@ func (mfs *MockFileStore) GetTempFilePath(allocID, connID, fileName, filePathHas
return ""
}
+func (mfs *MockFileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error {
+ return nil
+}
+
func (mfs *MockFileStore) GetFileBlock(in *filestore.ReadBlockInput) (*filestore.FileDownloadResponse, error) {
return &filestore.FileDownloadResponse{
Data: mockFileBlock,
diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go
index b5881b4bb..b5acbe174 100644
--- a/code/go/0chain.net/blobbercore/handler/worker.go
+++ b/code/go/0chain.net/blobbercore/handler/worker.go
@@ -39,6 +39,10 @@ func cleanupAllocationFiles(ctx context.Context, allocationObj allocation.Alloca
db := datastore.GetStore().GetTransaction(ctx)
_ = filestore.GetFileStore().IterateObjects(allocationObj.ID, func(hash string, contentSize int64) {
+ // thumbnail suffix makes hash greater than 65
+ if len(hash) > 65 {
+ return
+ }
var refs []reference.Ref
version := 0
if len(hash) > 64 {
@@ -46,8 +50,7 @@ func cleanupAllocationFiles(ctx context.Context, allocationObj allocation.Alloca
hash = hash[:64]
}
err := db.Table((reference.Ref{}).TableName()).
- Where(reference.Ref{ValidationRoot: hash, Type: reference.FILE}).
- Or(reference.Ref{ThumbnailHash: hash, Type: reference.FILE}).
+ Where(reference.Ref{LookupHash: hash, Type: reference.FILE}).
Find(&refs).Error
if err != nil {
@@ -84,6 +87,10 @@ func cleanupTempFiles(ctx context.Context) {
for i := 0; i < len(openConnectionsToDelete); i++ {
connection := &openConnectionsToDelete[i]
logging.Logger.Info("Deleting temp files for the connection", zap.Any("connection", connection.ID))
+ processor := allocation.GetConnectionProcessor(connection.ID)
+ if processor != nil {
+ continue
+ }
connection.ComputeProperties()
nctx := datastore.GetStore().CreateTransaction(ctx)
diff --git a/code/go/0chain.net/blobbercore/handler/zcncore.go b/code/go/0chain.net/blobbercore/handler/zcncore.go
deleted file mode 100644
index 36df8290a..000000000
--- a/code/go/0chain.net/blobbercore/handler/zcncore.go
+++ /dev/null
@@ -1,128 +0,0 @@
-package handler
-
-import (
- "sync"
-
- "github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/gosdk/zcncore"
-)
-
-type ZCNStatus struct {
- wg *sync.WaitGroup
- success bool
- balance int64
- info string
-}
-
-func (zcn *ZCNStatus) OnBalanceAvailable(status int, value int64, info string) {
- defer zcn.wg.Done()
- if status == zcncore.StatusSuccess {
- zcn.success = true
- } else {
- zcn.success = false
- }
- zcn.balance = value
-}
-
-func (zcn *ZCNStatus) OnInfoAvailable(op, status int, info, err string) {
- defer zcn.wg.Done()
- if status == zcncore.StatusSuccess {
- zcn.success = true
- } else {
- zcn.success = false
- }
- zcn.info = info
-}
-
-func (zcn *ZCNStatus) OnTransactionComplete(t *zcncore.Transaction, status int) {
- defer zcn.wg.Done()
- if status == zcncore.StatusSuccess {
- zcn.success = true
- } else {
- zcn.success = false
- }
-}
-
-func (zcn *ZCNStatus) OnVerifyComplete(t *zcncore.Transaction, status int) {
- defer zcn.wg.Done()
- if status == zcncore.StatusSuccess {
- zcn.success = true
- } else {
- zcn.success = false
- }
-}
-
-func (zcn *ZCNStatus) OnAuthComplete(t *zcncore.Transaction, status int) {}
-
-func CheckBalance() (float64, error) {
- wg := &sync.WaitGroup{}
- statusBar := &ZCNStatus{wg: wg}
- wg.Add(1)
- err := zcncore.GetBalance(statusBar)
- if err != nil {
- return 0, common.NewError("check_balance_failed", "Call to GetBalance failed with err: "+err.Error())
- }
- wg.Wait()
- if !statusBar.success {
- return 0, nil
- }
- return zcncore.ConvertToToken(statusBar.balance), nil
-}
-
-func CallFaucet() error {
- wg := &sync.WaitGroup{}
- statusBar := &ZCNStatus{wg: wg}
- txn, err := zcncore.NewTransaction(statusBar, 0, 0)
- if err != nil {
- return common.NewError("call_faucet_failed", "Failed to create new transaction with err: "+err.Error())
- }
- wg.Add(1)
- _, err = txn.ExecuteSmartContract(zcncore.FaucetSmartContractAddress, "pour", "Blobber Registration", zcncore.ConvertToValue(0))
- if err != nil {
- return common.NewError("call_faucet_failed", "Failed to execute smart contract with err: "+err.Error())
- }
- wg.Wait()
- if !statusBar.success {
- return common.NewError("call_faucet_failed", "Failed to execute smart contract with statusBar success failed")
- }
- statusBar.success = false
- wg.Add(1)
- err = txn.Verify()
- if err != nil {
- return common.NewError("call_faucet_failed", "Failed to verify smart contract with err: "+err.Error())
- }
- wg.Wait()
- if !statusBar.success {
- return common.NewError("call_faucet_failed", "Failed to verify smart contract with statusBar success failed")
- }
- return nil
-}
-
-func Transfer(token float64, clientID string) error {
- wg := &sync.WaitGroup{}
- statusBar := &ZCNStatus{wg: wg}
- txn, err := zcncore.NewTransaction(statusBar, 0, 0)
- if err != nil {
- return common.NewError("call_transfer_failed", "Failed to create new transaction with err: "+err.Error())
- }
- wg.Add(1)
- err = txn.Send(clientID, zcncore.ConvertToValue(token), "Blobber delegate transfer")
- if err != nil {
- return common.NewError("call_transfer_failed", "Failed to send tokens with err: "+err.Error())
- }
- wg.Wait()
- if !statusBar.success {
- return common.NewError("call_transfer_failed", "Failed to send tokens with statusBar success failed")
- }
- statusBar.success = false
- wg.Add(1)
- err = txn.Verify()
- if err != nil {
- return common.NewError("call_transfer_failed", "Failed to verify send transaction with err: "+err.Error())
- }
- wg.Wait()
- if !statusBar.success {
- return common.NewError("call_transfer_failed", "Failed to verify send transaction with statusBar success failed")
- }
- return nil
-}
diff --git a/code/go/0chain.net/blobbercore/mock/ctx.go b/code/go/0chain.net/blobbercore/mock/ctx.go
index b41344c2e..ac7195abf 100644
--- a/code/go/0chain.net/blobbercore/mock/ctx.go
+++ b/code/go/0chain.net/blobbercore/mock/ctx.go
@@ -5,7 +5,7 @@ import (
"net/http"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
)
func SetupHandlerContext(ctx context.Context, r *http.Request, allocation string) context.Context {
diff --git a/code/go/0chain.net/blobbercore/mock/init.go b/code/go/0chain.net/blobbercore/mock/init.go
index e1c2fa978..624cb850c 100644
--- a/code/go/0chain.net/blobbercore/mock/init.go
+++ b/code/go/0chain.net/blobbercore/mock/init.go
@@ -6,9 +6,9 @@ import (
"net/http"
"net/http/httptest"
- "github.com/0chain/gosdk/core/zcncrypto"
- "github.com/0chain/gosdk/sdks"
- "github.com/0chain/gosdk/sdks/blobber"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
+ "github.com/0chain/gosdk_common/sdks"
+ "github.com/0chain/gosdk_common/sdks/blobber"
)
const (
diff --git a/code/go/0chain.net/blobbercore/readmarker/protocol.go b/code/go/0chain.net/blobbercore/readmarker/protocol.go
index e568f0070..51423fc89 100644
--- a/code/go/0chain.net/blobbercore/readmarker/protocol.go
+++ b/code/go/0chain.net/blobbercore/readmarker/protocol.go
@@ -6,12 +6,9 @@ import (
"time"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- zLogger "github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
"gorm.io/gorm"
-
- "go.uber.org/zap"
)
type ReadRedeem struct {
@@ -41,35 +38,7 @@ func (rme *ReadMarkerEntity) PendNumBlocks() (pendNumBlocks int64, err error) {
// RedeemReadMarker redeems the read marker.
func (rme *ReadMarkerEntity) RedeemReadMarker(ctx context.Context) (err error) {
- tx, err := transaction.NewTransactionEntity()
- if err != nil {
- return common.NewErrorf("redeem_read_marker", "creating transaction: %v", err)
- }
-
- sn := &ReadRedeem{
- ReadMarker: rme.LatestRM,
- }
-
- err = tx.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.READ_REDEEM, sn, 0)
- if err != nil {
- zLogger.Logger.Info("Failed submitting read redeem", zap.Error(err))
- return common.NewErrorf("redeem_read_marker", "sending transaction: %v", err)
- }
-
- time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second)
-
- var logHash = tx.Hash // keep transaction hash for error logs
- tx, err = transaction.VerifyTransactionWithNonce(tx.Hash, tx.GetTransaction().GetTransactionNonce())
- if err != nil {
- zLogger.Logger.Error("Error verifying the read redeem transaction", zap.Error(err), zap.String("txn", logHash))
- return common.NewErrorf("redeem_read_marker", "verifying transaction: %v", err)
- }
-
- err = rme.UpdateStatus(ctx, tx.TransactionOutput, tx.Hash)
- if err != nil {
- return common.NewErrorf("redeem_read_marker", "updating read marker status: %v", err)
- }
-
+ // Depreciated
return
}
diff --git a/code/go/0chain.net/blobbercore/readmarker/readmarker.go b/code/go/0chain.net/blobbercore/readmarker/readmarker.go
index b2df8007a..2fce04489 100644
--- a/code/go/0chain.net/blobbercore/readmarker/readmarker.go
+++ b/code/go/0chain.net/blobbercore/readmarker/readmarker.go
@@ -12,7 +12,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
zLogger "github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"go.uber.org/zap"
"gorm.io/gorm"
)
diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go
index 37c3cc808..5531bcf60 100644
--- a/code/go/0chain.net/blobbercore/reference/dbCollector.go
+++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go
@@ -2,48 +2,131 @@ package reference
import (
"context"
+ "sync"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+ "go.uber.org/zap"
)
type QueryCollector interface {
CreateRefRecord(ref *Ref)
DeleteRefRecord(ref *Ref)
- Finalize(ctx context.Context) error
+ Finalize(ctx context.Context, allocationID string, allocationVersion int64) error
+ AddToCache(ref *Ref)
+ GetFromCache(lookupHash string) *Ref
+ DeleteLookupRefRecord(ref *Ref)
+ LockTransaction()
+ UnlockTransaction()
}
type dbCollector struct {
createdRefs []*Ref
deletedRefs []*Ref
+ refCache RefCache
+ refMap map[string]*Ref
+ txnLock sync.Mutex
+ lock sync.Mutex
}
+type RefCache struct {
+ AllocationVersion int64
+ CreatedRefs []*Ref
+ DeletedRefs []*Ref
+}
+
+var (
+ cacheMap = make(map[string]*RefCache)
+ cacheMapLock sync.RWMutex
+)
+
func NewCollector(changes int) QueryCollector {
return &dbCollector{
- createdRefs: make([]*Ref, 0, changes*4),
- deletedRefs: make([]*Ref, 0, changes*4),
+ createdRefs: make([]*Ref, 0, changes*2),
+ deletedRefs: make([]*Ref, 0, changes*2),
+ refCache: RefCache{
+ CreatedRefs: make([]*Ref, 0, changes),
+ DeletedRefs: make([]*Ref, 0, changes),
+ },
+ refMap: make(map[string]*Ref),
}
}
func (dc *dbCollector) CreateRefRecord(ref *Ref) {
+ dc.lock.Lock()
dc.createdRefs = append(dc.createdRefs, ref)
+ if ref.Type == FILE {
+ dc.refCache.CreatedRefs = append(dc.refCache.CreatedRefs, ref)
+ }
+ dc.lock.Unlock()
}
func (dc *dbCollector) DeleteRefRecord(ref *Ref) {
+ dc.lock.Lock()
dc.deletedRefs = append(dc.deletedRefs, ref)
+ if ref.Type == FILE {
+ dc.refCache.DeletedRefs = append(dc.refCache.DeletedRefs, ref)
+ }
+ dc.lock.Unlock()
}
-func (dc *dbCollector) Finalize(ctx context.Context) error {
+func (dc *dbCollector) DeleteLookupRefRecord(ref *Ref) {
+ dc.refCache.DeletedRefs = append(dc.refCache.DeletedRefs, ref)
+}
+
+func (dc *dbCollector) Finalize(ctx context.Context, allocationID string, allocationVersion int64) error {
db := datastore.GetStore().GetTransaction(ctx)
if len(dc.deletedRefs) > 0 {
- err := db.Delete(dc.deletedRefs).Error
+ err := db.Delete(&(dc.deletedRefs)).Error
if err != nil {
return err
}
}
- err := db.Create(dc.createdRefs).Error
- if err != nil {
- return err
+ if len(dc.createdRefs) > 0 {
+ err := db.Create(&(dc.createdRefs)).Error
+ if err != nil {
+ for ind, ref := range dc.createdRefs {
+ logging.Logger.Error("create_ref_error", zap.String("lookup_hash", ref.LookupHash), zap.String("path", ref.Path), zap.Int("index", ind), zap.Int64("allocation_version", allocationVersion))
+ }
+ return err
+ }
}
-
+ dc.refCache.AllocationVersion = allocationVersion
+ cacheMapLock.Lock()
+ cacheMap[allocationID] = &(dc.refCache)
+ logging.Logger.Info("Finalize", zap.Int("created", len(dc.createdRefs)), zap.Int("deleted", len(dc.deletedRefs)), zap.Int64("allocation_version", cacheMap[allocationID].AllocationVersion), zap.String("allocation_id", allocationID))
+ cacheMapLock.Unlock()
return nil
}
+
+func (dc *dbCollector) AddToCache(ref *Ref) {
+ dc.lock.Lock()
+ dc.refMap[ref.LookupHash] = ref
+ dc.lock.Unlock()
+}
+
+func (dc *dbCollector) GetFromCache(lookupHash string) *Ref {
+ dc.lock.Lock()
+ defer dc.lock.Unlock()
+ return dc.refMap[lookupHash]
+}
+
+func GetRefCache(allocationID string) *RefCache {
+ cacheMapLock.RLock()
+ defer cacheMapLock.RUnlock()
+ return cacheMap[allocationID]
+}
+
+func DeleteRefCache(allocationID string) {
+ cacheMapLock.Lock()
+ cacheMap[allocationID] = nil
+ cacheMapLock.Unlock()
+}
+
+func (dc *dbCollector) LockTransaction() {
+ dc.txnLock.Lock()
+}
+
+func (dc *dbCollector) UnlockTransaction() {
+ dc.txnLock.Unlock()
+}
diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go
index 31d2f0f87..f0a721ed0 100644
--- a/code/go/0chain.net/blobbercore/reference/object.go
+++ b/code/go/0chain.net/blobbercore/reference/object.go
@@ -2,74 +2,45 @@ package reference
import (
"context"
- "path/filepath"
+ "database/sql"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+ "go.uber.org/zap"
)
-func DeleteObject(ctx context.Context, rootRef *Ref, allocationID, objPath string, ts common.Timestamp) error {
- likePath := objPath + "/%"
- if objPath == "/" {
- likePath = "/%"
- }
-
+func DeleteObject(ctx context.Context, allocationID, lookupHash, _type string, ts common.Timestamp, allocationVersion int64, collector QueryCollector) error {
db := datastore.GetStore().GetTransaction(ctx)
-
- err := db.Exec("UPDATE reference_objects SET is_precommit=? WHERE allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", true, allocationID, "/", objPath, likePath).Error
- if err != nil {
- return err
- }
-
- err = db.Delete(&Ref{}, "allocation_id=? AND path != ? AND (path=? OR path LIKE ?)",
- allocationID, "/", objPath, likePath).Error
-
- if err != nil {
- return err
- }
- if objPath == "/" {
- rootRef.Children = nil
- rootRef.HashToBeComputed = true
- rootRef.childrenLoaded = true
- rootRef.UpdatedAt = ts
- return nil
+ if _type == DIRECTORY {
+ ref, err := GetLimitedRefFieldsByLookupHashWith(ctx, allocationID, lookupHash, []string{"id", "type"})
+ if err != nil {
+ logging.Logger.Error("delete_object_error", zap.Error(err))
+ return err
+ }
+ isEmpty, err := IsDirectoryEmpty(ctx, ref.ID)
+ if err != nil {
+ logging.Logger.Error("delete_object_error", zap.Error(err))
+ return err
+ }
+ if !isEmpty {
+ return common.NewError("invalid_operation", "Directory is not empty")
+ }
+ _type = ref.Type
}
- parentPath, deleteFileName := filepath.Split(objPath)
- rootRef.UpdatedAt = ts
- fields, err := common.GetPathFields(parentPath)
+ err := db.Exec("UPDATE reference_objects SET deleted_at=? WHERE lookup_hash=?", sql.NullTime{
+ Time: common.ToTime(ts),
+ Valid: true,
+ }, lookupHash).Error
if err != nil {
+ logging.Logger.Error("delete_object_error", zap.Error(err))
return err
}
-
- dirRef := rootRef
-
- for _, name := range fields {
- var found bool
- for _, ref := range dirRef.Children {
- if ref.Name == name {
- ref.HashToBeComputed = true
- ref.childrenLoaded = true
- ref.UpdatedAt = ts
- found = true
- dirRef = ref
- break
- }
- }
-
- if !found {
- return common.NewError("invalid_reference_path", "Reference path has invalid references")
+ if _type == FILE {
+ deletedRef := &Ref{
+ LookupHash: lookupHash,
}
+ collector.DeleteLookupRefRecord(deletedRef)
}
-
- for i, child := range dirRef.Children {
- basePath := filepath.Base(child.Path)
- if basePath == deleteFileName || child.Path == objPath {
- dirRef.RemoveChild(i)
- break
- }
- }
-
- rootRef.HashToBeComputed = true
- rootRef.childrenLoaded = true
- return nil
+ return err
}
diff --git a/code/go/0chain.net/blobbercore/reference/objectpath.go b/code/go/0chain.net/blobbercore/reference/objectpath.go
index d9d19fcb3..34d817b35 100644
--- a/code/go/0chain.net/blobbercore/reference/objectpath.go
+++ b/code/go/0chain.net/blobbercore/reference/objectpath.go
@@ -31,7 +31,7 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O
if rootRef.NumBlocks == 0 {
var retObj ObjectPath
- retObj.RootHash = rootRef.Hash
+ // retObj.RootHash = rootRef.Hash
retObj.FileBlockNum = 0
result := rootRef.GetListingData(ctx)
list := make([]map[string]interface{}, len(rootRef.Children))
@@ -68,7 +68,7 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O
break
}
curRef, err = GetRefWithSortedChildren(ctx, allocationID, child.Path)
- if err != nil || curRef.Hash == "" {
+ if err != nil {
return nil, common.NewError("failed_object_path", "Failed to get the object path")
}
curResult = list[idx]
@@ -80,7 +80,6 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O
}
var retObj ObjectPath
- retObj.RootHash = rootRef.Hash
retObj.Meta = curRef.GetListingData(ctx)
retObj.Path = result
retObj.FileBlockNum = remainingBlocks
diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go
index 6c50cf1b3..53df6c7da 100644
--- a/code/go/0chain.net/blobbercore/reference/ref.go
+++ b/code/go/0chain.net/blobbercore/reference/ref.go
@@ -2,7 +2,7 @@ package reference
import (
"context"
- "errors"
+ "database/sql"
"fmt"
"math"
"path/filepath"
@@ -39,41 +39,33 @@ func init() {
field := refType.Field(i)
dirListTag := field.Tag.Get(DIR_LIST_TAG)
- if dirListTag != "" {
+ if dirListTag != "" && dirListTag != "is_empty" && dirListTag != "allocation_version" {
dirListFields = append(dirListFields, dirListTag)
}
}
- dirListFields = append(dirListFields, "parent_path")
+ dirListFields = append(dirListFields, "parent_path", "id")
}
type Ref struct {
ID int64 `gorm:"column:id;primaryKey"`
- FileID string `gorm:"column:file_id" dirlist:"file_id" filelist:"file_id"`
+ ParentID *int64 `gorm:"column:parent_id"`
Type string `gorm:"column:type;size:1" dirlist:"type" filelist:"type"`
AllocationID string `gorm:"column:allocation_id;size:64;not null;index:idx_path_alloc,priority:1;index:idx_parent_path_alloc,priority:1;index:idx_validation_alloc,priority:1" dirlist:"allocation_id" filelist:"allocation_id"`
LookupHash string `gorm:"column:lookup_hash;size:64;not null;index:idx_lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash"`
Name string `gorm:"column:name;size:100;not null;index:idx_name_gin" dirlist:"name" filelist:"name"` // uses GIN tsvector index for full-text search
Path string `gorm:"column:path;size:1000;not null;index:idx_path_alloc,priority:2;index:path_idx;index:idx_path_gin_trgm" dirlist:"path" filelist:"path"`
FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"`
- Hash string `gorm:"column:hash;size:64;not null" dirlist:"hash" filelist:"hash"`
NumBlocks int64 `gorm:"column:num_of_blocks;not null;default:0" dirlist:"num_of_blocks" filelist:"num_of_blocks"`
- PathHash string `gorm:"column:path_hash;size:64;not null" dirlist:"path_hash" filelist:"path_hash"`
ParentPath string `gorm:"column:parent_path;size:999;index:idx_parent_path_alloc,priority:2"`
PathLevel int `gorm:"column:level;not null;default:0"`
CustomMeta string `gorm:"column:custom_meta;not null" filelist:"custom_meta" dirlist:"custom_meta"`
- ValidationRoot string `gorm:"column:validation_root;size:64;not null;index:idx_validation_alloc,priority:2" filelist:"validation_root"`
- PrevValidationRoot string `gorm:"column:prev_validation_root" filelist:"prev_validation_root" json:"prev_validation_root"`
- ValidationRootSignature string `gorm:"column:validation_root_signature;size:64" filelist:"validation_root_signature" json:"validation_root_signature,omitempty"`
Size int64 `gorm:"column:size;not null;default:0" dirlist:"size" filelist:"size"`
- FixedMerkleRoot string `gorm:"column:fixed_merkle_root;size:64;not null" filelist:"fixed_merkle_root"`
ActualFileSize int64 `gorm:"column:actual_file_size;not null;default:0" dirlist:"actual_file_size" filelist:"actual_file_size"`
ActualFileHashSignature string `gorm:"column:actual_file_hash_signature;size:64" filelist:"actual_file_hash_signature" json:"actual_file_hash_signature,omitempty"`
ActualFileHash string `gorm:"column:actual_file_hash;size:64;not null" filelist:"actual_file_hash"`
MimeType string `gorm:"column:mimetype;size:255;not null" filelist:"mimetype"`
- AllocationRoot string `gorm:"column:allocation_root;size:64;not null"`
ThumbnailSize int64 `gorm:"column:thumbnail_size;not null;default:0" filelist:"thumbnail_size"`
ThumbnailHash string `gorm:"column:thumbnail_hash;size:64;not null" filelist:"thumbnail_hash"`
- PrevThumbnailHash string `gorm:"column:prev_thumbnail_hash" filelist:"prev_thumbnail_hash"`
ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size;not null;default:0" filelist:"actual_thumbnail_size"`
ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash;size:64;not null" filelist:"actual_thumbnail_hash"`
EncryptedKey string `gorm:"column:encrypted_key;size:64" filelist:"encrypted_key"`
@@ -84,11 +76,14 @@ type Ref struct {
UpdatedAt common.Timestamp `gorm:"column:updated_at;index:idx_updated_at,sort:desc;" dirlist:"updated_at" filelist:"updated_at"`
DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion
- IsPrecommit bool `gorm:"column:is_precommit;not null;default:false" filelist:"is_precommit" dirlist:"is_precommit"`
ChunkSize int64 `gorm:"column:chunk_size;not null;default:65536" dirlist:"chunk_size" filelist:"chunk_size"`
NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"`
NumBlockDownloads int64 `gorm:"column:num_of_block_downloads" json:"num_of_block_downloads"`
FilestoreVersion int `gorm:"column:filestore_version" json:"-"`
+ DataHash string `gorm:"column:data_hash" filelist:"data_hash"`
+ DataHashSignature string `gorm:"column:data_hash_signature" filelist:"data_hash_signature"`
+ AllocationVersion int64 `gorm:"allocation_version" dirlist:"allocation_version" filelist:"allocation_version"`
+ IsEmpty bool `gorm:"-" dirlist:"is_empty"`
HashToBeComputed bool `gorm:"-"`
prevID int64 `gorm:"-"`
}
@@ -117,34 +112,27 @@ func (Ref) TableName() string {
type PaginatedRef struct { //Gorm smart select fields.
ID int64 `gorm:"column:id" json:"id,omitempty"`
- FileID string `gorm:"file_id" json:"file_id"`
Type string `gorm:"column:type" json:"type,omitempty"`
AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"`
LookupHash string `gorm:"column:lookup_hash" json:"lookup_hash,omitempty"`
Name string `gorm:"column:name" json:"name,omitempty"`
Path string `gorm:"column:path" json:"path,omitempty"`
- Hash string `gorm:"column:hash" json:"hash,omitempty"`
- NumBlocks int64 `gorm:"column:num_of_blocks" json:"num_blocks,omitempty"`
- PathHash string `gorm:"column:path_hash" json:"path_hash,omitempty"`
+ NumBlocks int64 `gorm:"column:num_of_blocks" json:"num_of_blocks,omitempty"`
ParentPath string `gorm:"column:parent_path" json:"parent_path,omitempty"`
PathLevel int `gorm:"column:level" json:"level,omitempty"`
CustomMeta string `gorm:"column:custom_meta" json:"custom_meta,omitempty"`
- ValidationRootSignature string `gorm:"column:validation_root_signature" json:"validation_root_signature,omitempty"`
- ValidationRoot string `gorm:"column:validation_root" json:"validation_root,omitempty"`
Size int64 `gorm:"column:size" json:"size,omitempty"`
- FixedMerkleRoot string `gorm:"column:fixed_merkle_root" json:"fixed_merkle_root,omitempty"`
ActualFileSize int64 `gorm:"column:actual_file_size" json:"actual_file_size,omitempty"`
ActualFileHashSignature string `gorm:"column:actual_file_hash_signature" json:"actual_file_hash_signature,omitempty"`
ActualFileHash string `gorm:"column:actual_file_hash" json:"actual_file_hash,omitempty"`
MimeType string `gorm:"column:mimetype" json:"mimetype,omitempty"`
- AllocationRoot string `gorm:"column:allocation_root" json:"allocation_root,omitempty"`
ThumbnailSize int64 `gorm:"column:thumbnail_size" json:"thumbnail_size,omitempty"`
ThumbnailHash string `gorm:"column:thumbnail_hash" json:"thumbnail_hash,omitempty"`
ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" json:"actual_thumbnail_size,omitempty"`
ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" json:"actual_thumbnail_hash,omitempty"`
EncryptedKey string `gorm:"column:encrypted_key" json:"encrypted_key,omitempty"`
EncryptedKeyPoint string `gorm:"column:encrypted_key_point" json:"encrypted_key_point,omitempty"`
- FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"`
+ FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" json:"file_meta_hash"`
CreatedAt common.Timestamp `gorm:"column:created_at" json:"created_at,omitempty"`
UpdatedAt common.Timestamp `gorm:"column:updated_at" json:"updated_at,omitempty"`
@@ -158,49 +146,128 @@ func GetReferenceLookup(allocationID, path string) string {
}
func NewDirectoryRef() *Ref {
- return &Ref{Type: DIRECTORY, IsPrecommit: true}
+ return &Ref{Type: DIRECTORY}
}
func NewFileRef() *Ref {
- return &Ref{Type: FILE, IsPrecommit: true}
+ return &Ref{Type: FILE}
}
// Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be return without child
-func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) {
- var dirRef *Ref
+func Mkdir(ctx context.Context, allocationID, destpath string, allocationVersion int64, ts common.Timestamp, collector QueryCollector) (*Ref, error) {
+ var err error
db := datastore.GetStore().GetTransaction(ctx)
- // cleaning path to avoid edge case issues: append '/' prefix if not added and removing suffix '/' if added
- destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/")
- dirs := strings.Split(destpath, "/")
-
- for i := range dirs {
- currentPath := filepath.Join("/", filepath.Join(dirs[:i+1]...))
- ref, err := GetReference(ctx, allocationID, currentPath)
- if err == nil {
- dirRef = ref
- continue
+ if destpath != "/" {
+ destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/")
+ }
+ destLookupHash := GetReferenceLookup(allocationID, destpath)
+ var destRef *Ref
+ cachedRef := collector.GetFromCache(destLookupHash)
+ if cachedRef != nil {
+ destRef = cachedRef
+ } else {
+ destRef, err = GetReferenceByLookupHashWithNewTransaction(destLookupHash)
+ if err != nil && err != gorm.ErrRecordNotFound {
+ return nil, err
}
+ if destRef != nil {
+ destRef.LookupHash = destLookupHash
+ defer collector.AddToCache(destRef)
+ }
+ }
+ if destRef != nil {
+ if destRef.Type != DIRECTORY {
+ return nil, common.NewError("invalid_dir_tree", "parent path is not a directory")
+ }
+ return destRef, nil
+ }
+ fields, err := common.GetAllParentPaths(destpath)
+ if err != nil {
+ logging.Logger.Error("mkdir: failed to get all parent paths", zap.Error(err), zap.String("destpath", destpath))
+ return nil, err
+ }
+ parentLookupHashes := make([]string, 0, len(fields))
+ for i := 0; i < len(fields); i++ {
+ parentLookupHashes = append(parentLookupHashes, GetReferenceLookup(allocationID, fields[i]))
+ }
+ var parentRefs []*Ref
+ collector.LockTransaction()
+ defer collector.UnlockTransaction()
+ cachedRef = collector.GetFromCache(destLookupHash)
+ if cachedRef != nil {
+ if cachedRef.Type != DIRECTORY {
+ return nil, common.NewError("invalid_dir_tree", "parent path is not a directory")
+ }
+ return cachedRef, nil
+ } else {
+ logging.Logger.Info("noEntryFound: ", zap.String("destLookupHash", destLookupHash), zap.String("destpath", destpath))
+ }
- if !errors.Is(err, gorm.ErrRecordNotFound) {
- // unexpected sql error
- return nil, err
+ tx := db.Model(&Ref{}).Select("id", "path", "type")
+ for i := 0; i < len(fields); i++ {
+ tx = tx.Or(Ref{LookupHash: parentLookupHashes[i]})
+ }
+ err = tx.Order("path").Find(&parentRefs).Error
+ if err != nil && err != gorm.ErrRecordNotFound {
+ return nil, err
+ }
+ var (
+ parentID int64
+ parentPath = "/"
+ )
+ if len(parentRefs) > 0 {
+ parentID = parentRefs[len(parentRefs)-1].ID
+ parentPath = parentRefs[len(parentRefs)-1].Path
+ for i := 0; i < len(parentRefs); i++ {
+ if parentRefs[i].Type != DIRECTORY {
+ return nil, common.NewError("invalid_dir_tree", "parent path is not a directory")
+ }
+ if parentRefs[i].ID == 0 {
+ return nil, common.NewError("invalid_dir_tree", "parent path not found")
+ }
}
+ }
+ if destpath != "/" {
+ fields = append(fields, destpath)
+ parentLookupHashes = append(parentLookupHashes, destLookupHash)
+ }
- // dir doesn't exists , create it
+ for i := len(parentRefs); i < len(fields); i++ {
+ logging.Logger.Info("mkdir: creating directory", zap.String("path", fields[i]), zap.Int("parentID", int(parentID)))
+ var parentIDRef *int64
+ if parentID > 0 {
+ parentIDRef = &parentID
+ } else if parentPath != "/" {
+ return nil, common.NewError("invalid_dir_tree", "parent path not found")
+ }
newRef := NewDirectoryRef()
newRef.AllocationID = allocationID
- newRef.Path = currentPath
- newRef.ParentPath = filepath.Join("/", filepath.Join(dirs[:i]...))
- newRef.Name = dirs[i]
- newRef.Type = DIRECTORY
+ newRef.Path = fields[i]
+ if newRef.Path != "/" {
+ newRef.ParentPath = parentPath
+ }
+ newRef.Name = filepath.Base(fields[i])
newRef.PathLevel = i + 1
- newRef.LookupHash = GetReferenceLookup(allocationID, newRef.Path)
+ newRef.ParentID = parentIDRef
+ newRef.LookupHash = parentLookupHashes[i]
+ newRef.CreatedAt = ts
+ newRef.UpdatedAt = ts
+ newRef.FileMetaHash = encryption.FastHash(newRef.GetFileMetaHashData())
+ newRef.AllocationVersion = allocationVersion
err = db.Create(newRef).Error
if err != nil {
+ logging.Logger.Error("mkdir: failed to create directory", zap.Error(err), zap.String("path", fields[i]))
return nil, err
}
+ collector.AddToCache(newRef)
+ parentID = newRef.ID
+ parentPath = newRef.Path
+ }
- dirRef = newRef
+ dirRef := &Ref{
+ AllocationID: allocationID,
+ ID: parentID,
+ Path: parentPath,
}
return dirRef, nil
@@ -262,6 +329,16 @@ func GetReferenceByLookupHash(ctx context.Context, allocationID, pathHash string
return ref, nil
}
+func GetPaginatedRefByLookupHash(ctx context.Context, pathHash string) (*PaginatedRef, error) {
+ ref := &PaginatedRef{}
+ db := datastore.GetStore().GetTransaction(ctx)
+ err := db.Model(&Ref{}).Where(&Ref{LookupHash: pathHash}).Take(ref).Error
+ if err != nil {
+ return nil, err
+ }
+ return ref, nil
+}
+
func GetReferenceByLookupHashForDownload(ctx context.Context, allocationID, pathHash string) (*Ref, error) {
ref := &Ref{}
db := datastore.GetStore().GetTransaction(ctx)
@@ -307,6 +384,22 @@ func IsRefExist(ctx context.Context, allocationID, path string) (bool, error) {
return Found, nil
}
+func GetObjectSizeByLookupHash(ctx context.Context, lookupHash string) (int64, error) {
+ db := datastore.GetStore().GetTransaction(ctx)
+ var size int64
+ err := db.Model(&Ref{}).
+ Select("size").
+ Where("lookup_hash = ?", lookupHash).
+ Take(&size).Error
+ if err != nil {
+ if err == gorm.ErrRecordNotFound {
+ return 0, nil
+ }
+ return 0, err
+ }
+ return size, nil
+}
+
// GetRefsTypeFromPaths Give list of paths it will return refs of respective path with only Type and Path selected in sql query
func GetRefsTypeFromPaths(ctx context.Context, allocationID string, paths []string) (refs []*Ref, err error) {
if len(paths) == 0 {
@@ -342,7 +435,7 @@ func GetSubDirsFromPath(p string) []string {
func GetRefWithChildren(ctx context.Context, parentRef *Ref, allocationID, path string, offset, pageLimit int) (*Ref, error) {
var refs []*Ref
t := datastore.GetStore().GetTransaction(ctx)
- db := t.Where(Ref{ParentPath: path, AllocationID: allocationID})
+ db := t.Where(Ref{ParentID: &parentRef.ID})
err := db.Order("path").
Offset(offset).
Limit(pageLimit).
@@ -415,43 +508,38 @@ func (r *Ref) GetFileMetaHashData() string {
func (fr *Ref) GetFileHashData() string {
return fmt.Sprintf(
- "%s:%s:%s:%s:%d:%s:%s:%d:%s:%d:%s",
+ "%s:%s:%s:%s:%d:%d:%s:%d",
fr.AllocationID,
fr.Type, // don't need to add it as well
fr.Name, // don't see any utility as fr.Path below has name in it
fr.Path,
fr.Size,
- fr.ValidationRoot,
- fr.FixedMerkleRoot,
fr.ActualFileSize,
fr.ActualFileHash,
fr.ChunkSize,
- fr.FileID,
)
}
func (r *Ref) GetHashData() string {
- return fmt.Sprintf("%s:%s:%s", r.AllocationID, r.Path, r.FileID)
+ return fmt.Sprintf("%s:%s", r.AllocationID, r.Path)
}
func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) {
fr.FileMetaHash = encryption.Hash(fr.GetFileMetaHashData())
- fr.Hash = encryption.Hash(fr.GetFileHashData())
fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / float64(fr.ChunkSize)))
fr.PathLevel = len(strings.Split(strings.TrimRight(fr.Path, "/"), "/"))
fr.LookupHash = GetReferenceLookup(fr.AllocationID, fr.Path)
- fr.PathHash = fr.LookupHash
var err error
if saveToDB && fr.HashToBeComputed {
err = fr.SaveFileRef(ctx, collector)
}
- return fr.Hash, err
+ return fr.FileMetaHash, err
}
func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector QueryCollector) (h string, err error) {
if !r.HashToBeComputed {
- h = r.Hash
+ h = r.FileMetaHash
return
}
@@ -464,9 +552,7 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que
}
}()
- childHashes := make([]string, l)
childFileMetaHashes := make([]string, l)
- childPathHashes := make([]string, l)
var refNumBlocks, size, actualSize int64
for i, childRef := range r.Children {
@@ -478,22 +564,18 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que
}
childFileMetaHashes[i] = childRef.FileMetaHash
- childHashes[i] = childRef.Hash
- childPathHashes[i] = childRef.PathHash
refNumBlocks += childRef.NumBlocks
size += childRef.Size
actualSize += childRef.ActualFileSize
}
r.FileMetaHash = encryption.Hash(r.Path + strings.Join(childFileMetaHashes, ":"))
- r.Hash = encryption.Hash(r.GetHashData() + strings.Join(childHashes, ":"))
- r.PathHash = encryption.Hash(strings.Join(childPathHashes, ":"))
r.NumBlocks = refNumBlocks
r.Size = size
r.ActualFileSize = actualSize
r.PathLevel = len(GetSubDirsFromPath(r.Path)) + 1
r.LookupHash = GetReferenceLookup(r.AllocationID, r.Path)
- return r.Hash, err
+ return r.FileMetaHash, err
}
func (r *Ref) CalculateHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) {
@@ -549,17 +631,8 @@ func (r *Ref) UpdatePath(newPath, parentPath string) {
r.LookupHash = GetReferenceLookup(r.AllocationID, r.Path)
}
-func DeleteReference(ctx context.Context, refID int64, pathHash string) error {
- if refID <= 0 {
- return common.NewError("invalid_ref_id", "Invalid reference ID to delete")
- }
- db := datastore.GetStore().GetTransaction(ctx)
- return db.Where("path_hash = ?", pathHash).Delete(&Ref{ID: refID}).Error
-}
-
func (r *Ref) SaveFileRef(ctx context.Context, collector QueryCollector) error {
r.prevID = r.ID
- r.IsPrecommit = true
r.NumUpdates += 1
if r.ID > 0 {
deleteRef := &Ref{ID: r.ID}
@@ -573,7 +646,6 @@ func (r *Ref) SaveFileRef(ctx context.Context, collector QueryCollector) error {
func (r *Ref) SaveDirRef(ctx context.Context, collector QueryCollector) error {
r.prevID = r.ID
- r.IsPrecommit = true
r.NumUpdates += 1
if r.ID > 0 {
deleteRef := &Ref{ID: r.ID}
@@ -675,3 +747,48 @@ func UpdateCustomMeta(ctx context.Context, ref *Ref, customMeta string) error {
db := datastore.GetStore().GetTransaction(ctx)
return db.Exec("UPDATE reference_objects SET custom_meta = ? WHERE id = ?", customMeta, ref.ID).Error
}
+
+func IsDirectoryEmpty(ctx context.Context, id int64) (bool, error) {
+ db := datastore.GetStore().GetTransaction(ctx)
+ var ref Ref
+ err := db.Model(&Ref{}).Select("id").Where("parent_id = ?", &id).Take(&ref).Error
+ if err != nil {
+ if err == gorm.ErrRecordNotFound {
+ return true, nil
+ }
+ return false, err
+ }
+ if ref.ID > 0 {
+ return false, nil
+ }
+
+ return true, nil
+}
+
+func GetReferenceByLookupHashWithNewTransaction(lookupHash string) (*Ref, error) {
+ var ref *Ref
+ err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+ txn := datastore.GetStore().GetTransaction(ctx)
+ return txn.Model(&Ref{}).Select("id", "type").Where("lookup_hash = ?", lookupHash).Take(&ref).Error
+ }, &sql.TxOptions{
+ ReadOnly: true,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return ref, nil
+}
+
+func GetFullReferenceByLookupHashWithNewTransaction(lookupHash string) (*Ref, error) {
+ var ref *Ref
+ err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+ txn := datastore.GetStore().GetTransaction(ctx)
+ return txn.Model(&Ref{}).Where("lookup_hash = ?", lookupHash).Take(&ref).Error
+ }, &sql.TxOptions{
+ ReadOnly: true,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return ref, nil
+}
diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go
index 458fe9b24..f2bfc7830 100644
--- a/code/go/0chain.net/blobbercore/reference/referencepath.go
+++ b/code/go/0chain.net/blobbercore/reference/referencepath.go
@@ -238,7 +238,7 @@ func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error)
// Might need to consider covering index for efficient search https://blog.crunchydata.com/blog/why-covering-indexes-are-incredibly-helpful
// To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs
// very easily and effectively; Same case for offsetDate.
-func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) {
+func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit, offsetTime int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) {
var (
pRefs = make([]PaginatedRef, 0, pageLimit/4)
dbError error
@@ -246,22 +246,40 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string,
)
path = filepath.Clean(path)
tx := datastore.GetStore().GetTransaction(ctx)
- pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + 1
- if pathLevel == level {
- dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND parent_path = ? and level = ?", allocationID, path, level)
+ pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/"))
+ logging.Logger.Info("getRefs: CheckSingleRef", zap.Int("pathLevel", pathLevel), zap.Int("level", level), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("pageLimit", pageLimit))
+ if (pageLimit == 1 && offsetPath == "" && (pathLevel == level || level == 0) && _type != FILE) || (parentRef != nil && parentRef.Type == FILE) {
+ pRefs = append(pRefs, *parentRef)
+ refs = &pRefs
+ newOffsetPath = parentRef.Path
+ return
+ }
+
+ if pathLevel+1 == level {
+ dbQuery = tx.Model(&Ref{}).Where("parent_id = ?", parentRef.ID)
if _type != "" {
dbQuery = dbQuery.Where("type = ?", _type)
}
dbQuery = dbQuery.Where("path > ?", offsetPath)
+ if offsetTime != 0 {
+ dbQuery = dbQuery.Where("created_at < ?", offsetTime)
+ }
dbQuery = dbQuery.Order("path")
} else {
- dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND (path=? OR path LIKE ?)", allocationID, path, path+"%")
+ listPath := path
+ if path != "/" {
+ listPath = path + "/"
+ }
+ dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%")
if _type != "" {
dbQuery = dbQuery.Where("type = ?", _type)
}
if level != 0 {
dbQuery = dbQuery.Where("level = ?", level)
}
+ if offsetTime != 0 {
+ dbQuery = dbQuery.Where("created_at < ?", offsetTime)
+ }
dbQuery = dbQuery.Where("path > ?", offsetPath)
diff --git a/code/go/0chain.net/blobbercore/util/json.go b/code/go/0chain.net/blobbercore/util/json.go
index dbfce39ac..39afbdbc1 100644
--- a/code/go/0chain.net/blobbercore/util/json.go
+++ b/code/go/0chain.net/blobbercore/util/json.go
@@ -5,14 +5,15 @@ import (
"reflect"
"strings"
- "github.com/0chain/gosdk/zboxcore/fileref"
+ "github.com/0chain/gosdk_common/zboxcore/fileref"
)
// Validate unmarshalled data with tag-based rules
// Example:
-// struct {
-// Name string `json:"name" validation:"required"`
-// }
+//
+// struct {
+// Name string `json:"name" validation:"required"`
+// }
func UnmarshalValidation(v interface{}) error {
fields := reflect.ValueOf(v).Elem()
diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go
index 4bf3663dd..006dd33ff 100644
--- a/code/go/0chain.net/blobbercore/writemarker/mutex.go
+++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go
@@ -9,7 +9,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/errors"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"go.uber.org/zap"
)
diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go
index c33c30d07..79c16b5bf 100644
--- a/code/go/0chain.net/blobbercore/writemarker/protocol.go
+++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go
@@ -3,17 +3,15 @@ package writemarker
import (
"context"
"fmt"
- "time"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
- "github.com/0chain/blobber/code/go/0chain.net/core/chain"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
- "github.com/0chain/blobber/code/go/0chain.net/core/transaction"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
+ "github.com/0chain/gosdk_common/core/transaction"
"go.uber.org/zap"
)
@@ -25,7 +23,19 @@ type CommitConnection struct {
ChainData []byte `json:"chain_data"`
}
-const timeGap = 180
+const (
+ ADD_BLOBBER_SC_NAME = "add_blobber"
+ UPDATE_BLOBBER_SC_NAME = "update_blobber_settings"
+ ADD_VALIDATOR_SC_NAME = "add_validator"
+ CLOSE_CONNECTION_SC_NAME = "commit_connection"
+ READ_REDEEM = "read_redeem"
+ CHALLENGE_RESPONSE = "challenge_response"
+ BLOBBER_HEALTH_CHECK = "blobber_health_check"
+ FINALIZE_ALLOCATION = "finalize_allocation"
+ VALIDATOR_HEALTH_CHECK = "validator_health_check"
+ STORAGE_CONTRACT_ADDRESS = "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d7"
+ timeGap = 180
+)
// VerifyMarker verify WriteMarker's hash and check allocation_root if it is unique
func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *allocation.Allocation, co *allocation.AllocationChangeCollector, latestWM *WriteMarkerEntity) error {
@@ -113,7 +123,7 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al
func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64) error {
if len(wme.CloseTxnID) > 0 {
- t, err := transaction.VerifyTransaction(wme.CloseTxnID, chain.GetServerChain())
+ t, err := transaction.VerifyTransaction(wme.CloseTxnID)
if err == nil {
wme.Status = Committed
wme.StatusMessage = t.TransactionOutput
@@ -123,19 +133,14 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64)
}
}
- txn, err := transaction.NewTransactionEntity()
- if err != nil {
- wme.StatusMessage = "Error creating transaction entity. " + err.Error()
- if err := wme.UpdateStatus(ctx, Failed, "Error creating transaction entity. "+err.Error(), "", startSeq, wme.Sequence); err != nil {
- Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err))
- }
- return err
- }
-
+ var out, hash string
+ var nonce int64
+ txn := &transaction.Transaction{}
sn := &CommitConnection{}
sn.AllocationRoot = wme.WM.AllocationRoot
sn.PrevAllocationRoot = wme.WM.PreviousAllocationRoot
sn.WriteMarker = &wme.WM
+ var err error
err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
sn.ChainData, err = GetMarkersForChain(ctx, wme.WM.AllocationID, startSeq, wme.Sequence-1)
return err
@@ -148,10 +153,9 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64)
return err
}
- if sn.AllocationRoot == sn.PrevAllocationRoot {
+ if sn.AllocationRoot == sn.PrevAllocationRoot && wme.WM.Version != MARKER_VERSION {
// get nonce of prev WM
- var prevWM *WriteMarkerEntity
- prevWM, err = GetPreviousWM(ctx, sn.AllocationRoot, wme.WM.Timestamp)
+ _, err = GetPreviousWM(ctx, sn.AllocationRoot, wme.WM.Timestamp)
if err != nil {
wme.StatusMessage = "Error getting previous write marker. " + err.Error()
if err := wme.UpdateStatus(ctx, Failed, "Error getting previous write marker. "+err.Error(), "", startSeq, wme.Sequence); err != nil {
@@ -159,10 +163,13 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64)
}
return err
}
- err = txn.ExecuteRollbackWM(transaction.STORAGE_CONTRACT_ADDRESS, transaction.CLOSE_CONNECTION_SC_NAME, sn, 0, prevWM.CloseTxnNonce)
- } else {
- err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.CLOSE_CONNECTION_SC_NAME, sn, 0)
+
}
+
+ hash, out, nonce, txn, err = transaction.SmartContractTxn(STORAGE_CONTRACT_ADDRESS, transaction.SmartContractTxnData{
+ Name: CLOSE_CONNECTION_SC_NAME,
+ InputArgs: sn,
+ }, true)
if err != nil {
Logger.Error("Failed during sending close connection to the miner. ", zap.String("err:", err.Error()))
wme.Status = Failed
@@ -173,23 +180,11 @@ func (wme *WriteMarkerEntity) redeemMarker(ctx context.Context, startSeq int64)
return err
}
- time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second)
- t, err := transaction.VerifyTransactionWithNonce(txn.Hash, txn.GetTransaction().GetTransactionNonce())
wme.CloseTxnID = txn.Hash
- wme.CloseTxnNonce = txn.GetTransaction().GetTransactionNonce()
- if err != nil {
- Logger.Error("Error verifying the close connection transaction", zap.String("err:", err.Error()), zap.String("txn", txn.Hash))
- wme.Status = Failed
- wme.StatusMessage = "Error verifying the close connection transaction." + err.Error()
- // TODO Is this single try?
- if err := wme.UpdateStatus(ctx, Failed, "Error verifying the close connection transaction."+err.Error(), txn.Hash, startSeq, wme.Sequence); err != nil {
- Logger.Error("WriteMarkerEntity_UpdateStatus", zap.Error(err))
- }
- return err
- }
+ wme.CloseTxnNonce = nonce
wme.Status = Committed
- wme.StatusMessage = t.TransactionOutput
- err = wme.UpdateStatus(ctx, Committed, t.TransactionOutput, t.Hash, startSeq, wme.Sequence)
+ wme.StatusMessage = out
+ err = wme.UpdateStatus(ctx, Committed, out, hash, startSeq, wme.Sequence)
return err
}
diff --git a/code/go/0chain.net/blobbercore/writemarker/version_marker.go b/code/go/0chain.net/blobbercore/writemarker/version_marker.go
new file mode 100644
index 000000000..757c7723c
--- /dev/null
+++ b/code/go/0chain.net/blobbercore/writemarker/version_marker.go
@@ -0,0 +1,69 @@
+package writemarker
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
+ "github.com/0chain/blobber/code/go/0chain.net/core/common"
+ "github.com/0chain/blobber/code/go/0chain.net/core/encryption"
+ "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+ "go.uber.org/zap"
+)
+
+type VersionMarker struct {
+ ID int64 `gorm:"column:id;primaryKey"`
+ ClientID string `gorm:"client_id" json:"client_id"`
+ BlobberID string `gorm:"blobber_id" json:"blobber_id"`
+ AllocationID string `gorm:"allocation_id" json:"allocation_id"`
+ Version int64 `gorm:"version" json:"version"`
+ Timestamp int64 `gorm:"timestamp" json:"timestamp"`
+ Signature string `gorm:"signature" json:"signature"`
+ IsRepair bool `gorm:"is_repair" json:"is_repair"`
+ RepairVersion int64 `gorm:"repair_version" json:"repair_version"`
+ RepairOffset string `gorm:"repair_offset" json:"repair_offset"`
+}
+
+func (VersionMarker) TableName() string {
+ return "version_markers"
+}
+
+func GetCurrentVersion(ctx context.Context, allocationID string) (*VersionMarker, error) {
+ db := datastore.GetStore().GetTransaction(ctx)
+ var vm VersionMarker
+ err := db.Where("allocation_id = ?", allocationID).Order("id DESC").Take(&vm).Error
+ return &vm, err
+}
+
+func GetVersionMarker(ctx context.Context, allocationID string, version int64) (*VersionMarker, error) {
+ db := datastore.GetStore().GetTransaction(ctx)
+ var vm VersionMarker
+ err := db.Where("allocation_id = ? and version = ?", allocationID, version).Order("id DESC").Take(&vm).Error
+ return &vm, err
+}
+
+func (vm *VersionMarker) Verify(allocationID, clientPubKey string) error {
+ if vm.AllocationID != allocationID {
+ return common.NewError("version_marker_validation_failed", "Invalid allocation id")
+ }
+
+ if vm.Signature == "" {
+ return common.NewError("version_marker_validation_failed", "Signature is missing")
+ }
+
+ hashData := vm.GetHashData()
+ signatureHash := encryption.Hash(hashData)
+ sigOK, err := encryption.Verify(clientPubKey, vm.Signature, signatureHash)
+ if err != nil {
+ return common.NewError("version_marker_validation_failed", "Error during verifying signature. "+err.Error())
+ }
+ if !sigOK {
+ logging.Logger.Error("write_marker_sig_error", zap.Any("vm", vm))
+ return common.NewError("version_marker_validation_failed", "Version marker signature is not valid")
+ }
+ return nil
+}
+
+func (vm *VersionMarker) GetHashData() string {
+ return fmt.Sprintf("%s:%s:%s:%d:%d", vm.AllocationID, vm.ClientID, vm.BlobberID, vm.Version, vm.Timestamp)
+}
diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go
index 7d5a3f924..7801a7c82 100644
--- a/code/go/0chain.net/blobbercore/writemarker/worker.go
+++ b/code/go/0chain.net/blobbercore/writemarker/worker.go
@@ -98,19 +98,19 @@ func deleteMarkerData(allocationID string) {
// )
func SetupWorkers(ctx context.Context) {
- var res []allocation.Res
+ // var res []allocation.Res
- err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
- res = allocation.Repo.GetAllocationIds(ctx)
- return nil
- })
- if err != nil && err != gorm.ErrRecordNotFound {
- logging.Logger.Error("error_getting_allocations_worker",
- zap.Any("error", err))
- }
+ // err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+ // res = allocation.Repo.GetAllocationIds(ctx)
+ // return nil
+ // })
+ // if err != nil && err != gorm.ErrRecordNotFound {
+ // logging.Logger.Error("error_getting_allocations_worker",
+ // zap.Any("error", err))
+ // }
- startRedeem(ctx, res)
- go startCollector(ctx)
+ // startRedeem(ctx, res)
+ // go startCollector(ctx)
// go startCleanupWorker(ctx)
}
diff --git a/code/go/0chain.net/blobbercore/zcn/query.go b/code/go/0chain.net/blobbercore/zcn/query.go
index 7b122ca77..1f00f1eb6 100644
--- a/code/go/0chain.net/blobbercore/zcn/query.go
+++ b/code/go/0chain.net/blobbercore/zcn/query.go
@@ -6,30 +6,23 @@ import (
"fmt"
"sync"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk/zboxcore/sdk"
+ "github.com/0chain/gosdk_common/zcncore"
)
var ErrBlobberNotFound = errors.New("blobber is not found on chain")
// GetBlobber try to get blobber info from chain.
-func GetBlobber(blobberID string) (*zcncore.Blobber, error) {
- cb := &getBlobberCallback{}
- cb.wg.Add(1)
- if err := zcncore.GetBlobber(blobberID, cb); err != nil {
- cb.wg.Done()
+func GetBlobber(blobberID string) (*sdk.Blobber, error) {
+ var (
+ blobber *sdk.Blobber
+ err error
+ )
+ if blobber, err = sdk.GetBlobber(blobberID); err != nil {
return nil, err
}
- cb.wg.Wait()
- if cb.Error != nil {
- return nil, cb.Error
- }
-
- if cb.Blobber == nil {
- return nil, ErrBlobberNotFound
- }
-
- return cb.Blobber, nil
+ return blobber, nil
}
diff --git a/code/go/0chain.net/core/chain/entity.go b/code/go/0chain.net/core/chain/entity.go
index 7360a4f93..4c82c3d13 100644
--- a/code/go/0chain.net/core/chain/entity.go
+++ b/code/go/0chain.net/core/chain/entity.go
@@ -23,13 +23,13 @@ func GetServerChain() *Chain {
/*Chain - data structure that holds the chain data*/
type Chain struct {
- ID string
- Version string
- CreationDate common.Timestamp
- OwnerID string
- ParentChainID string
- BlockWorker string
-
+ ID string
+ Version string
+ CreationDate common.Timestamp
+ OwnerID string
+ ParentChainID string
+ BlockWorker string
+ ZauthServer string
GenesisBlockHash string
}
@@ -44,12 +44,13 @@ func (c *Chain) Validate(ctx context.Context) error {
return nil
}
-//NewChainFromConfig - create a new chain from config
+// NewChainFromConfig - create a new chain from config
func NewChainFromConfig() *Chain {
chain := Provider()
chain.ID = common.ToKey(config.Configuration.ChainID)
chain.OwnerID = viper.GetString("server_chain.owner")
chain.BlockWorker = viper.GetString("block_worker")
+ chain.ZauthServer = viper.GetString("zauth_server")
return chain
}
diff --git a/code/go/0chain.net/core/common/utils.go b/code/go/0chain.net/core/common/utils.go
index d3ee7a82d..7aa8cdd2b 100644
--- a/code/go/0chain.net/core/common/utils.go
+++ b/code/go/0chain.net/core/common/utils.go
@@ -47,6 +47,29 @@ func GetParentPaths(fPath string) ([]string, error) {
return paths[2:], nil
}
+func GetAllParentPaths(fPath string) ([]string, error) {
+ if fPath == "" {
+ return nil, nil
+ }
+ if fPath == "/" {
+ return []string{"/"}, nil
+ }
+
+ fPath = filepath.Clean(fPath)
+ if !filepath.IsAbs(fPath) {
+ return nil, NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fPath))
+ }
+ splittedPaths := strings.Split(fPath, "/")
+ var paths []string
+ for i := 0; i < len(splittedPaths); i++ {
+ subPath := strings.Join(splittedPaths[0:i], "/")
+ paths = append(paths, subPath)
+ }
+ returnPaths := []string{"/"}
+ returnPaths = append(returnPaths, paths[2:]...)
+ return returnPaths, nil
+}
+
// GetPathFields will return slice of fields of path.
// For path /a/b/c/d/e/f.txt it will return [a, b, c, d, e, f.txt],nil
func GetPathFields(p string) ([]string, error) {
diff --git a/code/go/0chain.net/core/encryption/hash.go b/code/go/0chain.net/core/encryption/hash.go
index 53f649343..b6859cb1b 100644
--- a/code/go/0chain.net/core/encryption/hash.go
+++ b/code/go/0chain.net/core/encryption/hash.go
@@ -3,6 +3,8 @@ package encryption
import (
"crypto/sha1"
"encoding/hex"
+ "hash"
+ "sync"
"github.com/minio/sha256-simd"
"golang.org/x/crypto/sha3"
@@ -10,6 +12,12 @@ import (
const HASH_LENGTH = 32
+var sha3Pool = sync.Pool{
+ New: func() interface{} {
+ return sha3.New256()
+ },
+}
+
type HashBytes [HASH_LENGTH]byte
/*Hash - hash the given data and return the hash as hex string */
@@ -30,9 +38,12 @@ func RawHash(data interface{}) []byte {
default:
panic("unknown type")
}
- hash := sha3.New256()
+ hash := sha3Pool.Get().(hash.Hash)
hash.Write(databuf)
- return hash.Sum(nil)
+ res := hash.Sum(nil)
+ hash.Reset()
+ sha3Pool.Put(hash)
+ return res
}
func ShaHash(data interface{}) []byte {
diff --git a/code/go/0chain.net/core/encryption/keys.go b/code/go/0chain.net/core/encryption/keys.go
index bb652f1bf..58327656c 100644
--- a/code/go/0chain.net/core/encryption/keys.go
+++ b/code/go/0chain.net/core/encryption/keys.go
@@ -9,12 +9,14 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/config"
. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/core/zcncrypto"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
"github.com/herumi/bls-go-binary/bls"
)
-/*ReadKeys - reads a publicKey and a privateKey from a Reader.
-They are assumed to be in two separate lines one followed by the other*/
+/*
+ReadKeys - reads a publicKey and a privateKey from a Reader.
+They are assumed to be in two separate lines one followed by the other
+*/
func ReadKeys(reader io.Reader) (publicKey, privateKey, publicIp, port string) {
scanner := bufio.NewScanner(reader)
scanner.Scan()
@@ -45,13 +47,14 @@ func Verify(publicKey, signature, hash string) (bool, error) {
}
// If input is normal herumi/bls public key, it returns it immmediately.
-// So this is completely backward compatible with herumi/bls.
+//
+// So this is completely backward compatible with herumi/bls.
+//
// If input is MIRACL public key, convert it to herumi/bls public key.
//
// This is an example of the raw public key we expect from MIRACL
var miraclExamplePK = `0418a02c6bd223ae0dfda1d2f9a3c81726ab436ce5e9d17c531ff0a385a13a0b491bdfed3a85690775ee35c61678957aaba7b1a1899438829f1dc94248d87ed36817f6dfafec19bfa87bf791a4d694f43fec227ae6f5a867490e30328cac05eaff039ac7dfc3364e851ebd2631ea6f1685609fc66d50223cc696cb59ff2fee47ac`
-//
// This is an example of the same MIRACL public key serialized with ToString().
// pk ([1bdfed3a85690775ee35c61678957aaba7b1a1899438829f1dc94248d87ed368,18a02c6bd223ae0dfda1d2f9a3c81726ab436ce5e9d17c531ff0a385a13a0b49],[039ac7dfc3364e851ebd2631ea6f1685609fc66d50223cc696cb59ff2fee47ac,17f6dfafec19bfa87bf791a4d694f43fec227ae6f5a867490e30328cac05eaff])
func MiraclToHerumiPK(pk string) string {
diff --git a/code/go/0chain.net/core/encryption/keys_test.go b/code/go/0chain.net/core/encryption/keys_test.go
index b0a71afc3..1b411a0e3 100644
--- a/code/go/0chain.net/core/encryption/keys_test.go
+++ b/code/go/0chain.net/core/encryption/keys_test.go
@@ -3,8 +3,8 @@ package encryption
import (
"testing"
- "github.com/0chain/gosdk/core/sys"
- "github.com/0chain/gosdk/zboxcore/client"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/core/sys"
"github.com/herumi/bls-go-binary/bls"
"github.com/stretchr/testify/require"
)
@@ -12,7 +12,8 @@ import (
func TestSignatureVerify(t *testing.T) {
allocationId := "4f928c7857fabb5737347c42204eea919a4777f893f35724f563b932f64e2367"
walletConfig := "{\"client_id\":\"9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85\",\"client_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"keys\":[{\"public_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"private_key\":\"a3a88aad5d89cec28c6e37c2925560ce160ac14d2cdcf4a4654b2bb358fe7514\"}],\"mnemonics\":\"inside february piece turkey offer merry select combine tissue wave wet shift room afraid december gown mean brick speak grant gain become toy clown\",\"version\":\"1.0\",\"date_created\":\"2021-05-21 17:32:29.484657 +0545 +0545 m=+0.072791323\"}"
- require.NoError(t, client.PopulateClient(walletConfig, "bls0chain"))
+ _, err := client.PopulateClient(walletConfig, "bls0chain")
+ require.NoError(t, err)
sig, serr := client.Sign(allocationId)
require.Nil(t, serr)
require.NotNil(t, sig)
diff --git a/code/go/0chain.net/core/node/context.go b/code/go/0chain.net/core/node/context.go
index 083851263..d5d0bb97c 100644
--- a/code/go/0chain.net/core/node/context.go
+++ b/code/go/0chain.net/core/node/context.go
@@ -3,7 +3,7 @@ package node
import (
"context"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
)
const SELF_NODE constants.ContextKey = "SELF_NODE"
diff --git a/code/go/0chain.net/core/node/self_node.go b/code/go/0chain.net/core/node/self_node.go
index c58692f59..047980d91 100644
--- a/code/go/0chain.net/core/node/self_node.go
+++ b/code/go/0chain.net/core/node/self_node.go
@@ -4,9 +4,10 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
+
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/config"
- "github.com/0chain/gosdk/core/zcncrypto"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
"golang.org/x/crypto/sha3"
)
diff --git a/code/go/0chain.net/core/transaction/entity.go b/code/go/0chain.net/core/transaction/entity.go
index 6a41ff980..c0c80e7d6 100644
--- a/code/go/0chain.net/core/transaction/entity.go
+++ b/code/go/0chain.net/core/transaction/entity.go
@@ -1,20 +1,10 @@
package transaction
import (
- "encoding/json"
"sync"
"time"
- "github.com/0chain/gosdk/core/transaction"
-
- "github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "go.uber.org/zap"
-
- "github.com/0chain/gosdk/zcncore"
-
- "github.com/0chain/blobber/code/go/0chain.net/core/chain"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/blobber/code/go/0chain.net/core/node"
)
var (
@@ -22,25 +12,6 @@ var (
last50TransactionsMutex sync.Mutex
)
-// Transaction entity that encapsulates the transaction related data and meta data
-type Transaction struct {
- Hash string `json:"hash,omitempty"`
- Version string `json:"version,omitempty"`
- ClientID string `json:"client_id,omitempty"`
- PublicKey string `json:"public_key,omitempty"`
- ToClientID string `json:"to_client_id,omitempty"`
- ChainID string `json:"chain_id,omitempty"`
- TransactionData string `json:"transaction_data,omitempty"`
- Value int64 `json:"transaction_value,omitempty"`
- Signature string `json:"signature,omitempty"`
- CreationDate common.Timestamp `json:"creation_date,omitempty"`
- TransactionType int `json:"transaction_type,omitempty"`
- TransactionOutput string `json:"transaction_output,omitempty"`
- OutputHash string `json:"txn_output_hash"`
- zcntxn zcncore.TransactionScheme
- wg *sync.WaitGroup
-}
-
type SmartContractTxnData struct {
Name string `json:"name"`
InputArgs interface{} `json:"input"`
@@ -73,6 +44,7 @@ type StorageNode struct {
PublicKey string `json:"-"`
StakePoolSettings StakePoolSettings `json:"stake_pool_settings"`
IsEnterprise bool `json:"is_enterprise"`
+ StorageVersion int `json:"storage_version"`
}
type BlobberAllocation struct {
@@ -81,19 +53,20 @@ type BlobberAllocation struct {
}
type StorageAllocation struct {
- ID string `json:"id"`
- Tx string `json:"tx"`
- OwnerPublicKey string `json:"owner_public_key"`
- OwnerID string `json:"owner_id"`
- Size int64 `json:"size"`
- UsedSize int64 `json:"used_size"`
- Expiration common.Timestamp `json:"expiration_date"`
- BlobberDetails []*BlobberAllocation `json:"blobber_details"`
- Finalized bool `json:"finalized"`
- TimeUnit time.Duration `json:"time_unit"`
- WritePool uint64 `json:"write_pool"`
- FileOptions uint16 `json:"file_options"`
- StartTime common.Timestamp `json:"start_time"`
+ ID string `json:"id"`
+ Tx string `json:"tx"`
+ OwnerPublicKey string `json:"owner_public_key"`
+ OwnerID string `json:"owner_id"`
+ Size int64 `json:"size"`
+ UsedSize int64 `json:"used_size"`
+ Expiration common.Timestamp `json:"expiration_date"`
+ BlobberDetails []*BlobberAllocation `json:"blobber_details"`
+ Finalized bool `json:"finalized"`
+ TimeUnit time.Duration `json:"time_unit"`
+ WritePool uint64 `json:"write_pool"`
+ FileOptions uint16 `json:"file_options"`
+ StartTime common.Timestamp `json:"start_time"`
+ OwnerSigningPublicKey string `json:"owner_signing_public_key"`
DataShards int64 `json:"data_shards"`
ParityShards int64 `json:"parity_shards"`
@@ -120,216 +93,6 @@ const (
const STORAGE_CONTRACT_ADDRESS = "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d7"
-func NewTransactionEntity() (*Transaction, error) {
- txn := &Transaction{}
- txn.Version = "1.0"
- txn.ClientID = node.Self.ID
- txn.CreationDate = common.Now()
- txn.ChainID = chain.GetServerChain().ID
- txn.PublicKey = node.Self.PublicKey
- txn.wg = &sync.WaitGroup{}
- zcntxn, err := zcncore.NewTransaction(txn, 0, 0)
- if err != nil {
- return nil, err
- }
- txn.zcntxn = zcntxn
- return txn, nil
-}
-
-func (t *Transaction) GetTransaction() zcncore.TransactionScheme {
- return t.zcntxn
-}
-
-func (t *Transaction) ExecuteSmartContract(address, methodName string, input interface{}, val uint64) error {
- t.wg.Add(1)
-
- sn := transaction.SmartContractTxnData{Name: methodName, InputArgs: input}
- snBytes, err := json.Marshal(sn)
- if err != nil {
- return err
- }
-
- updateLast50Transactions(string(snBytes))
-
- nonce := monitor.getNextUnusedNonce()
- if err := t.zcntxn.SetTransactionNonce(nonce); err != nil {
- logging.Logger.Error("Failed to set nonce.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", nonce),
- zap.Any("error", err))
- }
-
- logging.Logger.Info("Transaction nonce set.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", nonce))
-
- _, err = t.zcntxn.ExecuteSmartContract(address, methodName, input, uint64(val))
- if err != nil {
- t.wg.Done()
- logging.Logger.Error("Failed to execute SC.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", t.zcntxn.GetTransactionNonce()),
- zap.Any("error", err))
- monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce())
- return err
- }
-
- t.wg.Wait()
-
- t.Hash = t.zcntxn.GetTransactionHash()
- if len(t.zcntxn.GetTransactionError()) > 0 {
- logging.Logger.Error("Failed to submit SC.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", t.zcntxn.GetTransactionNonce()),
- zap.Any("error", t.zcntxn.GetTransactionError()))
- monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce())
- return common.NewError("transaction_send_error", t.zcntxn.GetTransactionError())
- }
- return nil
-}
-
-func (t *Transaction) ExecuteRollbackWM(address, methodName string, input interface{}, val uint64, prevNonce int64) error {
- t.wg.Add(1)
-
- sn := transaction.SmartContractTxnData{Name: methodName, InputArgs: input}
- snBytes, err := json.Marshal(sn)
- if err != nil {
- return err
- }
-
- updateLast50Transactions(string(snBytes))
-
- nonce := monitor.getNextUnusedNonce()
- if nonce < prevNonce {
- t.wg.Done()
- logging.Logger.Error("Failed to set nonce as prevNonce is greater",
- zap.Any("nonce", nonce),
- zap.Any("prevNonce", prevNonce),
- )
- monitor.recordFailedNonce(nonce)
- return common.NewError("transaction_send_error", "Failed to set nonce as prevNonce is greater")
- }
- if err := t.zcntxn.SetTransactionNonce(nonce); err != nil {
- logging.Logger.Error("Failed to set nonce.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", nonce),
- zap.Any("error", err))
- }
-
- logging.Logger.Info("Transaction nonce set.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", nonce))
-
- _, err = t.zcntxn.ExecuteSmartContract(address, methodName, input, uint64(val))
- if err != nil {
- t.wg.Done()
- logging.Logger.Error("Failed to execute SC.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", t.zcntxn.GetTransactionNonce()),
- zap.Any("error", err))
- monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce())
- return err
- }
-
- t.wg.Wait()
-
- t.Hash = t.zcntxn.GetTransactionHash()
- if len(t.zcntxn.GetTransactionError()) > 0 {
- logging.Logger.Error("Failed to submit SC.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", t.zcntxn.GetTransactionNonce()),
- zap.Any("error", t.zcntxn.GetTransactionError()))
- monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce())
- return common.NewError("transaction_send_error", t.zcntxn.GetTransactionError())
- }
- return nil
-}
-
-func (t *Transaction) Verify() error {
- if err := t.zcntxn.SetTransactionHash(t.Hash); err != nil {
- monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce())
- logging.Logger.Error("Failed to set txn hash.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", t.zcntxn.GetTransactionNonce()),
- zap.Any("error", err))
- return err
- }
- t.wg.Add(1)
- err := t.zcntxn.Verify()
- if err != nil {
- t.wg.Done()
- logging.Logger.Error("Failed to start txn verification.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", t.zcntxn.GetTransactionNonce()),
- zap.Any("error", err))
- monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce())
- return err
- }
- t.wg.Wait()
- if len(t.zcntxn.GetVerifyError()) > 0 {
- logging.Logger.Error("Failed to verify txn.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", t.zcntxn.GetTransactionNonce()),
- zap.Any("error", t.zcntxn.GetVerifyError()),
- zap.Any("verify_output", t.zcntxn.GetVerifyOutput()))
- monitor.recordFailedNonce(t.zcntxn.GetTransactionNonce())
- return common.NewError("transaction_verify_error", t.zcntxn.GetVerifyError())
- } else {
- logging.Logger.Info("Successful txn verification.",
- zap.Any("hash", t.zcntxn.GetTransactionHash()),
- zap.Any("nonce", t.zcntxn.GetTransactionNonce()))
- monitor.recordSuccess(t.zcntxn.GetTransactionNonce())
- }
-
- output := t.zcntxn.GetVerifyOutput()
-
- var objmap map[string]json.RawMessage
- err = json.Unmarshal([]byte(output), &objmap)
- if err != nil {
- // it is a plain error message from blockchain. The format is `error_code: error message`. eg verify_challenge: could not find challenge, value not present
- // so it is impossible to decode as map[string]json.RawMessage.
- return common.NewError("transaction_verify_error", string(output))
- }
-
- err = json.Unmarshal(objmap["txn"], t)
- if err != nil {
- var confirmation map[string]json.RawMessage
- err = json.Unmarshal(objmap["confirmation"], &confirmation)
- if err != nil {
- return common.NewError("transaction_verify_error", "Error unmarshaling verify output->confirmation: "+string(output)+" "+err.Error())
- }
- err = json.Unmarshal(confirmation["txn"], t)
- if err != nil {
- return common.NewError("transaction_verify_error", "Error unmarshaling verify output->confirmation->txn: "+string(output)+" "+err.Error())
- }
- }
- return nil
-}
-
-// func (t *Transaction) ComputeHashAndSign() error {
-// hashdata := fmt.Sprintf("%v:%v:%v:%v:%v", t.CreationDate, t.ClientID,
-// t.ToClientID, t.Value, encryption.Hash(t.TransactionData))
-// t.Hash = encryption.Hash(hashdata)
-// var err error
-// t.Signature, err = node.Self.Sign(t.Hash)
-// if err != nil {
-// return err
-// }
-// return nil
-// }
-
-func (t *Transaction) OnTransactionComplete(zcntxn *zcncore.Transaction, status int) {
- t.wg.Done()
-}
-
-func (t *Transaction) OnVerifyComplete(zcntxn *zcncore.Transaction, status int) {
- t.wg.Done()
-}
-
-func (t *Transaction) OnAuthComplete(zcntxn *zcncore.Transaction, status int) {
-
-}
-
func updateLast50Transactions(data string) {
last50TransactionsMutex.Lock()
defer last50TransactionsMutex.Unlock()
diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go
index 52feee60c..f38f0bd06 100644
--- a/code/go/0chain.net/core/transaction/http.go
+++ b/code/go/0chain.net/core/transaction/http.go
@@ -1,9 +1,8 @@
package transaction
import (
- "github.com/0chain/blobber/code/go/0chain.net/core/chain"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/gosdk/zboxcore/zboxutil"
+ "github.com/0chain/gosdk_common/core/screstapi"
)
const TXN_SUBMIT_URL = "v1/transaction/put"
@@ -16,39 +15,8 @@ const (
)
var ErrNoTxnDetail = common.NewError("missing_transaction_detail", "No transaction detail was found on any of the sharders")
-var MakeSCRestAPICall func(scAddress string, relativePath string, params map[string]string) ([]byte, error) = MakeSCRestAPICallNoHandler
+var MakeSCRestAPICall func(scAddress string, relativePath string, params map[string]string, options ...string) ([]byte, error) = MakeSCRestAPICallNoHandler
-func MakeSCRestAPICallNoHandler(address string, path string, params map[string]string) ([]byte, error) {
- return zboxutil.MakeSCRestAPICall(address, path, params, nil)
-}
-
-func VerifyTransaction(txnHash string, chain *chain.Chain) (*Transaction, error) {
- txn, err := NewTransactionEntity()
- if err != nil {
- return nil, err
- }
-
- txn.Hash = txnHash
- err = txn.Verify()
- if err != nil {
- return nil, err
- }
- return txn, nil
-}
-
-// VerifyTransactionWithNonce verifies a transaction with known nonce.
-func VerifyTransactionWithNonce(txnHash string, nonce int64) (*Transaction, error) {
- txn, err := NewTransactionEntity()
- if err != nil {
- return nil, err
- }
-
- txn.Hash = txnHash
- _ = txn.zcntxn.SetTransactionNonce(nonce)
-
- err = txn.Verify()
- if err != nil {
- return nil, err
- }
- return txn, nil
+func MakeSCRestAPICallNoHandler(address string, path string, params map[string]string, options ...string) ([]byte, error) {
+ return screstapi.MakeSCRestAPICall(address, path, params, options...)
}
diff --git a/code/go/0chain.net/core/transaction/nonce.go b/code/go/0chain.net/core/transaction/nonce.go
index 44013c7a6..403031e27 100644
--- a/code/go/0chain.net/core/transaction/nonce.go
+++ b/code/go/0chain.net/core/transaction/nonce.go
@@ -5,7 +5,8 @@ import (
"time"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/zcncore"
"go.uber.org/zap"
)
@@ -88,23 +89,7 @@ func (m *nonceMonitor) refreshFromBalance() {
// sync lock not necessary, this is expected to be called within a synchronized function.
m.shouldRefreshFromBalance = false
- cb := &getNonceCallBack{waitCh: make(chan struct{})}
- if err := zcncore.GetNonce(cb); err != nil {
- return
- }
-
- <-cb.waitCh
-
- newNonce := int64(0)
- if cb.hasError {
- logging.Logger.Info("Couldn't get nonce from remote, use 0")
- newNonce = int64(0)
- } else {
- logging.Logger.Info("Got nonce from balance.", zap.Any("nonce", cb.nonce), zap.Any("highestSuccess", m.highestSuccess))
- newNonce = cb.nonce
- }
-
- m.highestSuccess = newNonce
+ m.highestSuccess = client.Nonce()
m.failed = make(map[int64]int64)
m.used = make(map[int64]time.Time)
diff --git a/code/go/0chain.net/validator/main.go b/code/go/0chain.net/validator/main.go
index bf5ca7f12..330387e93 100644
--- a/code/go/0chain.net/validator/main.go
+++ b/code/go/0chain.net/validator/main.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"flag"
"fmt"
"log"
@@ -21,12 +22,13 @@ import (
. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
- "github.com/0chain/blobber/code/go/0chain.net/core/util"
"github.com/0chain/blobber/code/go/0chain.net/validatorcore/config"
"github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage"
+ coreTxn "github.com/0chain/gosdk_common/core/transaction"
"github.com/0chain/gosdk/zboxcore/sdk"
- "github.com/0chain/gosdk/zcncore"
+ "github.com/0chain/gosdk_common/core/client"
+ "github.com/0chain/gosdk_common/zcncore"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/spf13/viper"
@@ -202,28 +204,26 @@ func RegisterValidator() {
}
for {
- txn, err := storage.GetProtocolImpl().RegisterValidator(common.GetRootContext())
+ sn := &transaction.StorageNode{}
+ sn.ID = node.Self.ID
+ sn.BaseURL = node.Self.GetURLBase()
+ sn.StakePoolSettings.DelegateWallet = config.Configuration.DelegateWallet
+ sn.StakePoolSettings.NumDelegates = config.Configuration.NumDelegates
+ sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge
+
+ hash, out, _, _, err := coreTxn.SmartContractTxn(transaction.STORAGE_CONTRACT_ADDRESS, coreTxn.SmartContractTxnData{
+ Name: transaction.ADD_VALIDATOR_SC_NAME,
+ InputArgs: sn,
+ }, true)
if err != nil {
- Logger.Error("Error registering validator", zap.Any("err", err))
+ Logger.Error("Add validator transaction could not be verified", zap.Any("err", err), zap.String("txn.Hash", hash))
continue
}
- time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second)
- txnVerified := false
- verifyRetries := 0
- for verifyRetries < util.MAX_RETRIES {
- time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second)
- t, err := transaction.VerifyTransactionWithNonce(txn.Hash, txn.GetTransaction().GetTransactionNonce())
- if err == nil {
- Logger.Info("Transaction for adding validator accepted and verified", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput))
- go handler.StartHealthCheck(common.GetRootContext(), common.ProviderTypeValidator)
- return
- }
- verifyRetries++
- }
- if !txnVerified {
- Logger.Error("Add validator transaction could not be verified", zap.Any("err", err), zap.String("txn.Hash", txn.Hash))
- }
+ Logger.Info("Transaction for adding validator accepted and verified", zap.String("txn_hash", hash), zap.Any("txn_output", out))
+
+ go handler.StartHealthCheck(common.GetRootContext(), common.ProviderTypeValidator)
+ break
}
}
@@ -232,17 +232,16 @@ func SetupValidatorOnBC(logDir string) error {
var logName = logDir + "/validator.log"
zcncore.SetLogFile(logName, false)
zcncore.SetLogLevel(3)
- if err := zcncore.InitZCNSDK(serverChain.BlockWorker, config.Configuration.SignatureScheme); err != nil {
- return err
- }
- if err := zcncore.SetWalletInfo(node.Self.GetWalletString(), false); err != nil {
+ err := client.InitSDK("{}", serverChain.BlockWorker, config.Configuration.ChainID, config.Configuration.SignatureScheme, int64(0), false)
+ if err != nil {
return err
}
- var blob []string
- if err := sdk.InitStorageSDK(node.Self.GetWalletString(), serverChain.BlockWorker,
- config.Configuration.ChainID, config.Configuration.SignatureScheme, blob, int64(0)); err != nil {
+
+ err = zcncore.SetGeneralWalletInfo(node.Self.GetWalletString(), config.Configuration.SignatureScheme)
+ if err != nil {
return err
}
+
go RegisterValidator()
return nil
}
@@ -254,7 +253,7 @@ func HomePageHandler(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Working on the chain: %v
\n", mc.ID)
fmt.Fprintf(w, "I am a validator with
- id:%v
- public_key:%v
- build_tag:%v
Miners ...\n")
- network := zcncore.GetNetwork()
+ network, _ := client.GetNetwork(context.Background())
for _, miner := range network.Miners {
fmt.Fprintf(w, "%v\n", miner)
}
diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler.go b/code/go/0chain.net/validatorcore/storage/challenge_handler.go
index e88aaba8f..53999cf32 100644
--- a/code/go/0chain.net/validatorcore/storage/challenge_handler.go
+++ b/code/go/0chain.net/validatorcore/storage/challenge_handler.go
@@ -13,6 +13,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
+ "github.com/0chain/blobber/code/go/0chain.net/core/transaction"
"go.uber.org/zap"
"golang.org/x/crypto/sha3"
@@ -41,7 +42,7 @@ func challengeHandler(ctx context.Context, r *http.Request) (interface{}, error)
time.Sleep(1 * time.Second)
- allocationObj, err := GetProtocolImpl().VerifyAllocationTransaction(ctx, challengeObj.AllocationID)
+ allocationObj, err := requestAllocation(challengeObj.AllocationID)
if err != nil {
logging.Logger.Error("Error verifying the allocation from BC", zap.String("allocation_id", challengeObj.AllocationID), zap.Error(err))
return nil, common.NewError("invalid_parameters", "Allocation could not be verified. "+err.Error())
@@ -58,6 +59,33 @@ func challengeHandler(ctx context.Context, r *http.Request) (interface{}, error)
return ValidValidationTicket(challengeObj, challengeRequest.ChallengeID, challengeHash)
}
+func requestAllocation(allocID string) (allocation *Allocation, err error) {
+ var b []byte
+ b, err = transaction.MakeSCRestAPICall(
+ transaction.STORAGE_CONTRACT_ADDRESS,
+ "/allocation",
+ map[string]string{"allocation": allocID})
+ if err != nil {
+ return
+ }
+ sa := new(transaction.StorageAllocation)
+ err = json.Unmarshal(b, sa)
+ if err != nil {
+ return
+ }
+ allocation = &Allocation{
+ ID: sa.ID,
+ DataShards: sa.DataShards,
+ ParityShards: sa.ParityShards,
+ Size: sa.Size,
+ Expiration: sa.Expiration,
+ Owner: sa.OwnerID,
+ OwnerPublicKey: sa.OwnerPublicKey,
+ OwnerSigningPublicKey: sa.OwnerSigningPublicKey,
+ }
+ return
+}
+
func NewChallengeRequest(r *http.Request) (*ChallengeRequest, string, error) {
if r.Method == "GET" {
return nil, "", common.NewError("invalid_method", "Invalid method used for the upload URL. Use multi-part form POST instead")
diff --git a/code/go/0chain.net/validatorcore/storage/context.go b/code/go/0chain.net/validatorcore/storage/context.go
index 86a2acb5d..7b8449ea0 100644
--- a/code/go/0chain.net/validatorcore/storage/context.go
+++ b/code/go/0chain.net/validatorcore/storage/context.go
@@ -5,7 +5,7 @@ import (
"net/http"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
)
func SetupContext(handler common.JSONResponderF) common.JSONResponderF {
diff --git a/code/go/0chain.net/validatorcore/storage/handler_integration_tests.go b/code/go/0chain.net/validatorcore/storage/handler_integration_tests.go
index 6fb11006b..a4fa31ad7 100644
--- a/code/go/0chain.net/validatorcore/storage/handler_integration_tests.go
+++ b/code/go/0chain.net/validatorcore/storage/handler_integration_tests.go
@@ -8,7 +8,7 @@ import (
"net/http"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/gosdk/constants"
+ "github.com/0chain/gosdk_common/constants"
"github.com/gorilla/mux"
)
diff --git a/code/go/0chain.net/validatorcore/storage/models.go b/code/go/0chain.net/validatorcore/storage/models.go
index ba47ef2f8..00746d2c9 100644
--- a/code/go/0chain.net/validatorcore/storage/models.go
+++ b/code/go/0chain.net/validatorcore/storage/models.go
@@ -11,7 +11,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
"github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage/writemarker"
- "github.com/0chain/gosdk/core/util"
+ "github.com/0chain/gosdk_common/core/util"
"github.com/mitchellh/mapstructure"
@@ -263,14 +263,15 @@ func (op *ObjectPath) Verify(allocationID string, challengeRand int64) error {
}
type Allocation struct {
- ID string `json:"id"`
- DataShards int `json:"data_shards"`
- ParityShards int `json:"parity_shards"`
- Size int64 `json:"size"`
- UsedSize int64 `json:"used_size"`
- Expiration common.Timestamp `json:"expiration_date"`
- Owner string `json:"owner_id"`
- OwnerPublicKey string `json:"owner_public_key"`
+ ID string `json:"id"`
+ DataShards int64 `json:"data_shards"`
+ ParityShards int64 `json:"parity_shards"`
+ Size int64 `json:"size"`
+ UsedSize int64 `json:"used_size"`
+ Expiration common.Timestamp `json:"expiration_date"`
+ Owner string `json:"owner_id"`
+ OwnerPublicKey string `json:"owner_public_key"`
+ OwnerSigningPublicKey string `json:"owner_signing_public_key"`
}
type ChallengeProof struct {
diff --git a/code/go/0chain.net/validatorcore/storage/models_test.go b/code/go/0chain.net/validatorcore/storage/models_test.go
index 221053cdb..eef445acb 100644
--- a/code/go/0chain.net/validatorcore/storage/models_test.go
+++ b/code/go/0chain.net/validatorcore/storage/models_test.go
@@ -10,8 +10,8 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/node"
"github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage"
- "github.com/0chain/gosdk/core/util"
- "github.com/0chain/gosdk/core/zcncrypto"
+ "github.com/0chain/gosdk_common/core/util"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
diff --git a/code/go/0chain.net/validatorcore/storage/protocol.go b/code/go/0chain.net/validatorcore/storage/protocol.go
index 521d3ee01..1871a008a 100644
--- a/code/go/0chain.net/validatorcore/storage/protocol.go
+++ b/code/go/0chain.net/validatorcore/storage/protocol.go
@@ -4,17 +4,13 @@ import (
"context"
"encoding/json"
"sync"
- "time"
"github.com/0chain/blobber/code/go/0chain.net/core/chain"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
- "github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/blobber/code/go/0chain.net/core/node"
"github.com/0chain/blobber/code/go/0chain.net/core/transaction"
- "github.com/0chain/blobber/code/go/0chain.net/validatorcore/config"
- "github.com/0chain/gosdk/constants"
- "go.uber.org/zap"
+ "github.com/0chain/gosdk_common/constants"
)
const CHUNK_SIZE = 64 * 1024
@@ -67,19 +63,6 @@ func GetProtocolImpl() *ValidatorProtocolImpl {
// return txn.Hash, nil
// }
-func (sp *ValidatorProtocolImpl) VerifyAllocationTransaction(ctx context.Context, allocationID string) (*Allocation, error) {
- t, err := transaction.VerifyTransaction(allocationID, sp.ServerChain)
- if err != nil {
- return nil, common.NewError("invalid_allocation", "Invalid Allocation id. Allocation not found in blockchain. "+err.Error())
- }
- var allocationObj Allocation
- err = json.Unmarshal([]byte(t.TransactionOutput), &allocationObj)
- if err != nil {
- return nil, common.NewError("transaction_output_decode_error", "Error decoding the allocation transaction output."+err.Error())
- }
- return &allocationObj, nil
-}
-
func (sp *ValidatorProtocolImpl) VerifyChallengeTransaction(ctx context.Context, challengeRequest *ChallengeRequest) (*Challenge, error) {
blobberID := ctx.Value(constants.ContextKeyClient).(string)
if blobberID == "" {
@@ -125,28 +108,3 @@ func (wb *WalletCallback) OnWalletCreateComplete(status int, wallet, err string)
wb.err = err
wb.wg.Done()
}
-
-func (sp *ValidatorProtocolImpl) RegisterValidator(ctx context.Context) (*transaction.Transaction, error) {
- time.Sleep(transaction.SLEEP_FOR_TXN_CONFIRMATION * time.Second)
-
- txn, err := transaction.NewTransactionEntity()
- if err != nil {
- return nil, err
- }
-
- sn := &transaction.StorageNode{}
- sn.ID = node.Self.ID
- sn.BaseURL = node.Self.GetURLBase()
- sn.StakePoolSettings.DelegateWallet = config.Configuration.DelegateWallet
- sn.StakePoolSettings.NumDelegates = config.Configuration.NumDelegates
- sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge
-
- logging.Logger.Info("Adding validator to the blockchain.")
- err = txn.ExecuteSmartContract(transaction.STORAGE_CONTRACT_ADDRESS, transaction.ADD_VALIDATOR_SC_NAME, sn, 0)
- if err != nil {
- logging.Logger.Info("Failed during registering validator to the mining network", zap.String("err:", err.Error()))
- return nil, err
- }
-
- return txn, nil
-}
diff --git a/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go b/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go
index 7f02f46a1..86a4ef3bf 100644
--- a/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go
+++ b/code/go/0chain.net/validatorcore/storage/writemarker/entity_test.go
@@ -9,7 +9,7 @@ import (
"github.com/0chain/blobber/code/go/0chain.net/core/encryption"
"github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage/writemarker"
- "github.com/0chain/gosdk/core/zcncrypto"
+ "github.com/0chain/gosdk_common/core/zcncrypto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml
index 55b5bcf37..758bab823 100755
--- a/config/0chain_blobber.yaml
+++ b/config/0chain_blobber.yaml
@@ -76,11 +76,13 @@ rate_limiters:
# Max upload limit in a month for a client. Default is 2000GB(the value needs to be in blocks which is data/64KB)
upload_limit_monthly: 31250000
# Max commit limit in a month for a client. Default is 30000
- commit_limit_monthly: 30000
+ commit_limit_monthly: 1000000000
# Max commit limit in a day for a client. Default is 1600
commit_limit_daily: 1600
# Max commit limit with size zero or less in a day for a client. Default is 400
commit_zero_limit_daily: 400
+ # Max connection changes in a batch. Default is 100
+ max_connection_changes: 100
server_chain:
id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe"
diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile
index ae00982f2..f5d3dc6bf 100644
--- a/docker.local/base.Dockerfile
+++ b/docker.local/base.Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine3.18 as blobber_base
+FROM golang:1.22-alpine3.18 as blobber_base
LABEL zchain="blobber"
diff --git a/go.mod b/go.mod
index e747651f4..93195d1f6 100644
--- a/go.mod
+++ b/go.mod
@@ -1,10 +1,12 @@
module github.com/0chain/blobber
-go 1.21
+go 1.22.0
+
+toolchain go1.23.1
require (
github.com/0chain/errors v1.0.3
- github.com/0chain/gosdk v1.17.0-RC6
+ github.com/0chain/gosdk v1.19.6
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/didip/tollbooth/v6 v6.1.2
github.com/go-openapi/runtime v0.26.0
@@ -19,26 +21,27 @@ require (
github.com/remeh/sizedwaitgroup v1.0.0
github.com/selvatico/go-mocket v1.0.7
github.com/spf13/viper v1.16.0
- github.com/stretchr/testify v1.8.4
- go.uber.org/zap v1.24.0
- golang.org/x/crypto v0.17.0
- golang.org/x/net v0.19.0 // indirect
- golang.org/x/sys v0.15.0
+ github.com/stretchr/testify v1.9.0
+ go.uber.org/zap v1.27.0
+ golang.org/x/crypto v0.21.0
+ golang.org/x/net v0.23.0 // indirect
+ golang.org/x/sys v0.18.0
golang.org/x/time v0.3.0 // indirect
google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect
google.golang.org/grpc v1.56.2
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0
- google.golang.org/protobuf v1.31.0
+ google.golang.org/protobuf v1.33.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gorm.io/datatypes v1.2.0
gorm.io/driver/postgres v1.5.2
gorm.io/driver/sqlite v1.5.2
- gorm.io/gorm v1.25.5
+ gorm.io/gorm v1.25.11
)
require (
+ github.com/0chain/gosdk_common v0.0.0-20250318142402-b7bd1ebd4f66
github.com/lithammer/shortuuid/v3 v3.0.7
- golang.org/x/sync v0.5.0
+ golang.org/x/sync v0.7.0
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc
)
@@ -53,6 +56,7 @@ require (
github.com/aws/aws-sdk-go-v2/config v1.26.1
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.5
github.com/pressly/goose/v3 v3.13.4
+ moul.io/zapgorm2 v1.3.0
)
require (
@@ -68,15 +72,17 @@ require (
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect
github.com/aws/smithy-go v1.19.0 // indirect
+ github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/hack-pad/go-webworkers v0.1.0 // indirect
github.com/hack-pad/safejs v0.1.1 // indirect
- github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 // indirect
+ github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.51.0 // indirect
+ github.com/x448/float16 v0.8.4 // indirect
)
require (
- github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565
+ github.com/0chain/common v1.18.3
github.com/Luzifer/go-openssl/v3 v3.1.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/btcsuite/btcd v0.23.4 // indirect
@@ -148,9 +154,8 @@ require (
go.dedis.ch/fixbuf v1.0.3 // indirect
go.dedis.ch/kyber/v3 v3.1.0 // indirect
go.mongodb.org/mongo-driver v1.11.3 // indirect
- go.uber.org/atomic v1.10.0 // indirect
- go.uber.org/multierr v1.10.0 // indirect
- golang.org/x/text v0.14.0 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
+ golang.org/x/text v0.16.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
diff --git a/go.sum b/go.sum
index a4f5c3b19..c052bf306 100644
--- a/go.sum
+++ b/go.sum
@@ -36,12 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEsT2XtRu4X7GfBiMnz9dYvWYs9V0B4=
-github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E=
+github.com/0chain/common v1.18.3 h1:42dYOv2KyMTSanuS67iDtfv+ErbSRqR8NJ3MG72MwaI=
+github.com/0chain/common v1.18.3/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw=
github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
-github.com/0chain/gosdk v1.17.0-RC6 h1:NB4pfu0VbVWfQQEfnFqn5inaiQuyhSaUW7VwgH3LW8U=
-github.com/0chain/gosdk v1.17.0-RC6/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg=
+github.com/0chain/gosdk v1.19.6 h1:GfNdBBC0uQqaooT4Vh6B7FP1m76lyTagugizwfYYleQ=
+github.com/0chain/gosdk v1.19.6/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw=
+github.com/0chain/gosdk_common v0.0.0-20250318142402-b7bd1ebd4f66 h1:lDzI8ZW/mdqfT0ySRnNke2c5KoqVh+FNXY8536yn0DA=
+github.com/0chain/gosdk_common v0.0.0-20250318142402-b7bd1ebd4f66/go.mod h1:+FBmkG4JocGmD+8gyE4kyst15p+3xm3Nf+nrg4cu5V4=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
@@ -107,7 +109,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNIC
github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
-github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -148,7 +149,6 @@ github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -163,14 +163,18 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8=
-github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk=
+github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
+github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
+github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
+github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk=
-github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM=
-github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
-github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
+github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
+github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
+github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -230,12 +234,14 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
-github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
-github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
+github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
+github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
@@ -485,8 +491,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/herumi/bls-go-binary v1.31.0 h1:L1goQ2tMtGgpXCg5AwHAdJQpLs/pfnWWEc3Wog6OhmI=
github.com/herumi/bls-go-binary v1.31.0/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y=
-github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 h1:Z6Mu2JCsW2hbqx91L0HNPRPQ10RyAFvPocQHlrRo1Jk=
-github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk=
+github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c h1:lDSIbcLu5TdT+uwb4wPzZgo1pQvKjP/tArL5QKjDJdI=
+github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk=
github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8=
@@ -783,8 +789,9 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -794,8 +801,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
@@ -827,6 +834,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
@@ -840,6 +849,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs=
@@ -870,23 +880,24 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
-go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
-go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -904,8 +915,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -917,8 +928,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
+golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -942,8 +953,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -987,11 +999,12 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1013,8 +1026,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1073,17 +1086,19 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1094,8 +1109,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1164,8 +1179,9 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg=
-golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1281,8 +1297,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1330,9 +1346,10 @@ gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc=
gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0=
gorm.io/driver/sqlserver v1.4.1/go.mod h1:DJ4P+MeZbc5rvY58PnmN1Lnyvb5gw5NPzGshHDnJLig=
+gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
-gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
-gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
+gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -1361,6 +1378,8 @@ modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk=
+moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs=
nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql
index 950dd0e6a..9496a21b4 100644
--- a/goose/migrations/1698861371_full_db_snapshot.sql
+++ b/goose/migrations/1698861371_full_db_snapshot.sql
@@ -103,7 +103,10 @@ CREATE TABLE allocations (
cleaned_up boolean DEFAULT false NOT NULL,
finalized boolean DEFAULT false NOT NULL,
file_options integer DEFAULT 63 NOT NULL,
- start_time bigint NOT NULL
+ start_time bigint NOT NULL,
+ allocation_version bigint DEFAULT 0 NOT NULL,
+ prev_used_size bigint DEFAULT 0 NOT NULL,
+ prev_blobber_size_used bigint DEFAULT 0 NOT NULL
);
@@ -329,7 +332,6 @@ ALTER TABLE read_pools OWNER TO blobber_user;
CREATE TABLE reference_objects (
id bigint NOT NULL,
- file_id text,
type character varying(1),
allocation_id character varying(64) NOT NULL,
lookup_hash character varying(64) NOT NULL,
@@ -337,25 +339,17 @@ CREATE TABLE reference_objects (
thumbnail_filename text,
path character varying(1000) NOT NULL COLLATE pg_catalog."POSIX",
file_meta_hash character varying(64) NOT NULL,
- hash character varying(64) NOT NULL,
num_of_blocks bigint DEFAULT 0 NOT NULL,
- path_hash character varying(64) NOT NULL,
parent_path character varying(999),
level bigint DEFAULT 0 NOT NULL,
custom_meta text NOT NULL,
- validation_root character varying(64) NOT NULL,
- prev_validation_root text,
- validation_root_signature character varying(64),
size bigint DEFAULT 0 NOT NULL,
- fixed_merkle_root character varying(64) NOT NULL,
actual_file_size bigint DEFAULT 0 NOT NULL,
actual_file_hash_signature character varying(64),
actual_file_hash character varying(64) NOT NULL,
mimetype character varying(255) NOT NULL,
- allocation_root character varying(64) NOT NULL,
thumbnail_size bigint DEFAULT 0 NOT NULL,
thumbnail_hash character varying(64) NOT NULL,
- prev_thumbnail_hash text,
actual_thumbnail_size bigint DEFAULT 0 NOT NULL,
actual_thumbnail_hash character varying(64) NOT NULL,
encrypted_key character varying(64),
@@ -363,10 +357,13 @@ CREATE TABLE reference_objects (
created_at bigint,
updated_at bigint,
deleted_at timestamp with time zone,
- is_precommit boolean DEFAULT false NOT NULL,
chunk_size bigint DEFAULT 65536 NOT NULL,
num_of_updates bigint,
- num_of_block_downloads bigint
+ num_of_block_downloads bigint,
+ data_hash character varying(64),
+ data_hash_signature character varying(64),
+ parent_id bigint DEFAULT NULL,
+ allocation_version bigint DEFAULT 0 NOT NULL
);
@@ -575,8 +572,6 @@ ALTER TABLE ONLY marketplace_share_info ALTER COLUMN id SET DEFAULT nextval('mar
ALTER TABLE ONLY reference_objects ALTER COLUMN id SET DEFAULT nextval('reference_objects_id_seq'::regclass);
-ALTER TABLE ONLY reference_objects ADD CONSTRAINT path_commit UNIQUE(lookup_hash,is_precommit);
-
--
-- Name: terms id; Type: DEFAULT; Schema: public; Owner: blobber_user
--
@@ -759,7 +754,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC);
-- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user
--
-CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash);
+CREATE UNIQUE INDEX idx_lookup_hash_deleted ON reference_objects USING btree (lookup_hash,(deleted_at IS NULL)) INCLUDE(id,type,num_of_updates);
--
@@ -787,14 +782,20 @@ CREATE INDEX idx_name_gin ON reference_objects USING gin (to_tsvector('english':
-- Name: idx_parent_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user
--
-CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path);
+-- CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path) WHERE deleted_at IS NULL;
+
+--
+-- Name: idx_parent_id; Type: INDEX; Schema: public; Owner: blobber_user
+--
+
+CREATE INDEX idx_parent_id ON reference_objects USING btree (parent_id);
--
-- Name: idx_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user
--
-CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path);
+CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path) WHERE deleted_at IS NULL;
--
@@ -839,13 +840,6 @@ CREATE INDEX idx_updated_at ON reference_objects USING btree (updated_at DESC);
CREATE INDEX idx_write_pools_cab ON write_pools USING btree (allocation_id);
---
--- Name: path_idx; Type: INDEX; Schema: public; Owner: blobber_user
---
-
-CREATE INDEX path_idx ON reference_objects USING btree (path);
-
-
--
-- Name: allocation_changes fk_allocation_connections_changes; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user
--
@@ -855,18 +849,18 @@ ALTER TABLE ONLY allocation_changes
--
- -- Name: connection_id_index; Type: INDEX; Schema: public; Owner: blobber_user
- --
+ -- Name: fk_reference_objects; TYPE FK CONSTRAINT; Schema: public; Owner: blobber_user
+ --
-CREATE INDEX connection_id_index ON allocation_changes USING btree (connection_id);
+ ALTER TABLE ONLY reference_objects
+ ADD CONSTRAINT fk_reference_objects FOREIGN KEY (parent_id) REFERENCES reference_objects(id) ON DELETE CASCADE;
---
--- Name: file_stats fk_file_stats_ref; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user
---
+ --
+ -- Name: connection_id_index; Type: INDEX; Schema: public; Owner: blobber_user
+ --
-ALTER TABLE ONLY file_stats
- ADD CONSTRAINT fk_file_stats_ref FOREIGN KEY (ref_id) REFERENCES reference_objects(id) ON DELETE CASCADE;
+CREATE INDEX connection_id_index ON allocation_changes USING btree (connection_id);
--
diff --git a/goose/migrations/1717416291_change_lookuphash.sql b/goose/migrations/1717416291_change_lookuphash.sql
index e2d8e71dd..f55ac0827 100644
--- a/goose/migrations/1717416291_change_lookuphash.sql
+++ b/goose/migrations/1717416291_change_lookuphash.sql
@@ -1,6 +1,6 @@
-- +goose Up
-- +goose StatementBegin
-ALTER TABLE allocation_changes ADD COLUMN lookup_hash character varying(64);
+ALTER TABLE allocation_changes ADD COLUMN lookup_hash character varying(64);
-- CREATE UNIQUE INDEX idx_allocation_changes_lookup_hash ON allocation_changes USING HASH(lookup_hash,connection_id);
-- +goose StatementEnd
\ No newline at end of file
diff --git a/goose/migrations/1718188301_change_idx.sql b/goose/migrations/1718188301_change_idx.sql
index 564e37891..14e6c3ba0 100644
--- a/goose/migrations/1718188301_change_idx.sql
+++ b/goose/migrations/1718188301_change_idx.sql
@@ -1,5 +1,10 @@
-- +goose Up
-- +goose StatementBegin
-CREATE INDEX idx_allocation_changes_lookup_hash ON allocation_changes USING HASH(lookup_hash);
+ --
+ -- Name: connection_id_lookup_hash; Type: UNIQUE CONSTRAINT; Schema: public; Owner: blobber_user
+ --
+
+ALTER TABLE ONLY allocation_changes ADD CONSTRAINT connection_id_lookup_hash UNIQUE(connection_id,lookup_hash);
+
-- +goose StatementEnd
\ No newline at end of file
diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql
index 4380bc232..6890115ea 100644
--- a/goose/migrations/1718391849_ref_index.sql
+++ b/goose/migrations/1718391849_ref_index.sql
@@ -1,4 +1,10 @@
-- +goose Up
-- +goose StatementBegin
-DROP INDEX idx_created_at,idx_updated_at,idx_lookup_hash,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash;
+DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin;
+
+CREATE INDEX idx_is_allocation_version_deleted_at on reference_objects(allocation_id,allocation_version) WHERE type='f' AND deleted_at IS NULL;
+
+CREATE INDEX idx_is_deleted on reference_objects(allocation_id) WHERE deleted_at IS NOT NULL;
+
+CREATE INDEX idx_path_alloc_level ON reference_objects USING btree (allocation_id,level,type,path) WHERE deleted_at IS NULL;
-- +goose StatementEnd
\ No newline at end of file
diff --git a/goose/migrations/1721021811_version_marker.sql b/goose/migrations/1721021811_version_marker.sql
new file mode 100644
index 000000000..dcf563a9a
--- /dev/null
+++ b/goose/migrations/1721021811_version_marker.sql
@@ -0,0 +1,61 @@
+-- +goose Up
+-- +goose StatementBegin
+
+CREATE TABLE version_markers(
+ id bigint NOT NULL,
+ allocation_id character varying(64) NOT NULL,
+ blobber_id character varying(64) NOT NULL,
+ client_id character varying(64) NOT NULL,
+ "version" bigint NOT NULL,
+ "timestamp" bigint NOT NULL,
+ signature character varying(64),
+ is_repair boolean NOT NULL DEFAULT false,
+ repair_version bigint,
+ repair_offset character varying(1000)
+);
+
+ALTER TABLE version_markers OWNER TO blobber_user;
+
+--
+-- Name: version_markers_id_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user
+--
+
+CREATE SEQUENCE version_markers_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER TABLE version_markers_id_seq OWNER TO blobber_user;
+
+
+--
+-- Name: version_markers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user
+--
+
+ALTER SEQUENCE version_markers_id_seq OWNED BY version_markers.id;
+
+
+--
+-- Name: version_markers version_markers_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user
+--
+
+ALTER TABLE ONLY version_markers
+ ADD CONSTRAINT version_markers_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: version_markers id; Type: DEFAULT; Schema: public; Owner: blobber_user
+--
+
+ALTER TABLE ONLY version_markers ALTER COLUMN id SET DEFAULT nextval('version_markers_id_seq'::regclass);
+
+
+--
+-- Name: version_markers_allocation_id_idx; Type: INDEX; Schema: public; Owner: blobber_user
+--
+
+CREATE INDEX version_markers_allocation_id_idx ON version_markers USING btree (allocation_id,version);
+
+-- +goose StatementEnd