Skip to content

Commit 6031712

Browse files
Hitenjain14Hitenjain14
committed
implement challenge for v2
1 parent 68f2601 commit 6031712

File tree

8 files changed

+332
-134
lines changed

8 files changed

+332
-134
lines changed

‎code/go/0chain.net/blobbercore/allocation/entity.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ var pendingMapLock = common.GetNewLocker()
3232

3333
const (
3434
TableNameAllocation = "allocations"
35+
StorageV2 = 1
3536
CanUploadMask = uint16(1) // 0000 0001
3637
CanDeleteMask = uint16(2) // 0000 0010
3738
CanUpdateMask = uint16(4) // 0000 0100
@@ -127,6 +128,10 @@ func (a *Allocation) CanRename() bool {
127128
return (a.FileOptions & CanRenameMask) > 0
128129
}
129130

131+
func (a *Allocation) IsStorageV2() bool {
132+
return a.StorageVersion == StorageV2
133+
}
134+
130135
// RestDurationInTimeUnits returns number (float point) of time units until
131136
// allocation ends.
132137
func (a *Allocation) RestDurationInTimeUnits(wmt common.Timestamp) (rdtu float64) {

‎code/go/0chain.net/blobbercore/challenge/protocol.go

Lines changed: 208 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package challenge
22

33
import (
44
"context"
5+
"encoding/hex"
56
"encoding/json"
67
"errors"
78
"math/rand"
@@ -99,45 +100,22 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error {
99100
return common.NewError("write_marker_not_found", "Could find the writemarker for the given allocation root on challenge")
100101
}
101102

102-
rootRef, err := reference.GetReference(ctx, cr.AllocationID, "/")
103-
if err != nil && err != gorm.ErrRecordNotFound {
104-
allocMu.RUnlock()
103+
var (
104+
postData map[string]any
105+
)
106+
107+
if allocationObj.IsStorageV2() {
108+
postData, err = cr.getPostDataV2(ctx, allocationObj)
109+
} else {
110+
postData, err = cr.getPostData(ctx, allocationObj)
111+
}
112+
allocMu.RUnlock()
113+
if err != nil {
114+
logging.Logger.Error("[challenge]load: ", zap.String("challenge_id", cr.ChallengeID), zap.Error(err))
105115
cr.CancelChallenge(ctx, err)
106116
return err
107117
}
108-
109-
blockNum := int64(0)
110-
var objectPath *reference.ObjectPath
111-
if rootRef != nil {
112-
if rootRef.Hash != allocationObj.AllocationRoot {
113-
logging.Logger.Error("root_mismatch", zap.Any("allocation_root", allocationObj.AllocationRoot), zap.Any("latest_write_marker", wms[len(wms)-1].WM.AllocationRoot), zap.Any("root_ref_hash", rootRef.Hash))
114-
}
115-
if rootRef.NumBlocks > 0 {
116-
r := rand.New(rand.NewSource(cr.RandomNumber))
117-
blockNum = r.Int63n(rootRef.NumBlocks)
118-
blockNum++
119-
cr.BlockNum = blockNum
120-
}
121-
122-
logging.Logger.Info("[challenge]rand: ", zap.Any("rootRef.NumBlocks", rootRef.NumBlocks), zap.Any("blockNum", blockNum), zap.Any("challenge_id", cr.ChallengeID), zap.Any("random_seed", cr.RandomNumber))
123-
objectPath, err = reference.GetObjectPath(ctx, cr.AllocationID, blockNum)
124-
if err != nil {
125-
allocMu.RUnlock()
126-
cr.CancelChallenge(ctx, err)
127-
return err
128-
}
129-
if objectPath != nil {
130-
cr.RefID = objectPath.RefID
131-
cr.ObjectPath = objectPath
132-
}
133-
}
134-
cr.RespondedAllocationRoot = allocationObj.AllocationRoot
135-
136-
postData := make(map[string]interface{})
137118
postData["challenge_id"] = cr.ChallengeID
138-
if objectPath != nil {
139-
postData["object_path"] = objectPath
140-
}
141119
markersArray := make([]map[string]interface{}, 0)
142120
for _, wm := range wms {
143121
markersMap := make(map[string]interface{})
@@ -146,81 +124,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error {
146124
markersArray = append(markersArray, markersMap)
147125
}
148126
postData["write_markers"] = markersArray
149-
150-
var proofGenTime int64 = -1
151-
152-
if blockNum > 0 {
153-
if objectPath.Meta["type"] != reference.FILE {
154-
allocMu.RUnlock()
155-
logging.Logger.Info("Block number to be challenged for file:", zap.Any("block", objectPath.FileBlockNum), zap.Any("meta", objectPath.Meta), zap.Any("obejct_path", objectPath))
156-
157-
cr.CancelChallenge(ctx, ErrInvalidObjectPath)
158-
return ErrInvalidObjectPath
159-
}
160-
161-
r := rand.New(rand.NewSource(cr.RandomNumber))
162-
blockoffset := r.Intn(sdkUtil.FixedMerkleLeaves)
163-
164-
fromPreCommit := true
165-
166-
if objectPath.Meta["is_precommit"] != nil {
167-
fromPreCommit = objectPath.Meta["is_precommit"].(bool)
168-
if fromPreCommit {
169-
fromPreCommit = objectPath.Meta["validation_root"].(string) != objectPath.Meta["prev_validation_root"].(string)
170-
}
171-
} else {
172-
logging.Logger.Error("is_precommit_is_nil", zap.Any("object_path", objectPath))
173-
}
174-
175-
challengeReadInput := &filestore.ChallengeReadBlockInput{
176-
Hash: objectPath.Meta["validation_root"].(string),
177-
FileSize: objectPath.Meta["size"].(int64),
178-
BlockOffset: blockoffset,
179-
AllocationID: cr.AllocationID,
180-
IsPrecommit: fromPreCommit,
181-
FilestoreVersion: objectPath.FilestoreVersion,
182-
}
183-
184-
t1 := time.Now()
185-
challengeResponse, err := filestore.GetFileStore().GetBlocksMerkleTreeForChallenge(challengeReadInput)
186-
187-
if err != nil {
188-
allocMu.RUnlock()
189-
cr.CancelChallenge(ctx, err)
190-
return common.NewError("blockdata_not_found", err.Error())
191-
}
192-
proofGenTime = time.Since(t1).Milliseconds()
193-
194-
if objectPath.Meta["size"] != nil {
195-
logging.Logger.Info("Proof gen logs: ",
196-
zap.Int64("block num", blockNum),
197-
zap.Int64("file size", objectPath.Meta["size"].(int64)),
198-
zap.String("file path", objectPath.Meta["name"].(string)),
199-
zap.Int64("proof gen time", proofGenTime),
200-
)
201-
}
202-
postData["challenge_proof"] = challengeResponse
203-
}
204-
205-
if objectPath == nil {
206-
objectPath = &reference.ObjectPath{}
207-
}
208-
err = UpdateChallengeTimingProofGenerationAndFileSize(
209-
cr.ChallengeID,
210-
proofGenTime,
211-
objectPath.Size,
212-
)
213-
if err != nil {
214-
logging.Logger.Error("[challengetiming]txnverification",
215-
zap.Any("challenge_id", cr.ChallengeID),
216-
zap.Time("created", common.ToTime(cr.CreatedAt)),
217-
zap.Int64("proof_gen_time", int64(proofGenTime)),
218-
zap.Error(err))
219-
220-
allocMu.RUnlock()
221-
return err
222-
}
223-
allocMu.RUnlock()
127+
postData["storage_version"] = allocationObj.StorageVersion
224128

225129
postDataBytes, err := json.Marshal(postData)
226130
logging.Logger.Info("[challenge]post: ", zap.Any("challenge_id", cr.ChallengeID), zap.Any("post_data_len", len(postDataBytes)/(1024*1024)))
@@ -432,3 +336,197 @@ func (cr *ChallengeEntity) SaveChallengeResult(ctx context.Context, t *transacti
432336
zap.Error(err))
433337
}
434338
}
339+
340+
func (cr *ChallengeEntity) getPostDataV2(ctx context.Context, allocationObj *allocation.Allocation) (map[string]any, error) {
341+
trie := allocationObj.GetTrie()
342+
var (
343+
blockNum = int64(0)
344+
postData = make(map[string]interface{})
345+
ref *reference.Ref
346+
objectSize int64
347+
proofGenTime int64 = -1
348+
)
349+
if trie.Weight() > 0 {
350+
r := rand.New(rand.NewSource(cr.RandomNumber))
351+
blockNum = r.Int63n(int64(trie.Weight()))
352+
blockNum++
353+
cr.BlockNum = blockNum
354+
355+
logging.Logger.Info("[challenge]rand: ", zap.Uint64("trie.NumBlocks", trie.Weight()), zap.Any("blockNum", blockNum), zap.Any("challenge_id", cr.ChallengeID), zap.Any("random_seed", cr.RandomNumber))
356+
key, objectProof, err := trie.GetBlockProof(uint64(blockNum))
357+
if err != nil {
358+
return nil, err
359+
}
360+
lookupHash := hex.EncodeToString(key)
361+
ref, err = reference.GetReferenceByLookupHash(ctx, cr.AllocationID, lookupHash)
362+
if err != nil {
363+
return nil, err
364+
}
365+
if ref.Type != reference.FILE {
366+
return nil, ErrInvalidObjectPath
367+
}
368+
cr.RefID = ref.ID
369+
postData["object_proof"] = objectProof
370+
}
371+
cr.RespondedAllocationRoot = allocationObj.AllocationRoot
372+
if blockNum > 0 {
373+
r := rand.New(rand.NewSource(cr.RandomNumber))
374+
blockoffset := r.Intn(sdkUtil.FixedMerkleLeaves)
375+
challengeReadInput := &filestore.ChallengeReadBlockInput{
376+
Hash: ref.LookupHash,
377+
FileSize: ref.Size,
378+
BlockOffset: blockoffset,
379+
AllocationID: cr.AllocationID,
380+
IsPrecommit: ref.AllocationRoot == allocationObj.AllocationRoot,
381+
FilestoreVersion: ref.FilestoreVersion,
382+
}
383+
t1 := time.Now()
384+
challengeResponse, err := filestore.GetFileStore().GetBlocksMerkleTreeForChallenge(challengeReadInput)
385+
if err != nil {
386+
return nil, err
387+
}
388+
proofGenTime := time.Since(t1).Milliseconds()
389+
logging.Logger.Info("Proof gen logs: ",
390+
zap.Int64("block num", blockNum),
391+
zap.Int64("file size", ref.Size),
392+
zap.String("file path", ref.Path),
393+
zap.Int64("proof gen time", proofGenTime),
394+
)
395+
postData["challenge_proof"] = challengeResponse
396+
objectSize = ref.Size
397+
metaRef := &reference.RefMeta{
398+
Path: ref.Path,
399+
LookupHash: ref.LookupHash,
400+
ActualFileSize: ref.ActualFileSize,
401+
ActualFileHashSignature: ref.ActualFileHashSignature,
402+
ActualFileHash: ref.ActualFileHash,
403+
ValidationRoot: ref.ValidationRoot,
404+
ValidationRootSignature: ref.ValidationRootSignature,
405+
FixedMerkleRoot: ref.FixedMerkleRoot,
406+
Size: ref.Size,
407+
FileMetaHash: ref.FileMetaHash,
408+
}
409+
postData["meta"] = metaRef
410+
}
411+
err := UpdateChallengeTimingProofGenerationAndFileSize(
412+
cr.ChallengeID,
413+
proofGenTime,
414+
objectSize,
415+
)
416+
if err != nil {
417+
logging.Logger.Error("[challengetiming]txnverification",
418+
zap.Any("challenge_id", cr.ChallengeID),
419+
zap.Time("created", common.ToTime(cr.CreatedAt)),
420+
zap.Int64("proof_gen_time", int64(proofGenTime)),
421+
zap.Error(err))
422+
423+
return nil, err
424+
}
425+
return postData, nil
426+
}
427+
428+
func (cr *ChallengeEntity) getPostData(ctx context.Context, allocationObj *allocation.Allocation) (map[string]any, error) {
429+
rootRef, err := reference.GetReference(ctx, cr.AllocationID, "/")
430+
if err != nil && err != gorm.ErrRecordNotFound {
431+
return nil, err
432+
}
433+
434+
blockNum := int64(0)
435+
var objectPath *reference.ObjectPath
436+
if rootRef != nil {
437+
if rootRef.Hash != allocationObj.AllocationRoot {
438+
logging.Logger.Error("root_mismatch", zap.Any("allocation_root", allocationObj.AllocationRoot), zap.Any("root_ref_hash", rootRef.Hash))
439+
}
440+
if rootRef.NumBlocks > 0 {
441+
r := rand.New(rand.NewSource(cr.RandomNumber))
442+
blockNum = r.Int63n(rootRef.NumBlocks)
443+
blockNum++
444+
cr.BlockNum = blockNum
445+
}
446+
447+
logging.Logger.Info("[challenge]rand: ", zap.Any("rootRef.NumBlocks", rootRef.NumBlocks), zap.Any("blockNum", blockNum), zap.Any("challenge_id", cr.ChallengeID), zap.Any("random_seed", cr.RandomNumber))
448+
objectPath, err = reference.GetObjectPath(ctx, cr.AllocationID, blockNum)
449+
if err != nil {
450+
return nil, err
451+
}
452+
if objectPath != nil {
453+
cr.RefID = objectPath.RefID
454+
cr.ObjectPath = objectPath
455+
}
456+
}
457+
cr.RespondedAllocationRoot = allocationObj.AllocationRoot
458+
459+
postData := make(map[string]interface{})
460+
if objectPath != nil {
461+
postData["object_path"] = objectPath
462+
}
463+
464+
var proofGenTime int64 = -1
465+
466+
if blockNum > 0 {
467+
if objectPath.Meta["type"] != reference.FILE {
468+
logging.Logger.Info("Block number to be challenged for file:", zap.Any("block", objectPath.FileBlockNum), zap.Any("meta", objectPath.Meta), zap.Any("obejct_path", objectPath))
469+
return nil, ErrInvalidObjectPath
470+
}
471+
472+
r := rand.New(rand.NewSource(cr.RandomNumber))
473+
blockoffset := r.Intn(sdkUtil.FixedMerkleLeaves)
474+
475+
fromPreCommit := true
476+
477+
if objectPath.Meta["is_precommit"] != nil {
478+
fromPreCommit = objectPath.Meta["is_precommit"].(bool)
479+
if fromPreCommit {
480+
fromPreCommit = objectPath.Meta["validation_root"].(string) != objectPath.Meta["prev_validation_root"].(string)
481+
}
482+
} else {
483+
logging.Logger.Error("is_precommit_is_nil", zap.Any("object_path", objectPath))
484+
}
485+
486+
challengeReadInput := &filestore.ChallengeReadBlockInput{
487+
Hash: objectPath.Meta["validation_root"].(string),
488+
FileSize: objectPath.Meta["size"].(int64),
489+
BlockOffset: blockoffset,
490+
AllocationID: cr.AllocationID,
491+
IsPrecommit: fromPreCommit,
492+
FilestoreVersion: objectPath.FilestoreVersion,
493+
}
494+
495+
t1 := time.Now()
496+
challengeResponse, err := filestore.GetFileStore().GetBlocksMerkleTreeForChallenge(challengeReadInput)
497+
498+
if err != nil {
499+
return nil, common.NewError("blockdata_not_found", err.Error())
500+
}
501+
proofGenTime = time.Since(t1).Milliseconds()
502+
503+
if objectPath.Meta["size"] != nil {
504+
logging.Logger.Info("Proof gen logs: ",
505+
zap.Int64("block num", blockNum),
506+
zap.Int64("file size", objectPath.Meta["size"].(int64)),
507+
zap.String("file path", objectPath.Meta["name"].(string)),
508+
zap.Int64("proof gen time", proofGenTime),
509+
)
510+
}
511+
postData["challenge_proof"] = challengeResponse
512+
}
513+
514+
if objectPath == nil {
515+
objectPath = &reference.ObjectPath{}
516+
}
517+
err = UpdateChallengeTimingProofGenerationAndFileSize(
518+
cr.ChallengeID,
519+
proofGenTime,
520+
objectPath.Size,
521+
)
522+
if err != nil {
523+
logging.Logger.Error("[challengetiming]txnverification",
524+
zap.Any("challenge_id", cr.ChallengeID),
525+
zap.Time("created", common.ToTime(cr.CreatedAt)),
526+
zap.Int64("proof_gen_time", int64(proofGenTime)),
527+
zap.Error(err))
528+
529+
return nil, err
530+
}
531+
return postData, nil
532+
}

‎code/go/0chain.net/blobbercore/handler/file_command_delete.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func (cmd *DeleteFileCommand) IsValidated(ctx context.Context, req *http.Request
5959
}
6060
return common.NewError("bad_db_operation", err.Error())
6161
}
62-
if allocationObj.StorageVersion == 1 && cmd.existingFileRef.Type == reference.DIRECTORY {
62+
if allocationObj.IsStorageV2() && cmd.existingFileRef.Type == reference.DIRECTORY {
6363
isEmpty, err := reference.IsDirectoryEmpty(ctx, allocationObj.ID, path)
6464
if err != nil {
6565
return err

‎code/go/0chain.net/blobbercore/handler/file_command_update.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj
151151
return result, common.NewError("upload_error", fmt.Sprintf("File size mismatch. Expected: %d, Actual: %d", cmd.fileChanger.Size, fileOutputData.ContentSize))
152152
}
153153
hash := cmd.fileChanger.ActualHash + cmd.fileChanger.ValidationRoot
154-
if allocationObj.StorageVersion == 1 {
154+
if allocationObj.IsStorageV2() {
155155
hashData := fmt.Sprintf("%s:%s:%s:%s", cmd.fileChanger.ActualHash, cmd.fileChanger.ValidationRoot, cmd.fileChanger.FixedMerkleRoot, node.Self.ID)
156156
hash = encryption.Hash(hashData)
157157
}

‎code/go/0chain.net/blobbercore/handler/file_command_upload.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj
163163
return result, common.NewError("upload_error", fmt.Sprintf("File size mismatch. Expected: %d, Actual: %d", cmd.fileChanger.Size, fileOutputData.ContentSize))
164164
}
165165
hash := cmd.fileChanger.ActualHash + cmd.fileChanger.ValidationRoot
166-
if allocationObj.StorageVersion == 1 {
166+
if allocationObj.IsStorageV2() {
167167
hashData := fmt.Sprintf("%s:%s:%s:%s", cmd.fileChanger.ActualHash, cmd.fileChanger.ValidationRoot, cmd.fileChanger.FixedMerkleRoot, node.Self.ID)
168168
hash = encryption.Hash(hashData)
169169
}

0 commit comments

Comments
 (0)