Merge pull request #549 from NethermindEth/lazytree

Lazytree

Author: tkstanczak

.travis.yml

@@ -26,6 +26,12 @@ jobs:
     - stage: run tests
       script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind/Ethereum.Basic.Test
       name: "Ethereum.Basic.Test"
+    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind.DataMarketplace.Consumers.Test
+      name: "Nethermind.DataMarketplace.Consumers.Test"
+    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind.DataMarketplace.Integration.Test
+      name: "Nethermind.DataMarketplace.Integration.Test"  
+    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind.DataMarketplace.Test
+      name: "Nethermind.DataMarketplace.Test"  
     - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind/Ethereum.Blockchain.Block.Test
       name: "Ethereum.Blockchain.Block.Test"
     - script: dotnet test src/Nethermind/Ethereum.Blockchain.Test

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -571,7 +571,7 @@ private static Block Convert(TestBlockJson testBlockJson)
             BlockHeader header = Convert(testBlockJson.BlockHeader);
             BlockHeader[] ommers = testBlockJson.UncleHeaders?.Select(Convert).ToArray() ?? new BlockHeader[0];
             Block block = new Block(header, ommers);
-            block.Transactions = testBlockJson.Transactions?.Select(Convert).ToArray();
+            block.Body.Transactions = testBlockJson.Transactions?.Select(Convert).ToArray();
             return block;
         }
 

src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs

@@ -167,7 +167,7 @@ public void Add_and_find_branch()
             BlockTree blockTree = BuildBlockTree();
             Block block = Build.A.Block.TestObject;
             blockTree.SuggestBlock(block);
-            Block found = blockTree.FindBlock(block.Hash, false);
+            Block found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.None);
             Assert.AreEqual(block.Hash, BlockHeader.CalculateHash(found.Header));
         }
 
@@ -177,7 +177,7 @@ public void Add_on_branch_move_find()
             BlockTree blockTree = BuildBlockTree();
             Block block = Build.A.Block.TestObject;
             AddToMain(blockTree, block);
-            Block found = blockTree.FindBlock(block.Hash, true);
+            Block found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.RequireCanonical);
             Assert.AreEqual(block.Hash, BlockHeader.CalculateHash(found.Header));
         }
 
@@ -187,7 +187,7 @@ public void Add_on_branch_and_not_find_on_main()
             BlockTree blockTree = BuildBlockTree();
             Block block = Build.A.Block.TestObject;
             blockTree.SuggestBlock(block);
-            Block found = blockTree.FindBlock(block.Hash, true);
+            Block found = blockTree.FindBlock(block.Hash, BlockTreeLookupOptions.RequireCanonical);
             Assert.IsNull(found);
         }
 
@@ -473,7 +473,7 @@ public void Stores_multiple_blocks_per_level()
             AddToMain(blockTree, block1);
             blockTree.SuggestBlock(block1B);
 
-            Block found = blockTree.FindBlock(block1B.Hash, false);
+            Block found = blockTree.FindBlock(block1B.Hash, BlockTreeLookupOptions.None);
 
             Assert.AreEqual(block1B.Hash, BlockHeader.CalculateHash(found.Header));
         }
@@ -1147,6 +1147,34 @@ public void Can_batch_insert_blocks()
 
             tree.Insert(blocks);
         }
+        
+        [Test]
+        public void Block_loading_is_lazy()
+        {
+            MemDb blocksDb = new MemDb();
+            MemDb blockInfosDb = new MemDb();
+            MemDb headersDb = new MemDb();
+            
+            SyncConfig syncConfig = new SyncConfig();
+            syncConfig.PivotNumber = 0L.ToString();
+
+            Block genesis = Build.A.Block.Genesis.TestObject;
+            BlockTree tree = new BlockTree(blocksDb, headersDb, blockInfosDb, MainNetSpecProvider.Instance, NullTxPool.Instance, syncConfig, LimboLogs.Instance);
+            tree.SuggestBlock(genesis);
+
+            Block previousBlock = genesis;
+            for (int i = 1; i < 10; i++)
+            {
+                Block block = Build.A.Block.WithNumber(i).WithParent(previousBlock).TestObject;
+                tree.SuggestBlock(block);
+                previousBlock = block;
+            }
+
+            Block lastBlock = previousBlock;
+            
+            BlockTree loadedTree = new BlockTree(blocksDb, headersDb, blockInfosDb, MainNetSpecProvider.Instance, NullTxPool.Instance, syncConfig, LimboLogs.Instance);
+            loadedTree.FindHeader(lastBlock.Hash, BlockTreeLookupOptions.None);
+        }
 
         static object[] SourceOfBSearchTestCases =
         {

src/Nethermind/Nethermind.Blockchain.Test/BlockchainProcessorTests.cs

@@ -348,7 +348,7 @@ public ProcessingTestContext IsDeletedAsInvalid()
                     _processingTestContext._resetEvent.WaitOne(IgnoreWait);
                     Assert.AreEqual(_processingTestContext._headBefore, _processingTestContext._blockTree.Head.Hash, "head");
                     _logger.Info($"Finished waiting for {_block.ToString(Block.Format.Short)} to be deleted");
-                    Assert.Null(_processingTestContext._blockTree.FindBlock(_block.Hash, false));
+                    Assert.Null(_processingTestContext._blockTree.FindBlock(_block.Hash, BlockTreeLookupOptions.None));
                     return _processingTestContext;
                 }
             }

src/Nethermind/Nethermind.Blockchain.Test/Synchronization/BlockDownloaderTests.cs

@@ -74,7 +74,7 @@ public async Task<BlockHeader[]> BuildHeaderResponse(long startNumber, int numbe
                     throw new TimeoutException();
                 }
 
-                BlockHeader startBlock = _blockTree.FindHeader(_testHeaderMapping[startNumber], false);
+                BlockHeader startBlock = _blockTree.FindHeader(_testHeaderMapping[startNumber], BlockTreeLookupOptions.None);
                 BlockHeader[] headers = new BlockHeader[number];
                 headers[0] = startBlock;
                 if (!justFirst)
@@ -116,7 +116,7 @@ public async Task<BlockBody[]> BuildBlocksResponse(Keccak[] blockHashes, Respons
                     throw new TimeoutException();
                 }
 
-                BlockHeader startHeader = _blockTree.FindHeader(blockHashes[0], false);
+                BlockHeader startHeader = _blockTree.FindHeader(blockHashes[0], BlockTreeLookupOptions.None);
                 if (startHeader == null) startHeader = Build.A.BlockHeader.WithHash(blockHashes[0]).TestObject;
 
                 BlockHeader[] blockHeaders = new BlockHeader[blockHashes.Length];

src/Nethermind/Nethermind.Blockchain.Test/Synchronization/FastBlocks/FastBlocksFeedTests.cs

@@ -655,15 +655,15 @@ private void AssertTreeSynced(IBlockTree tree, bool bodiesSync = false, bool rec
             Keccak nextHash = tree.Head.Hash;
             for (int i = 0; i < tree.Head.Number; i++)
             {
-                BlockHeader header = _localBlockTree.FindHeader(nextHash);
+                BlockHeader header = _localBlockTree.FindHeader(nextHash, BlockTreeLookupOptions.None);
                 Assert.NotNull(header, $"header {tree.Head.Number - i}");
                 if (bodiesSync)
                 {
-                    Block expectedBlock = _localBlockTree.FindBlock(nextHash, false);
+                    Block expectedBlock = _localBlockTree.FindBlock(nextHash, BlockTreeLookupOptions.None);
                     Assert.AreEqual(nextHash, expectedBlock?.Hash, $"hash difference {tree.Head.Number - i}");
                     if (expectedBlock != null)
                     {
-                        Block actualBlock = tree.FindBlock(expectedBlock.Hash, false);
+                        Block actualBlock = tree.FindBlock(expectedBlock.Hash, BlockTreeLookupOptions.None);
                         Rlp saved = Rlp.Encode(actualBlock);
                         Rlp expected = Rlp.Encode(expectedBlock);
                         Assert.AreEqual(expected, saved, $"body {tree.Head.Number - i}");
@@ -832,7 +832,7 @@ private void PrepareReceiptsResponse(ReceiptsSyncBatch receiptSyncBatch, Latency
             receiptSyncBatch.Response = new TxReceipt[receiptSyncBatch.Request.Length][];
             for (int i = 0; i < receiptSyncBatch.Request.Length; i++)
             {
-                Block block = tree.FindBlock(receiptSyncBatch.Request[i], false);
+                Block block = tree.FindBlock(receiptSyncBatch.Request[i], BlockTreeLookupOptions.None);
                 receiptSyncBatch.Response[i] = new TxReceipt[block.Transactions.Length];
                 for (int j = 0; j < block.Transactions.Length; j++)
                 {
@@ -861,7 +861,7 @@ private void PrepareBodiesResponse(BodiesSyncBatch bodiesSyncBatch, LatencySyncP
 
             for (int i = 0; i < Math.Min(maxResponseSize, requestSize); i++)
             {
-                Block block = tree.FindBlock(bodiesSyncBatch.Request[i], false);
+                Block block = tree.FindBlock(bodiesSyncBatch.Request[i], BlockTreeLookupOptions.None);
                 bodiesSyncBatch.Response[i] = new BlockBody(block.Transactions, block.Ommers);
             }
 

src/Nethermind/Nethermind.Blockchain.Test/Synchronization/OldStyleFullSynchronizerTests.cs

@@ -199,7 +199,7 @@ public void Can_sync_on_split_of_length_1()
 
             Assert.AreEqual(miner1Tree.BestSuggestedHeader.Hash, _blockTree.BestSuggestedHeader.Hash, "client agrees with miner before split");
 
-            Block splitBlock = Build.A.Block.WithParent(miner1Tree.FindParent(miner1Tree.Head)).WithDifficulty(miner1Tree.Head.Difficulty - 1).TestObject;
+            Block splitBlock = Build.A.Block.WithParent(miner1Tree.FindParent(miner1Tree.Head, BlockTreeLookupOptions.TotalDifficultyNotNeeded)).WithDifficulty(miner1Tree.Head.Difficulty - 1).TestObject;
             Block splitBlockChild = Build.A.Block.WithParent(splitBlock).TestObject;
 
             miner1Tree.SuggestBlock(splitBlock);

src/Nethermind/Nethermind.Blockchain.Test/Synchronization/SyncPeerMock.cs

@@ -85,7 +85,7 @@ public Task<BlockBody[]> GetBlocks(Keccak[] blockHashes, CancellationToken token
             BlockBody[] result = new BlockBody[blockHashes.Length];
             for (int i = 0; i < blockHashes.Length; i++)
             {
-                Block block = _remoteTree.FindBlock(blockHashes[i], true);
+                Block block = _remoteTree.FindBlock(blockHashes[i], BlockTreeLookupOptions.RequireCanonical);
                 result[i] = new BlockBody(block.Transactions, block.Ommers);
             }
 
@@ -95,7 +95,7 @@ public Task<BlockBody[]> GetBlocks(Keccak[] blockHashes, CancellationToken token
         public Task<BlockHeader[]> GetBlockHeaders(Keccak blockHash, int maxBlocks, int skip, CancellationToken token)
         {
             BlockHeader[] result = new BlockHeader[maxBlocks];
-            long? firstNumber = _remoteTree.FindHeader(blockHash, true)?.Number;
+            long? firstNumber = _remoteTree.FindHeader(blockHash, BlockTreeLookupOptions.RequireCanonical)?.Number;
             if (!firstNumber.HasValue)
             {
                 return Task.FromResult(result);

src/Nethermind/Nethermind.Blockchain/BlockExtensions.cs

@@ -28,16 +28,20 @@ public static class BlockExtensions
     {
         public static Keccak CalculateReceiptRoot(this Block block, ISpecProvider specProvider, TxReceipt[] txReceipts)
         {
-            PatriciaTree receiptTree = txReceipts.Length > 0 ? new PatriciaTree(NullDb.Instance, Keccak.EmptyTreeHash, false) : null;
+            if (txReceipts.Length == 0)
+            {
+                return PatriciaTree.EmptyTreeHash;
+            }
+            
+            PatriciaTree receiptTree = new PatriciaTree();
             for (int i = 0; i < txReceipts.Length; i++)
             {
                 Rlp receiptRlp = Rlp.Encode(txReceipts[i], specProvider.GetSpec(block.Number).IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None);
-                receiptTree?.Set(Rlp.Encode(i).Bytes, receiptRlp);
+                receiptTree.Set(Rlp.Encode(i).Bytes, receiptRlp);
             }
 
-            receiptTree?.UpdateRootHash();
-            Keccak receiptRoot = receiptTree?.RootHash ?? PatriciaTree.EmptyTreeHash;
-            return receiptRoot;
+            receiptTree.UpdateRootHash();
+            return receiptTree.RootHash;
         }
 
         public static Keccak CalculateTxRoot(this Block block)
@@ -60,7 +64,9 @@ public static Keccak CalculateTxRoot(this Block block)
 
         public static Keccak CalculateOmmersHash(this Block block)
         {
-            return Keccak.Compute(Rlp.Encode(block.Ommers));
+            return block.Ommers.Length == 0
+                ? Keccak.OfAnEmptySequenceRlp
+                : Keccak.Compute(Rlp.Encode(block.Ommers));
         }
     }
 }