core: add fixes

core/blockchain.go

@@ -1253,10 +1253,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 	return 0, nil
 }
 
-// AdjustTxLookupLimit is responsible for updating the txlookup limit
+// SetTxLookupLimit is responsible for updating the txlookup limit
 // to the original one stored in db if the new old mismatch with the old
 // one.
-func (bc *BlockChain) AdjustTxLookupLimit(limit uint64) {
+func (bc *BlockChain) SetTxLookupLimit(limit uint64) {
 	bc.txLookupLimit = limit
 }
 
@@ -2107,17 +2107,17 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) {
 			// This is a special case that user upgrades Geth to a new version
 			// which supports tx indices pruning feature but the tx index tail
 			// is missing. So that we can assume all blocks in db are indexed.
-			if bc.txLookupLimit == 0 || head <= bc.txLookupLimit {
+			if bc.txLookupLimit == 0 || head < bc.txLookupLimit {
 				// Nothing to delete, write the tail and return.
 				rawdb.WriteTxIndexTail(bc.db, 0)
 			} else {
 				// Prune all stale tx indices and record the tx index tail.
-				rawdb.RemoveTxsLookup(bc.db, 0, head-bc.txLookupLimit)
+				rawdb.RemoveTxsLookup(bc.db, 0, head-bc.txLookupLimit+1)
 			}
 			return
 		}
 		// All indices should be reserved.
-		if bc.txLookupLimit == 0 || head <= bc.txLookupLimit {
+		if bc.txLookupLimit == 0 || head < bc.txLookupLimit {
 			if *tail == 0 {
 				// Short circuit if nothing to delete.
 			} else {
@@ -2126,14 +2126,14 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) {
 			}
 			return
 		}
-		if head-bc.txLookupLimit < *tail {
+		if head-bc.txLookupLimit+1 < *tail {
 			// Reindex a part of missing indices and rewind oldest indexed
 			// point to HEAD-limit
-			rawdb.IndexTxLookup(bc.db, head-bc.txLookupLimit, *tail)
+			rawdb.IndexTxLookup(bc.db, head-bc.txLookupLimit+1, *tail)
 		} else {
 			// Unindex a part of stale indices and forward oldest indexed
 			// point to HEAD-limit
-			rawdb.RemoveTxsLookup(bc.db, *tail, head-bc.txLookupLimit)
+			rawdb.RemoveTxsLookup(bc.db, *tail, head-bc.txLookupLimit+1)
 		}
 	}
 	// Special case here: user might init Geth with an external ancient database.
@@ -2142,7 +2142,7 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) {
 	if ancients > 0 {
 		var from = uint64(0)
 		if bc.txLookupLimit != 0 && ancients > bc.txLookupLimit {
-			from = ancients - bc.txLookupLimit - 1
+			from = ancients - bc.txLookupLimit
 		}
 		rawdb.IndexTxLookup(bc.db, from, ancients)
 	}

core/blockchain_test.go

@@ -2181,7 +2181,7 @@ func TestTransactionIndices(t *testing.T) {
 				}
 				for _, tx := range block.Transactions() {
 					if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index != nil {
-						t.Logf("Transaction indice should be deleted, number %d hash %s", i, tx.Hash().Hex())
+						t.Fatalf("Transaction indice should be deleted, number %d hash %s", i, tx.Hash().Hex())
 					}
 				}
 			}
@@ -2232,7 +2232,7 @@ func TestTransactionIndices(t *testing.T) {
 		time.Sleep(50 * time.Millisecond) // Wait for indices initialisation
 		var tail uint64
 		if l != 0 {
-			tail = uint64(128) - l
+			tail = uint64(128) - l + 1
 		}
 		check(&tail, chain)
 		chain.Stop()
@@ -2247,7 +2247,7 @@ func TestTransactionIndices(t *testing.T) {
 	gspec.MustCommit(ancientDb)
 
 	limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */}
-	tails := []uint64{0, 66 /* 130 - 64 */, 99 /* 131 - 32 */, 68 /* 132 - 64 */, 0}
+	tails := []uint64{0, 67 /* 130 - 64 + 1 */, 100 /* 131 - 32 + 1 */, 69 /* 132 - 64 + 1 */, 0}
 	for i, l := range limit {
 		chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l)
 		if err != nil {
@@ -2260,6 +2260,92 @@ func TestTransactionIndices(t *testing.T) {
 	}
 }
 
+func TestSkipStaleTxIndicesInFastSync(t *testing.T) {
+	// Configure and generate a sample block chain
+	var (
+		gendb   = rawdb.NewMemoryDatabase()
+		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		address = crypto.PubkeyToAddress(key.PublicKey)
+		funds   = big.NewInt(1000000000)
+		gspec   = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}}
+		genesis = gspec.MustCommit(gendb)
+		signer  = types.NewEIP155Signer(gspec.Config.ChainID)
+	)
+	height := uint64(128)
+	blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), func(i int, block *BlockGen) {
+		tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, nil, nil), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		block.AddTx(tx)
+	})
+
+	check := func(tail *uint64, chain *BlockChain) {
+		stored := rawdb.ReadTxIndexTail(chain.db)
+		if tail == nil && stored != nil {
+			t.Fatalf("Oldest indexded block mismatch, want nil, have %d", *stored)
+		}
+		if tail != nil && *stored != *tail {
+			t.Fatalf("Oldest indexded block mismatch, want %d, have %d", *tail, *stored)
+		}
+		if tail != nil {
+			for i := *tail; i <= chain.CurrentBlock().NumberU64(); i++ {
+				block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i)
+				if block.Transactions().Len() == 0 {
+					continue
+				}
+				for _, tx := range block.Transactions() {
+					if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index == nil {
+						t.Fatalf("Miss transaction indice, number %d hash %s", i, tx.Hash().Hex())
+					}
+				}
+			}
+			for i := uint64(0); i < *tail; i++ {
+				block := rawdb.ReadBlock(chain.db, rawdb.ReadCanonicalHash(chain.db, i), i)
+				if block.Transactions().Len() == 0 {
+					continue
+				}
+				for _, tx := range block.Transactions() {
+					if index := rawdb.ReadTxLookupEntry(chain.db, tx.Hash()); index != nil {
+						t.Fatalf("Transaction indice should be deleted, number %d hash %s", i, tx.Hash().Hex())
+					}
+				}
+			}
+		}
+	}
+
+	frdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("failed to create temp freezer dir: %v", err)
+	}
+	defer os.Remove(frdir)
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "")
+	if err != nil {
+		t.Fatalf("failed to create temp freezer db: %v", err)
+	}
+	gspec.MustCommit(ancientDb)
+
+	// Import all blocks into ancient db, only HEAD-32 indices are kept.
+	l := uint64(32)
+	chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l)
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	headers := make([]*types.Header, len(blocks))
+	for i, block := range blocks {
+		headers[i] = block.Header()
+	}
+	if n, err := chain.InsertHeaderChain(headers, 0); err != nil {
+		t.Fatalf("failed to insert header %d: %v", n, err)
+	}
+	// The indices before ancient-N(32) should be ignored. After that all blocks should be indexed.
+	if n, err := chain.InsertReceiptChain(blocks, receipts, 64); err != nil {
+		t.Fatalf("block %d: failed to insert into chain: %v", n, err)
+	}
+	tail := uint64(32)
+	check(&tail, chain)
+}
+
 // Benchmarks large blocks with value transfers to non-existing accounts
 func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks int, recipientFn func(uint64) common.Address, dataFn func(uint64) []byte) {
 	var (

eth/sync.go

@@ -201,7 +201,7 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
 		if stored := rawdb.ReadFastTxLookupLimit(pm.chaindb); stored == nil {
 			rawdb.WriteFastTxLookupLimit(pm.chaindb, limit)
 		} else if *stored != limit {
-			pm.blockchain.AdjustTxLookupLimit(*stored)
+			pm.blockchain.SetTxLookupLimit(*stored)
 			log.Warn("Update txLookup limit", "provided", limit, "updated", *stored)
 		}
 	}