indexers: generate root summaries

Root summaries are now generated at the last 50,000 block interval and
is stored in the summaries flat file state. If the summaries are not
availble on start, the summaries are generated.

blockchain/indexers/flatutreexoproofindex.go

@@ -6,6 +6,7 @@ package indexers
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"os"
@@ -58,6 +59,10 @@ const (
 	// a utreexo accumulator proof should be generated.  An interval of 10 will
 	// make the proof be generated on blocks 10, 20, 30 and so on.
 	defaultProofGenInterval = 10
+
+	// rootSummaryGenInterval is the interval value used for generating the root
+	// summaries.
+	rootSummaryGenInterval = 50_000
 )
 
 var (
@@ -389,6 +394,18 @@ func (idx *FlatUtreexoProofIndex) ConnectBlock(dbTx database.Tx, block *btcutil.
 		return err
 	}
 
+	// isLastestSummaryRange checks to see if we're within rootSummaryGenInterval
+	// blocks from the best header block. This prevents the summaries to
+	// be created during ibd.
+	_, bestHeaderHeight := idx.chain.BestHeader()
+	isLastestSummaryRange := bestHeaderHeight-block.Height() < rootSummaryGenInterval
+	if isLastestSummaryRange && block.Height()%rootSummaryGenInterval == 0 {
+		err = idx.generateRootSummary(block.Height())
+		if err != nil {
+			return err
+		}
+	}
+
 	// Don't store proofs if the node is pruned.
 	if idx.config.Pruned {
 		return nil
@@ -868,6 +885,70 @@ func (idx *FlatUtreexoProofIndex) VerifyAccProof(toProve []utreexo.Hash,
 	return idx.utreexoState.state.Verify(toProve, *proof, false)
 }
 
+// generateRootSummary creates and saves the proofs and the roots up to the given endHeight.
+func (idx *FlatUtreexoProofIndex) generateRootSummary(endHeight int32) error {
+	log.Infof("generating root summaries up to height %v", endHeight)
+
+	// Delete all the existing data in the root summary state.
+	err := idx.rootSummaryState.ResetFlatFileState()
+	if err != nil {
+		return err
+	}
+
+	// Construct the accumulator.
+	rootHashes := make([][32]byte, 0, endHeight)
+	accumulator := utreexo.NewAccumulator()
+	for h := int32(1); h <= endHeight; h++ {
+		stump, err := idx.fetchRoots(h)
+		if err != nil {
+			return err
+		}
+
+		// Calculate a single hash of the roots.
+		bytes := make([]byte, 0, len(stump.Roots)*chainhash.HashSize)
+		for _, root := range stump.Roots {
+			bytes = append(bytes, root[:]...)
+		}
+		rootHash := sha256.Sum256(bytes)
+		rootHashes = append(rootHashes, rootHash)
+
+		// Add that root hash to the accumulator.
+		err = accumulator.Modify([]utreexo.Leaf{{Hash: rootHash}}, nil, utreexo.Proof{})
+		if err != nil {
+			return err
+		}
+	}
+
+	// Prove the leaves in the accumulator. This must be done in a separate loop
+	// as there will be additional hashes appended as the accumulator grows.
+	for h := int32(1); h <= endHeight; h++ {
+		proof, err := accumulator.Prove([]utreexo.Hash{rootHashes[h-1]})
+		if err != nil {
+			return err
+		}
+
+		buf := bytes.NewBuffer(make([]byte, 0, wire.BatchProofSerializeSize(&proof)))
+		err = wire.BatchProofSerialize(buf, &proof)
+		if err != nil {
+			return err
+		}
+
+		err = idx.rootSummaryState.StoreData(h, buf.Bytes())
+		if err != nil {
+			return err
+		}
+	}
+
+	// Save the roots themselves.
+	serialized, err := blockchain.SerializeUtreexoRoots(
+		accumulator.GetNumLeaves(), accumulator.GetRoots())
+	if err != nil {
+		return err
+	}
+
+	return idx.rootSummaryState.StoreData(endHeight+1, serialized)
+}
+
 // flatFilePath returns the path to the flatfile.
 func flatFilePath(dataDir, dataName string) string {
 	flatFileName := dataName + "_" + flatFileNameSuffix
@@ -945,6 +1026,15 @@ func NewFlatUtreexoProofIndex(pruned bool, chainParams *chaincfg.Params,
 		return nil, err
 	}
 	idx.rootSummaryState = *rootSummaryState
+	// Construct the root summary state if we don't have a root summary state.
+	if idx.rootSummaryState.BestHeight() == 0 {
+		bestHeight := idx.rootsState.BestHeight()
+		endHeight := bestHeight - (bestHeight % rootSummaryGenInterval)
+		err = idx.generateRootSummary(endHeight)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	err = idx.pStats.InitPStats(proofStatsState)
 	if err != nil {