WIP: Pollard hashes up to populated nodes, not all the way to roots

accumulator/batchproof.go

@@ -10,10 +10,15 @@ import (
 type BatchProof struct {
 	Targets []uint64
 	Proof   []Hash
-	// list of leaf locations to delete, along with a bunch of hashes that give the proof.
+	// list of leaf locations to delete, along with a bunch of hashes that
+	// give the proof.
 	// the position of the hashes is implied / computable from the leaf positions
 }
 
+type miniTree struct {
+	l, r, parent node // left, right, parent
+}
+
 /*
 Batchproof serialization is:
 4bytes numTargets
@@ -130,75 +135,89 @@ func (bp *BatchProof) ToString() string {
 	return s
 }
 
-// TODO OH WAIT -- this is not how to to it!  Don't hash all the way up to the
-// roots to verify -- just hash up to any populated node!  Saves a ton of CPU!
+// TODO :
+/*
+several changes needed & maybe easier to do them incrementally but at this
+point it's more of a rewrite.
+The batchProof no longer contains target hashes; those are obtained separately
+from the leaf data.  This makes sense as the verifying node will have to
+know the preimages anyway to do tx/sig checks, so they can also compute the
+hashes themselves instead of receiving them.
+
+prior to this change: verifyBatchProof() verifies up to the roots,
+and then returned all the new stuff it received / computed, so that it
+could be populated into the pollard (to allow for subsequent deletion)
+
+the new way it works: verifyBatchProof() and IngestBatchProof() will be
+merged, since really right now IngestBatchProof() is basically just a wrapper
+for verifyBatchProof().  It will get a batchProof as well as a slice of
+target hashes (the things being proven).  It will hash up to known branches,
+then not return anything as it's populating as it goes.  If the ingestion fails,
+we need to undo everything added.  It's also ok to trim everything down to
+just the roots in that case for now; can add the backtrack later
+(it doesn't seem too hard if you just keep track of every new populated position,
+then wipe them on an invalid proof.  Though... if you want to be really
+efficient / DDoS resistant, only wipe the invalid parts and leave the partially
+checked stuff that works.
+
+
+*/
 
 // verifyBatchProof verifies a batchproof by checking against the set of known
 // correct roots.
 // Takes a BatchProof, the accumulator roots, and the number of leaves in the forest.
 // Returns wether or not the proof verified correctly, the partial proof tree,
 // and the subset of roots that was computed.
-func verifyBatchProof(bp BatchProof, roots []Hash, numLeaves uint64,
-	// cached should be a function that fetches nodes from the pollard and
-	// indicates whether they exist or not, this is only useful for the pollard
-	// and nil should be passed for the forest.
-	cached func(pos uint64) (bool, Hash)) (bool, [][3]node, []node) {
+func (p *Pollard) verifyBatchProof(
+	bp BatchProof, targs []Hash) ([]miniTree, []node, error) {
 	if len(bp.Targets) == 0 {
-		return true, nil, nil
+		return nil, nil, nil
 	}
+	fmt.Printf("got proof %s\n", bp.ToString())
 
+	rootHashes := p.rootHashesReverse()
 	// copy targets to leave them in original order
 	targets := make([]uint64, len(bp.Targets))
 	copy(targets, bp.Targets)
 	sortUint64s(targets)
 
-	if cached == nil {
-		cached = func(_ uint64) (bool, Hash) { return false, empty }
-	}
-
-	rows := treeRows(numLeaves)
-	proofPositions, computablePositions :=
-		ProofPositions(targets, numLeaves, rows)
-
+	rows := treeRows(p.numLeaves)
+	proofPositions := ProofPositions(targets, p.numLeaves, rows)
+	numComputable := len(targets)
 	// The proof should have as many hashes as there are proof positions.
-	if len(proofPositions)+len(bp.Targets) != len(bp.Proof) {
-		return false, nil, nil
+	if len(proofPositions) != len(bp.Proof) {
+		// fmt.Printf(")
+		return nil, nil,
+			fmt.Errorf("verifyBatchProof %d proofPositions but %d proof hashes",
+				len(proofPositions), len(bp.Proof))
 	}
 
 	// targetNodes holds nodes that are known, on the bottom row those
 	// are the targets, on the upper rows it holds computed nodes.
 	// rootCandidates holds the roots that where computed, and have to be
 	// compared to the actual roots at the end.
 	targetNodes := make([]node, 0, len(targets)*int(rows))
-	rootCandidates := make([]node, 0, len(roots))
-	// trees is a slice of 3-Tuples, each tuple represents a parent and its children.
-	// tuple[0] is the parent, tuple[1] is the left child and tuple[2]
-	// is the right child.
-	// trees holds the entire proof tree of the batchproof in this way,
-	// sorted by the tuple[0].
-	trees := make([][3]node, 0, len(computablePositions))
+	rootCandidates := make([]node, 0, len(rootHashes))
+	// trees holds the entire proof tree of the batchproof, sorted by parents.
+	trees := make([]miniTree, 0, numComputable)
 	// initialise the targetNodes for row 0.
 	// TODO: this would be more straight forward if bp.Proofs wouldn't
 	// contain the targets
+	// TODO targets are now given in a separate argument
+	// bp.Proofs is now on from ProofPositions()
 	proofHashes := make([]Hash, 0, len(proofPositions))
 	var targetsMatched uint64
 	for len(targets) > 0 {
-		// check if the target is the row 0 root.
-		// this is the case if its the last leaf (pos==numLeaves-1)
-		// AND the tree has a root at row 0 (numLeaves&1==1)
-		if targets[0] == numLeaves-1 && numLeaves&1 == 1 {
-			// target is the row 0 root, append it to the root candidates.
-			rootCandidates = append(rootCandidates,
-				node{Val: roots[0], Pos: targets[0]})
-			bp.Proof = bp.Proof[1:]
-			break
-		}
+
+		// a row-0 root should never be given, as it can only be a target and
+		// targets aren't sent
 
 		// `targets` might contain a target and its sibling or just the target, if
 		// only the target is present the sibling will be in `proofPositions`.
+
 		if uint64(len(proofPositions)) > targetsMatched &&
 			targets[0]^1 == proofPositions[targetsMatched] {
-			// the sibling of the target is included in the proof positions.
+			// target's sibling is in proof positions.
 			lr := targets[0] & 1
 			targetNodes = append(targetNodes, node{Pos: targets[0], Val: bp.Proof[lr]})
 			proofHashes = append(proofHashes, bp.Proof[lr^1])
@@ -208,14 +227,15 @@ func verifyBatchProof(bp BatchProof, roots []Hash, numLeaves uint64,
 			continue
 		}
 
-		// the sibling is not included in the proof positions, therefore
-		// it has to be included in targets. if there are less than 2 proof
+		// the sibling is not included in proof positions, therefore
+		// it must also be a target. if there are fewer than 2 proof
 		// hashes or less than 2 targets left the proof is invalid because
 		// there is a target without matching proof.
 		if len(bp.Proof) < 2 || len(targets) < 2 {
-			return false, nil, nil
+			return nil, nil, fmt.Errorf("verifyBatchProof ran out of proof hashes")
 		}
 
+		// if we got this far there are 2 targets that are siblings; pop em both
 		targetNodes = append(targetNodes,
 			node{Pos: targets[0], Val: bp.Proof[0]},
 			node{Pos: targets[1], Val: bp.Proof[1]})
@@ -241,7 +261,7 @@ func verifyBatchProof(bp BatchProof, roots []Hash, numLeaves uint64,
 			// target should have its sibling in targetNodes
 			if len(targetNodes) == 1 {
 				// sibling not found
-				return false, nil, nil
+				return nil, nil, fmt.Errorf("%v sibling not found", targetNodes)
 			}
 
 			proof = targetNodes[1]
@@ -257,17 +277,25 @@ func verifyBatchProof(bp BatchProof, roots []Hash, numLeaves uint64,
 
 		// get the hash of the parent from the cache or compute it
 		parentPos := parent(target.Pos, rows)
-		isParentCached, cachedHash := cached(parentPos)
 		hash := parentHash(left.Val, right.Val)
-		if isParentCached && hash != cachedHash {
+
+		populatedNode, _, _, err := p.grabPos(parentPos)
+		if err != nil {
+			return nil, nil, fmt.Errorf("verify grabPos error %s", err.Error())
+		}
+		if populatedNode != nil && populatedNode.data != empty &&
+			hash != populatedNode.data {
 			// The hash did not match the cached hash
-			return false, nil, nil
+			return nil, nil, fmt.Errorf("verifyBatchProof pos %d have %x calc'd %x",
+				parentPos, populatedNode.data, hash)
 		}
 
-		trees = append(trees, [3]node{{Val: hash, Pos: parentPos}, left, right})
+		trees = append(trees,
+			miniTree{parent: node{Val: hash, Pos: parentPos}, l: left, r: right})
 
 		row := detectRow(parentPos, rows)
-		if numLeaves&(1<<row) > 0 && parentPos == rootPosition(numLeaves, row, rows) {
+		if p.numLeaves&(1<<row) > 0 && parentPos ==
+			rootPosition(p.numLeaves, row, rows) {
 			// the parent is a root -> store as candidate, to check against
 			// actual roots later.
 			rootCandidates = append(rootCandidates, node{Val: hash, Pos: parentPos})
@@ -278,14 +306,14 @@ func verifyBatchProof(bp BatchProof, roots []Hash, numLeaves uint64,
 
 	if len(rootCandidates) == 0 {
 		// no roots to verify
-		return false, nil, nil
+		return nil, nil, fmt.Errorf("verifyBatchProof no roots")
 	}
 
 	// `roots` is ordered, therefore to verify that `rootCandidates`
 	// holds a subset of the roots
 	// we count the roots that match in order.
 	rootMatches := 0
-	for _, root := range roots {
+	for _, root := range rootHashes {
 		if len(rootCandidates) > rootMatches &&
 			root == rootCandidates[rootMatches].Val {
 			rootMatches++
@@ -294,43 +322,8 @@ func verifyBatchProof(bp BatchProof, roots []Hash, numLeaves uint64,
 	if len(rootCandidates) != rootMatches {
 		// the proof is invalid because some root candidates were not
 		// included in `roots`.
-		return false, nil, nil
-	}
-
-	return true, trees, rootCandidates
-}
-
-// Reconstruct takes a number of leaves and rows, and turns a block proof back
-// into a partial proof tree. Should leave bp intact
-func (bp *BatchProof) Reconstruct(
-	numleaves uint64, forestRows uint8) (map[uint64]Hash, error) {
-
-	if verbose {
-		fmt.Printf("reconstruct blockproof %d tgts %d hashes nl %d fr %d\n",
-			len(bp.Targets), len(bp.Proof), numleaves, forestRows)
-	}
-	proofTree := make(map[uint64]Hash)
-
-	// If there is nothing to reconstruct, return empty map
-	if len(bp.Targets) == 0 {
-		return proofTree, nil
-	}
-
-	// copy bp.targets and send copy
-	targets := make([]uint64, len(bp.Targets))
-	copy(targets, bp.Targets)
-	sortUint64s(targets)
-	proofPositions, _ := ProofPositions(targets, numleaves, forestRows)
-	proofPositions = mergeSortedSlices(targets, proofPositions)
-
-	if len(proofPositions) != len(bp.Proof) {
-		return nil, fmt.Errorf("Reconstruct wants %d hashes, has %d",
-			len(proofPositions), len(bp.Proof))
-	}
-
-	for i, pos := range proofPositions {
-		proofTree[pos] = bp.Proof[i]
+		return nil, nil, fmt.Errorf("verifyBatchProof missing roots")
 	}
 
-	return proofTree, nil
+	return trees, rootCandidates, nil
 }

accumulator/batchproof_test.go

@@ -1,51 +1,9 @@
 package accumulator
 
-import (
-	"fmt"
-	"testing"
-)
-
-// TestIncompleteBatchProof tests that a incomplete (missing some hashes) batchproof does not pass verification.
-func TestIncompleteBatchProof(t *testing.T) {
-	// Create forest in memory
-	f := NewForest(nil, false, "", 0)
-
-	// last index to be deleted. Same as blockDels
-	lastIdx := uint64(7)
-
-	// Generate adds
-	adds := make([]Leaf, 8)
-	adds[0].Hash = Hash{1}
-	adds[1].Hash = Hash{2}
-	adds[2].Hash = Hash{3}
-	adds[3].Hash = Hash{4}
-	adds[4].Hash = Hash{5}
-	adds[5].Hash = Hash{6}
-	adds[6].Hash = Hash{7}
-	adds[7].Hash = Hash{8}
-
-	// Modify with the additions to simulate txos being added
-	_, err := f.Modify(adds, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// create blockProof based on the last add in the slice
-	blockProof, err := f.ProveBatch(
-		[]Hash{adds[lastIdx].Hash})
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	blockProof.Proof = blockProof.Proof[:len(blockProof.Proof)-1]
-	shouldBeFalse := f.VerifyBatchProof(blockProof)
-	if shouldBeFalse != false {
-		t.Fail()
-		t.Logf("Incomplete proof passes verification")
-	}
-}
+// forests don't accept or verify proofs since they have everything.
+// TODO rewrite these tests using pollard instead of forest
 
+/*
 // TestVerifyBlockProof tests that the computedTop is compared to the top in the
 // Utreexo forest.
 func TestVerifyBatchProof(t *testing.T) {
@@ -147,3 +105,4 @@ func TestProofShouldNotValidateAfterNodeDeleted(t *testing.T) {
 				proofIndex))
 	}
 }
+*/

accumulator/forest.go

@@ -747,9 +747,13 @@ func (f *Forest) Stats() string {
 func (f *Forest) ToString() string {
 
 	fh := f.rows
-	// tree rows should be 6 or less
-	if fh > 6 {
-		return "forest too big to print "
+	// tree rows should be 3 or less to print, otherwise gives stats / roots
+	if fh > 3 {
+		s := fmt.Sprintf("%d leaves, roots: ", f.numLeaves)
+		for _, r := range f.getRoots() {
+			s += r.PrefixString() + " "
+		}
+		return s
 	}
 
 	output := make([]string, (fh*2)+1)