feat: Local CAS support for writing IPFS-compatible v0 CIDs

The local CAS is now capable of generating v0 CIDs that are compatible with an IPFS node, assuming that it's running under default settings and that the data is less than 256KB. However, this capability is not currently exposed or used outside of testing in order to ensure our current sample data doesn't break. In a future commit, the CID format (v0 or v1) will be determined dynamically based on the format of the data received by this Orb server from another Orb server. Signed-off-by: Derek Trider <[email protected]>
trustbloc · May 19, 2021 · 99362af · 99362af
1 parent 4aca3a8
commit 99362af
Show file tree

Hide file tree

Showing 7 changed files with 839 additions and 19 deletions.
diff --git a/cmd/orb-server/go.sum b/cmd/orb-server/go.sum
diff --git a/go.mod b/go.mod
@@ -20,6 +20,8 @@ require (
 	github.com/igor-pavlenko/httpsignatures-go v0.0.21
 	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-ipfs-api v0.2.0
+	github.com/ipfs/go-merkledag v0.2.3
+	github.com/ipfs/go-unixfs v0.2.6
 	github.com/mr-tron/base58 v1.2.0
 	github.com/multiformats/go-multihash v0.0.14
 	github.com/ory/dockertest/v3 v3.6.3

diff --git a/go.sum b/go.sum
diff --git a/pkg/store/cas/cas.go b/pkg/store/cas/cas.go
@@ -11,16 +11,20 @@ import (
 	"fmt"
 
 	ariesstorage "github.com/hyperledger/aries-framework-go/spi/storage"
-	"github.com/ipfs/go-cid"
+	gocid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-merkledag"
+	"github.com/ipfs/go-unixfs"
 	mh "github.com/multiformats/go-multihash"
 )
 
 // ErrContentNotFound is used to indicate that content as a given address could not be found.
 var ErrContentNotFound = errors.New("content not found")
 
 // CAS represents a content-addressable storage provider.
+// TODO (#344) Support writing and reading both v0 and v1 CIDs.
 type CAS struct {
-	cas ariesstorage.Store
+	cas                     ariesstorage.Store
+	useIPFSCompatibleV0CIDs bool // TODO (#344) remove this temporary internal flag
 }
 
 // New returns a new CAS that uses the passed in provider as a backing store.
@@ -36,24 +40,34 @@ func New(provider ariesstorage.Provider) (*CAS, error) {
 // Write writes the given content to the underlying storage provider.
 // Returns the address of the content.
 func (p *CAS) Write(content []byte) (string, error) {
-	// TODO #318 figure out why the CIDs produced here differ from the ones that IPFS generates.
-	prefix := cid.Prefix{
-		Version:  0,
-		MhType:   mh.SHA2_256,
-		MhLength: -1, // default length
-	}
+	var cid string
+	if p.useIPFSCompatibleV0CIDs {
+		// The v0 CID produced below is equal to what an IPFS node would produce, assuming that:
+		// 1. The IPFS node is running with default settings, and
+		// 2. The size of the content passed in here is less than 256KB (the default chunk size).
+		// Two levels of wrapping are needed. First, the raw data is wrapped as a protobuf UnixFS file, then that needs
+		// to be further wrapped as a protobuf DAG node.
+		cid = merkledag.NodeWithData(unixfs.FilePBData(content, uint64(len(content)))).Cid().String()
+	} else {
+		prefix := gocid.Prefix{
+			Version:  0,
+			MhType:   mh.SHA2_256,
+			MhLength: -1, // default length
+		}
 
-	contentID, err := prefix.Sum(content)
-	if err != nil {
-		return "", fmt.Errorf("failed to generate CID: %w", err)
+		contentID, err := prefix.Sum(content)
+		if err != nil {
+			return "", fmt.Errorf("failed to generate CID: %w", err)
+		}
+
+		cid = contentID.String()
 	}
 
-	err = p.cas.Put(contentID.String(), content)
-	if err != nil {
+	if err := p.cas.Put(cid, content); err != nil {
 		return "", fmt.Errorf("failed to put content into underlying storage provider: %w", err)
 	}
 
-	return contentID.String(), nil
+	return cid, nil
 }
 
 // Read reads the content of the given address from the underlying storage provider.

diff --git a/pkg/store/cas/cas_internal_test.go b/pkg/store/cas/cas_internal_test.go
@@ -0,0 +1,167 @@
+/*
+Copyright SecureKey Technologies Inc. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package cas
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	ariesmemstorage "github.com/hyperledger/aries-framework-go/component/storageutil/mem"
+	ipfsshell "github.com/ipfs/go-ipfs-api"
+	dctest "github.com/ory/dockertest/v3"
+	dc "github.com/ory/dockertest/v3/docker"
+	"github.com/stretchr/testify/require"
+)
+
+const sampleAnchorCredential = `{
+  "@context": [
+    "https://www.w3.org/2018/credentials/v1",
+    "https://trustbloc.github.io/did-method-orb/contexts/anchor/v1",
+    "https://w3id.org/jws/v1"
+  ],
+  "id": "http://sally.example.com/transactions/bafkreihwsnuregceqh263vgdathcprnbvatyat6h6mu7ipjhhodcdbyhoy",
+  "type": [
+    "VerifiableCredential",
+    "AnchorCredential"
+  ],
+  "issuer": "https://sally.example.com/services/orb",
+  "issuanceDate": "2021-01-27T09:30:10Z",
+  "credentialSubject": {
+    "operationCount": 1,
+    "coreIndex": "bafkreihwsnuregceqh263vgdathcprnbvatyat6h6mu7ipjhhodcdbyhoy",
+    "namespace": "did:orb",
+    "version": "1",
+    "previousAnchors": {
+      "EiA329wd6Aj36YRmp7NGkeB5ADnVt8ARdMZMPzfXsjwTJA": "bafkreibmrmenuxhgaomod4m26ds5ztdujxzhjobgvpsyl2v2ndcskq2iay",
+      "EiABk7KK58BVLHMataxgYZjTNbsHgtD8BtjF0tOWFV29rw": "bafkreibh3whnisud76knkv7z7ucbf3k2rs6knhvajernrdabdbfaomakli"
+    },
+    "type": "Anchor"
+  },
+  "proof": [{
+    "type": "JsonWebSignature2020",
+    "proofPurpose": "assertionMethod",
+    "created": "2021-01-27T09:30:00Z",
+    "verificationMethod": "did:example:abcd#key",
+    "domain": "sally.example.com",
+    "jws": "eyJ..."
+  },
+  {
+    "type": "JsonWebSignature2020",
+    "proofPurpose": "assertionMethod",
+    "created": "2021-01-27T09:30:05Z",
+    "verificationMethod": "did:example:abcd#key",
+    "domain": "https://witness1.example.com/ledgers/maple2021",
+    "jws": "eyJ..."
+  },
+  {
+    "type": "JsonWebSignature2020",
+    "proofPurpose": "assertionMethod",
+    "created": "2021-01-27T09:30:06Z",
+    "verificationMethod": "did:example:efgh#key",
+    "domain": "https://witness2.example.com/ledgers/spruce2021",
+    "jws": "eyJ..."
+  }]                  
+}`
+
+func TestEnsureLocalCASAndIPFSProduceSameCIDs(t *testing.T) {
+	pool, ipfsResource := startIPFSDockerContainer(t)
+
+	defer func() {
+		require.NoError(t, pool.Purge(ipfsResource), "failed to purge IPFS resource")
+	}()
+
+	t.Run("v0", func(t *testing.T) {
+		ensureCIDsAreEqualBetweenLocalCASAndIPFS(t, true)
+	})
+	// TODO (#344) re-enable this test once the local CAS is capable of creating v1 CIDs.
+	// t.Run("v1", func(t *testing.T) {
+	//	ensureCIDsAreEqualBetweenLocalCASAndIPFS(t, false)
+	// })
+} // nolint:wsl
+
+func startIPFSDockerContainer(t *testing.T) (*dctest.Pool, *dctest.Resource) {
+	t.Helper()
+
+	pool, err := dctest.NewPool("")
+	require.NoError(t, err, "failed to create pool")
+
+	ipfsResource, err := pool.RunWithOptions(&dctest.RunOptions{
+		Repository: "ipfs/go-ipfs",
+		Tag:        "master-2021-04-22-eea198f",
+		PortBindings: map[dc.Port][]dc.PortBinding{
+			"5001/tcp": {{HostIP: "", HostPort: "5001"}},
+		},
+	})
+	if err != nil {
+		require.FailNow(t, "Failed to start IPFS Docker image."+
+			" This can happen if there is an IPFS container still running from a previous unit test run."+
+			` Try "docker ps" from the command line and kill the old container if it's still running.`)
+	}
+
+	return pool, ipfsResource
+}
+
+func ensureCIDsAreEqualBetweenLocalCASAndIPFS(t *testing.T, useV0CIDs bool) {
+	t.Helper()
+
+	smallSimpleData := []byte("content")
+
+	sampleAnchorCredentialBytes := []byte(sampleAnchorCredential)
+
+	cidFromIPFS := addDataToIPFS(t, smallSimpleData, useV0CIDs)
+	cidFromLocalCAS := addDataToLocalCAS(t, smallSimpleData, useV0CIDs)
+
+	require.Equal(t, cidFromIPFS, cidFromLocalCAS)
+
+	cidFromIPFS = addDataToIPFS(t, sampleAnchorCredentialBytes, useV0CIDs)
+	cidFromLocalCAS = addDataToLocalCAS(t, sampleAnchorCredentialBytes, useV0CIDs)
+
+	require.Equal(t, cidFromIPFS, cidFromLocalCAS)
+}
+
+func addDataToIPFS(t *testing.T, data []byte, useV0CIDs bool) string {
+	t.Helper()
+
+	shell := ipfsshell.NewShell("localhost:5001")
+
+	shell.SetTimeout(2 * time.Second)
+
+	// IPFS will need some time to start up, hence the need for retries.
+	var cid string
+
+	var v1AddOpt []ipfsshell.AddOpts
+
+	if !useV0CIDs {
+		v1AddOpt = []ipfsshell.AddOpts{ipfsshell.CidVersion(1)}
+	}
+
+	err := backoff.Retry(func() error {
+		var err error
+		cid, err = shell.Add(bytes.NewReader(data), v1AddOpt...)
+
+		return err
+	}, backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Millisecond*500), 10))
+	require.NoError(t, err)
+
+	return cid
+}
+
+func addDataToLocalCAS(t *testing.T, data []byte, useV0CIDs bool) string {
+	t.Helper()
+
+	cas, err := New(ariesmemstorage.NewProvider())
+	require.NoError(t, err)
+
+	cas.useIPFSCompatibleV0CIDs = useV0CIDs
+
+	cid, err := cas.Write(data)
+	require.NoError(t, err)
+
+	return cid
+}
diff --git a/test/bdd/features/activitypub.feature b/test/bdd/features/activitypub.feature
@@ -112,7 +112,7 @@ Feature:
     Then the JSON path "type" of the response equals "CollectionPage"
     And the JSON path "items" of the response does not contain "${domain2IRI}"
 
-""" TODO: Enable this test when CIDs for ipfs and local CAS are identical
+""" TODO (#344): Enable this test when CIDs for ipfs and local CAS are identical our local CAS and IPFS client implementations can switch between v0 and v1
   @activitypub_create
   Scenario: create/announce
     Given the authorization bearer token for "POST" requests to path "/services/orb/outbox" is set to "ADMIN_TOKEN"