Dev/jgough/9019 transpose mmrblob code

go.mod

@@ -2,10 +2,59 @@ module github.com/datatrails/go-datatrails-merklelog
 
 go 1.21
 
-require github.com/stretchr/testify v1.8.4
+require (
+	github.com/datatrails/go-datatrails-common v0.12.2
+	github.com/google/uuid v1.6.0
+	github.com/stretchr/testify v1.8.4
+)
 
 require (
+	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
+	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/KimMachineGun/automemlimit v0.2.6 // indirect
+	github.com/cilium/ebpf v0.9.1 // indirect
+	github.com/containerd/cgroups/v3 v3.0.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/godbus/dbus/v5 v5.0.4 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.17.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/opencontainers/runtime-spec v1.0.2 // indirect
+	github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect
+	github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/openzipkin-contrib/zipkin-go-opentracing v0.5.0 // indirect
+	github.com/openzipkin/zipkin-go v0.4.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	go.uber.org/automaxprocs v1.5.3 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	go.uber.org/zap v1.25.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/grpc v1.57.1 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

mmrblobs/blobread.go

@@ -0,0 +1,41 @@
+package mmrblobs
+
+import (
+	"context"
+	"errors"
+	"io"
+
+	"github.com/datatrails/go-datatrails-common/azblob"
+)
+
+// BlobRead reads the blob of the given store.
+func BlobRead(
+	ctx context.Context, blobPath string, store massifReader,
+	opts ...azblob.Option) (*azblob.ReaderResponse, []byte, error) {
+
+	rr, err := store.Reader(ctx, blobPath, opts...)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	data := make([]byte, rr.ContentLength)
+	read := int64(0)
+	for read < rr.ContentLength {
+		n, err := rr.Reader.Read(data[read:])
+		if err != nil && !errors.Is(err, io.EOF) {
+			return nil, nil, err
+		}
+		read += int64(n)
+	}
+
+	// The reader is now definitely exhausted for the purpose it was created. To
+	// avoid odd effects, or accidental misuse we nill it out. And we do so regardless of error.
+
+	rr.Reader = nil // The caller has no use for this
+
+	// If we read less. truncate the buffer
+	if read < int64(len(data)) {
+		data = data[0:read]
+	}
+	return rr, data, nil
+}

mmrblobs/doc.go

@@ -0,0 +1,7 @@
+package mmrblobs
+
+/*
+
+Module for reading mmr blob (massif) from blob storage.
+
+*/

mmrblobs/errors.go

@@ -0,0 +1,7 @@
+package mmrblobs
+
+import "errors"
+
+var (
+	ErrNotleaf = errors.New("mmr node not a leaf")
+)

mmrblobs/indexentry.go

@@ -0,0 +1,72 @@
+package mmrblobs
+
+import (
+	"encoding/binary"
+	"errors"
+
+	"github.com/google/uuid"
+)
+
+const (
+	IndexEntryBytes     = 32 * 2
+	KeyBitSizeLogBase2  = 8
+	KeyByteSizeLogBase2 = 5
+
+	EventIDFirst     = 0
+	EventIDEnd       = EventIDFirst + 16
+	SnowflakeIdFirst = 24
+	SnowflakeIdEnd   = SnowflakeIdFirst + 8
+	AssetIDFirst     = SnowflakeIdEnd
+	AssetIDEnd       = AssetIDFirst + 16
+)
+
+var (
+	ErrIndexEntryBadSize = errors.New("log index size invalid")
+)
+
+// EmptyIndexEntry is a convenience method for unit tests that don't require a valid index entry
+func EmptyIndexEntry() []byte {
+	return make([]byte, IndexEntryBytes)
+}
+
+func SetIndexSnowflakeID(
+	data []byte, offset uint64,
+	snowflakeId uint64,
+) {
+	binary.BigEndian.PutUint64(data[offset+SnowflakeIdFirst:offset+SnowflakeIdEnd], snowflakeId)
+}
+
+func GetIndexSnowflakeID(
+	data []byte, offset uint64,
+) uint64 {
+	return binary.BigEndian.Uint64(data[offset+SnowflakeIdFirst : offset+SnowflakeIdEnd])
+}
+
+// NewIndexEntry creates an index entry directly from the required components
+func NewIndexEntry(
+	assetId uuid.UUID, eventId uuid.UUID, snowflakeId uint64,
+) []byte {
+	index := [IndexEntryBytes]byte{}
+
+	SetIndexEntry(index[:], 0, assetId, eventId, snowflakeId)
+	return index[:]
+}
+
+// SetIndexEntry populates the mmr blob index entry at the provided data offset
+//
+// | 0 -   127 | 128 - 185| 184 - 191        | 192 -  255 |
+// | event uuid| reserved | reserved (epoch) | snowflakeid|
+// | 0  -   15 | 16 -   22|     23           | 24   -   31|
+// |     16    |     7    |     1            |      8     |
+// | asset uuid|        reserved        |
+// | 256 -  384| 384 -           -  512 |
+// |     16    |           16           |
+func SetIndexEntry(
+	data []byte, offset uint64,
+	assetId uuid.UUID, eventId uuid.UUID, snowflakeId uint64,
+) {
+	copy(data[offset+EventIDFirst:offset+EventIDEnd], eventId[:])
+	copy(data[offset+AssetIDFirst:offset+AssetIDEnd], assetId[:])
+
+	binary.BigEndian.PutUint64(data[offset+SnowflakeIdFirst:offset+SnowflakeIdEnd], snowflakeId)
+}

mmrblobs/indexheader.go

@@ -0,0 +1,29 @@
+package mmrblobs
+
+const (
+// .         | 0   |           |  21 - 22 | 23   26|27         27| 28 -  31 |
+
+)
+
+// IndexHeader exists to keep track of the number of leaves represented by the
+// mmr data.
+//
+// Background:
+//
+// By keeping the index and the log together, we guarantee mutual consistency -
+// provided the log and the idex values are correctly calculated, a single write
+// commits the change back to the blob store.
+//
+// Because the data is combined, we can't use file size as a proxy for the
+// membership count.
+//
+// Regardless of whether we pre-allocate the index data or whether we accumulate
+// it as we do the mmr, we need to know how many leaves are in the index.  An
+// algorithm to derive a leaf index form an MMR position exists, it is sub
+// linear but a bit fiddly to get right.
+//
+// At least for now, we are going to explicitly track the count of leaves in a
+// counter value in the blob.
+type IndexHeader struct {
+	Index uint64
+}

mmrblobs/logentry.go

@@ -0,0 +1,60 @@
+package mmrblobs
+
+import "errors"
+
+// To enable exclusion proofs and history independent proof of completion we
+// assemble the log as an array of KEY, VALUE. Each is both 32 bytes.
+
+const (
+	ValueBits             = 128
+	ValueBytes            = 32
+	IndexHeaderBytes      = 32
+	LogEntryBytes         = 32
+	EntryByteSizeLogBase2 = 5
+	ValueBitSizeLogBase2  = 8
+	ValueByteSizeLogBase2 = 5
+)
+
+var (
+	ErrLogEntryToSmall = errors.New("to few bytes to represent a valid log entry")
+	ErrLogValueToSmall = errors.New("to few bytes to represent a valid log value")
+	ErrLogValueBadSize = errors.New("log value size invalid")
+)
+
+func IndexFromBlobSize(size int) uint64 {
+	if size == 0 {
+		return 0
+	}
+	return uint64(size>>EntryByteSizeLogBase2) - 1
+}
+
+type LogEntry struct {
+	Data []byte
+}
+
+// IndexedValue returns the value bytes from log data corresponding to entry
+// index i. No range checks are performed, out of range will panic
+func IndexedLogValue(logData []byte, i uint64) []byte {
+	return logData[i*LogEntryBytes : i*LogEntryBytes+ValueBytes]
+}
+
+func (le LogEntry) Value() []byte {
+	return le.Data[32:64]
+}
+
+func (le LogEntry) Entry() []byte {
+	return le.Data
+}
+
+func (le *LogEntry) CopyBytes(b []byte) int {
+	le.Data = make([]byte, ValueBytes)
+	return copy(le.Data, b)
+}
+
+func (le *LogEntry) SetBytes(b []byte) error {
+	if len(b) < (1 << EntryByteSizeLogBase2) {
+		return ErrLogEntryToSmall
+	}
+	le.Data = b
+	return nil
+}