Problem: collections out of sync with released version (#567)

* Problem: collections out of sync with released version

Solution:
- sync with collections/v0.4.0, prepare for hacking.

* replace collections

* sync core to v0.11.0

* tidy

collections/CHANGELOG.md

@@ -31,6 +31,29 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ## [Unreleased]
 
+## [v0.4.0](https://github.com/cosmos/cosmos-sdk/releases/tag/collections%2Fv0.4.0)
+
+### Features
+
+* [#17024](https://github.com/cosmos/cosmos-sdk/pull/17024) - Introduces `Triple`, a composite key with three keys.
+
+### API Breaking
+
+* [#17290](https://github.com/cosmos/cosmos-sdk/pull/17290) - Collections iteration methods (Iterate, Walk) will not error when the collection is empty.
+
+### Improvements
+
+* [#17021](https://github.com/cosmos/cosmos-sdk/pull/17021) Make collections implement the `appmodule.HasGenesis` interface.
+
+## [v0.3.0](https://github.com/cosmos/cosmos-sdk/releases/tag/collections%2Fv0.3.0)
+
+### Features
+
+* [#16074](https://github.com/cosmos/cosmos-sdk/pull/16607) - Introduces `Clear` method for `Map` and `KeySet`
+* [#16773](https://github.com/cosmos/cosmos-sdk/pull/16773)
+    * Adds `AltValueCodec` which provides a way to decode a value in two ways.
+    * Adds the possibility to specify an alternative way to decode the values of `KeySet`, `indexes.Multi`, `indexes.ReversePair`. 
+
 ## [v0.2.0](https://github.com/cosmos/cosmos-sdk/releases/tag/collections%2Fv0.2.0)
 
 ### Features

collections/README.md

@@ -1117,3 +1117,82 @@ func (k Keeper) GetAccount(ctx sdk.context, addr sdk.AccAddress) (sdk.AccountI,
     return k.Accounts.Get(ctx, addr)
 }
 ```
+
+## Triple key
+
+The `collections.Triple` is a special type of key composed of three keys, it's identical to `collections.Pair`.
+
+Let's see an example.
+
+```go
+package example
+
+import (
+ "context"
+
+ "cosmossdk.io/collections"
+ storetypes "cosmossdk.io/store/types"
+ "github.com/cosmos/cosmos-sdk/codec"
+)
+
+type AccAddress = string
+type ValAddress = string
+
+type Keeper struct {
+ // let's simulate we have redelegations which are stored as a triple key composed of
+ // the delegator, the source validator and the destination validator.
+ Redelegations collections.KeySet[collections.Triple[AccAddress, ValAddress, ValAddress]]
+}
+
+func NewKeeper(storeKey *storetypes.KVStoreKey) Keeper {
+ sb := collections.NewSchemaBuilder(sdk.OpenKVStore(storeKey))
+ return Keeper{
+  Redelegations: collections.NewKeySet(sb, collections.NewPrefix(0), "redelegations", collections.TripleKeyCodec(collections.StringKey, collections.StringKey, collections.StringKey)
+ }
+}
+
+// RedelegationsByDelegator iterates over all the redelegations of a given delegator and calls onResult providing
+// each redelegation from source validator towards the destination validator.
+func (k Keeper) RedelegationsByDelegator(ctx context.Context, delegator AccAddress, onResult func(src, dst ValAddress) (stop bool, err error)) error {
+ rng := collections.NewPrefixedTripleRange[AccAddress, ValAddress, ValAddress](delegator)
+ return k.Redelegations.Walk(ctx, rng, func(key collections.Triple[AccAddress, ValAddress, ValAddress]) (stop bool, err error) {
+  return onResult(key.K2(), key.K3())
+ })
+}
+
+// RedelegationsByDelegatorAndValidator iterates over all the redelegations of a given delegator and its source validator and calls onResult for each
+// destination validator.
+func (k Keeper) RedelegationsByDelegatorAndValidator(ctx context.Context, delegator AccAddress, validator ValAddress, onResult func(dst ValAddress) (stop bool, err error)) error {
+ rng := collections.NewSuperPrefixedTripleRange[AccAddress, ValAddress, ValAddress](delegator, validator)
+ return k.Redelegations.Walk(ctx, rng, func(key collections.Triple[AccAddress, ValAddress, ValAddress]) (stop bool, err error) {
+  return onResult(key.K3())
+ })
+}
+```
+
+## Advanced Usages
+
+### Alternative Value Codec
+
+The `codec.AltValueCodec` allows a collection to decode values using a different codec than the one used to encode them.
+Basically it enables to decode two different byte representations of the same concrete value.
+It can be used to lazily migrate values from one bytes representation to another, as long as the new representation is
+not able to decode the old one.
+
+A concrete example can be found in `x/bank` where the balance was initially stored as `Coin` and then migrated to `Int`.
+
+```go
+
+var BankBalanceValueCodec = codec.NewAltValueCodec(sdk.IntValue, func(b []byte) (sdk.Int, error) {
+    coin := sdk.Coin{}
+    err := coin.Unmarshal(b)
+    if err != nil {
+        return sdk.Int{}, err
+    }
+    return coin.Amount, nil
+})
+```
+
+The above example shows how to create an `AltValueCodec` that can decode both `sdk.Int` and `sdk.Coin` values. The provided 
+decoder function will be used as a fallback in case the default decoder fails. When the value will be encoded back into state
+it will use the default encoder. This allows to lazily migrate values to a new bytes representation.

collections/codec/alternative_value.go

@@ -0,0 +1,47 @@
+package codec
+
+// NewAltValueCodec returns a new AltValueCodec. canonicalValueCodec is the codec that you want the value
+// to be encoded and decoded as, alternativeDecoder is a function that will attempt to decode the value
+// in case the canonicalValueCodec fails to decode it.
+func NewAltValueCodec[V any](canonicalValueCodec ValueCodec[V], alternativeDecoder func([]byte) (V, error)) ValueCodec[V] {
+	return AltValueCodec[V]{
+		canonicalValueCodec: canonicalValueCodec,
+		alternativeDecoder:  alternativeDecoder,
+	}
+}
+
+// AltValueCodec is a codec that can decode a value from state in an alternative format.
+// This is useful for migrating data from one format to another. For example, in x/bank
+// balances were initially encoded as sdk.Coin, now they are encoded as math.Int.
+// The AltValueCodec will be trying to decode the value as math.Int, and if that fails,
+// it will attempt to decode it as sdk.Coin.
+// NOTE: if the canonical format can also decode the alternative format, then this codec
+// will produce undefined and undesirable behavior.
+type AltValueCodec[V any] struct {
+	canonicalValueCodec ValueCodec[V]
+	alternativeDecoder  func([]byte) (V, error)
+}
+
+// Decode will attempt to decode the value from state using the canonical value codec.
+// If it fails to decode, it will attempt to decode the value using the alternative decoder.
+func (a AltValueCodec[V]) Decode(b []byte) (V, error) {
+	v, err := a.canonicalValueCodec.Decode(b)
+	if err != nil {
+		return a.alternativeDecoder(b)
+	}
+	return v, nil
+}
+
+// Below there is the implementation of ValueCodec relying on the canonical value codec.
+
+func (a AltValueCodec[V]) Encode(value V) ([]byte, error) { return a.canonicalValueCodec.Encode(value) }
+
+func (a AltValueCodec[V]) EncodeJSON(value V) ([]byte, error) {
+	return a.canonicalValueCodec.EncodeJSON(value)
+}
+
+func (a AltValueCodec[V]) DecodeJSON(b []byte) (V, error) { return a.canonicalValueCodec.DecodeJSON(b) }
+
+func (a AltValueCodec[V]) Stringify(value V) string { return a.canonicalValueCodec.Stringify(value) }
+
+func (a AltValueCodec[V]) ValueType() string { return a.canonicalValueCodec.ValueType() }

collections/codec/alternative_value_test.go

@@ -0,0 +1,53 @@
+package codec_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"cosmossdk.io/collections/codec"
+	"cosmossdk.io/collections/colltest"
+)
+
+type altValue struct {
+	Value uint64 `json:"value"`
+}
+
+func TestAltValueCodec(t *testing.T) {
+	// we assume we want to migrate the value from json(altValue) to just be
+	// the raw value uint64.
+	canonical := codec.KeyToValueCodec(codec.NewUint64Key[uint64]())
+	alternative := func(v []byte) (uint64, error) {
+		var alt altValue
+		err := json.Unmarshal(v, &alt)
+		if err != nil {
+			return 0, err
+		}
+		return alt.Value, nil
+	}
+
+	cdc := codec.NewAltValueCodec(canonical, alternative)
+
+	t.Run("decodes alternative value", func(t *testing.T) {
+		expected := uint64(100)
+		alternativeEncodedBytes, err := json.Marshal(altValue{Value: expected})
+		require.NoError(t, err)
+		got, err := cdc.Decode(alternativeEncodedBytes)
+		require.NoError(t, err)
+		require.Equal(t, expected, got)
+	})
+
+	t.Run("decodes canonical value", func(t *testing.T) {
+		expected := uint64(100)
+		canonicalEncodedBytes, err := cdc.Encode(expected)
+		require.NoError(t, err)
+		got, err := cdc.Decode(canonicalEncodedBytes)
+		require.NoError(t, err)
+		require.Equal(t, expected, got)
+	})
+
+	t.Run("conformance", func(t *testing.T) {
+		colltest.TestValueCodec(t, cdc, uint64(100))
+	})
+}

collections/codec/bool.go

@@ -33,7 +33,7 @@ func (b boolKey[T]) Decode(buffer []byte) (int, T, error) {
 	}
 }
 
-func (b boolKey[T]) Size(key T) int { return 1 }
+func (b boolKey[T]) Size(_ T) int { return 1 }
 
 func (b boolKey[T]) EncodeJSON(value T) ([]byte, error) {
 	return json.Marshal(value)

collections/colltest/codec.go

@@ -40,6 +40,11 @@ func TestKeyCodec[T any](t *testing.T, keyCodec codec.KeyCodec[T], key T) {
 	decoded, err := keyCodec.DecodeJSON(keyJSON)
 	require.NoError(t, err)
 	require.Equal(t, key, decoded, "json encoding and decoding did not produce the same results")
+
+	// check type
+	require.NotEmpty(t, keyCodec.KeyType())
+	// check string
+	_ = keyCodec.Stringify(key)
 }
 
 // TestValueCodec asserts the correct behavior of a ValueCodec over the type T.

collections/genesis.go

@@ -32,7 +32,7 @@ func (m Map[K, V]) importGenesis(ctx context.Context, reader io.Reader) error {
 }
 
 func (m Map[K, V]) exportGenesis(ctx context.Context, writer io.Writer) error {
-	_, err := writer.Write([]byte("["))
+	_, err := io.WriteString(writer, "[")
 	if err != nil {
 		return err
 	}
@@ -48,7 +48,7 @@ func (m Map[K, V]) exportGenesis(ctx context.Context, writer io.Writer) error {
 		// add a comma before encoding the object
 		// for all objects besides the first one.
 		if !first {
-			_, err = writer.Write([]byte(","))
+			_, err = io.WriteString(writer, ",")
 			if err != nil {
 				return err
 			}
@@ -91,7 +91,7 @@ func (m Map[K, V]) exportGenesis(ctx context.Context, writer io.Writer) error {
 		}
 	}
 
-	_, err = writer.Write([]byte("]"))
+	_, err = io.WriteString(writer, "]")
 	return err
 }
 
@@ -148,6 +148,6 @@ func (m Map[K, V]) doDecodeJSON(reader io.Reader, onEntry func(key K, value V) e
 }
 
 func (m Map[K, V]) defaultGenesis(writer io.Writer) error {
-	_, err := writer.Write([]byte(`[]`))
+	_, err := io.WriteString(writer, `[]`)
 	return err
 }

collections/genesis_test.go

@@ -91,6 +91,7 @@ type testFixture struct {
 }
 
 func initFixture(t *testing.T) *testFixture {
+	t.Helper()
 	sk, ctx := deps()
 	schemaBuilder := NewSchemaBuilder(sk)
 	m := NewMap(schemaBuilder, NewPrefix(1), "map", StringKey, Uint64Value)
@@ -110,6 +111,7 @@ func initFixture(t *testing.T) *testFixture {
 }
 
 func createTestGenesisSource(t *testing.T) appmodule.GenesisSource {
+	t.Helper()
 	expectedOrder := []string{"item", "key_set", "map", "sequence"}
 	currentIndex := 0
 	return func(field string) (io.ReadCloser, error) {
@@ -149,6 +151,7 @@ func (b *bufCloser) Close() error {
 }
 
 func newBufCloser(t *testing.T, str string) *bufCloser {
+	t.Helper()
 	b := &bufCloser{
 		Buffer: bytes.NewBufferString(str),
 		closed: false,

collections/go.mod

@@ -1,53 +1,55 @@
 module cosmossdk.io/collections
 
-go 1.21
+go 1.20
 
 require (
-	cosmossdk.io/core v0.11.0
-	github.com/cosmos/cosmos-db v1.0.2
-	github.com/stretchr/testify v1.9.0
+	cosmossdk.io/core v0.10.0
+	github.com/cosmos/cosmos-db v1.0.0
+	github.com/stretchr/testify v1.8.4
 	pgregory.net/rapid v1.1.0
 )
 
 require (
-	cosmossdk.io/api v0.7.5 // indirect
+	cosmossdk.io/api v0.7.0 // indirect
 	cosmossdk.io/depinject v1.0.0-alpha.4 // indirect
 	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.11.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
-	github.com/cockroachdb/pebble v1.1.0 // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230525220056-bb4fc9527b3b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
-	github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect
+	github.com/cosmos/cosmos-proto v1.0.0-beta.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/getsentry/sentry-go v0.27.0 // indirect
+	github.com/getsentry/sentry-go v0.23.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2 // indirect
-	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/linxGnu/grocksdb v1.8.12 // indirect
+	github.com/linxGnu/grocksdb v1.7.16 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/onsi/gomega v1.20.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.18.0 // indirect
-	github.com/prometheus/client_model v0.6.0 // indirect
-	github.com/prometheus/common v0.47.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
-	github.com/rogpeppe/go-internal v1.12.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
-	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
-	golang.org/x/net v0.20.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect
-	google.golang.org/grpc v1.58.3 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
+	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/grpc v1.58.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace cosmossdk.io/core => ../core