(Issue Nos. 5 & 6) Squashed commit of the following:

commit 4517504
Author: Malte Kliemann <[email protected]>
Date:   Tue Dec 10 18:51:24 2024 +0100

    Benchmark `DecisionMarketOracle`

commit 5832f0b
Author: Malte Kliemann <[email protected]>
Date:   Tue Dec 10 16:55:48 2024 +0100

    Implement `DecisionMarketOracle` using scoreboard

commit e684d47
Author: Malte Kliemann <[email protected]>
Date:   Mon Dec 9 21:24:47 2024 +0100

    Add `BlockNumber` parameter to `update`

commit 5f51378
Author: Malte Kliemann <[email protected]>
Date:   Mon Dec 9 21:17:20 2024 +0100

    Update oracles on each block

commit 2b831fb
Author: Malte Kliemann <[email protected]>
Date:   Mon Dec 9 20:40:26 2024 +0100

    Implement `try_mutate_all`

commit 59cb185
Author: Malte Kliemann <[email protected]>
Date:   Mon Dec 9 18:24:56 2024 +0100

    Use `ProposalStorage` and `MaxProposals`

commit 58afe87
Author: Malte Kliemann <[email protected]>
Date:   Mon Dec 9 17:56:53 2024 +0100

    Add `MaxProposals` value

commit 22068d8
Author: Malte Kliemann <[email protected]>
Date:   Mon Dec 9 14:15:33 2024 +0100

    Add `update` function to `FutarchyOracle` trait

primitives/src/traits/futarchy_oracle.rs

@@ -18,7 +18,12 @@
 use frame_support::pallet_prelude::Weight;
 
 pub trait FutarchyOracle {
+    type BlockNumber;
+
     /// Evaluates the query at the current block and returns the weight consumed and a `bool`
     /// indicating whether the query evaluated positively.
     fn evaluate(&self) -> (Weight, bool);
+
+    /// Updates the oracle's data and returns the weight consumed.
+    fn update(&mut self, now: Self::BlockNumber) -> Weight;
 }

runtime/battery-station/src/parameters.rs

@@ -165,9 +165,10 @@ parameter_types! {
     /// can lead to extrinsic with very big weight: see delegate for instance.
     pub const MaxVotes: u32 = 100;
     /// The maximum number of public proposals that can exist at any time.
-    pub const MaxProposals: u32 = 100;
+    pub const DemocracyMaxProposals: u32 = 100;
 
     // Futarchy
+    pub const FutarchyMaxProposals: u32 = 4;
     pub const MinDuration: BlockNumber = 7 * BLOCKS_PER_DAY;
 
     // Hybrid Router parameters

runtime/common/src/lib.rs

@@ -815,7 +815,7 @@ macro_rules! impl_config_traits {
             type PalletsOrigin = OriginCaller;
             type MaxVotes = MaxVotes;
             type WeightInfo = weights::pallet_democracy::WeightInfo<Runtime>;
-            type MaxProposals = MaxProposals;
+            type MaxProposals = DemocracyMaxProposals;
             type Preimages = Preimage;
             type MaxBlacklisted = ConstU32<100>;
             type MaxDeposits = ConstU32<100>;
@@ -1199,6 +1199,7 @@ macro_rules! impl_config_traits {
         impl zrml_futarchy::Config for Runtime {
             #[cfg(feature = "runtime-benchmarks")]
             type BenchmarkHelper = DecisionMarketBenchmarkHelper<Runtime>;
+            type MaxProposals = FutarchyMaxProposals;
             type MinDuration = MinDuration;
             type Oracle = DecisionMarketOracle<Runtime>;
             type RuntimeEvent = RuntimeEvent;
@@ -2286,7 +2287,7 @@ macro_rules! create_common_tests {
                 };
                 use zrml_futarchy::types::Proposal;
                 use zrml_market_commons::types::MarketBuilder;
-                use zrml_neo_swaps::types::DecisionMarketOracle;
+                use zrml_neo_swaps::types::{DecisionMarketOracle, DecisionMarketOracleScoreboard};
 
                 #[test]
                 fn futarchy_schedules_and_executes_call() {
@@ -2360,10 +2361,13 @@ macro_rules! create_common_tests {
                         };
                         let call =
                             Preimage::bound(RuntimeCall::from(remark_dispatched_as)).unwrap();
+                        let scoreboard =
+                            DecisionMarketOracleScoreboard::new(40_000, 10_000, one / 7, one);
                         let oracle = DecisionMarketOracle::new(
                             market_id,
                             Asset::CategoricalOutcome(market_id, 0),
                             Asset::CategoricalOutcome(market_id, 1),
+                            scoreboard,
                         );
                         let when = duration + 10;
                         let proposal = Proposal { when, call, oracle };

runtime/zeitgeist/src/parameters.rs

@@ -165,9 +165,10 @@ parameter_types! {
     /// can lead to extrinsic with very big weight: see delegate for instance.
     pub const MaxVotes: u32 = 100;
     /// The maximum number of public proposals that can exist at any time.
-    pub const MaxProposals: u32 = 100;
+    pub const DemocracyMaxProposals: u32 = 100;
 
     // Futarchy
+    pub const FutarchyMaxProposals: u32 = 4;
     pub const MinDuration: BlockNumber = 7 * BLOCKS_PER_DAY;
 
     // Hybrid Router parameters

zrml/futarchy/src/dispatchable_impls.rs

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Zeitgeist. If not, see <https://www.gnu.org/licenses/>.
 
-use crate::{types::Proposal, Config, Error, Event, Pallet, Proposals};
+use crate::{traits::ProposalStorage, types::Proposal, Config, Error, Event, Pallet};
 use frame_support::{ensure, require_transactional, traits::Get};
 use frame_system::pallet_prelude::BlockNumberFor;
 use sp_runtime::{DispatchResult, Saturating};
@@ -31,12 +31,10 @@ impl<T: Config> Pallet<T> {
         let now = frame_system::Pallet::<T>::block_number();
         let to_be_scheduled_at = now.saturating_add(duration);
 
-        let try_mutate_result = Proposals::<T>::try_mutate(to_be_scheduled_at, |proposals| {
-            proposals.try_push(proposal.clone()).map_err(|_| Error::<T>::CacheFull)
-        });
+        <Pallet<T> as ProposalStorage<T>>::add(to_be_scheduled_at, proposal.clone())?;
 
         Self::deposit_event(Event::<T>::Submitted { duration, proposal });
 
-        Ok(try_mutate_result?)
+        Ok(())
     }
 }

zrml/futarchy/src/lib.rs

@@ -42,6 +42,7 @@ mod benchmarking;
 mod dispatchable_impls;
 pub mod mock;
 mod pallet_impls;
+mod proposal_storage;
 mod tests;
 pub mod traits;
 pub mod types;
@@ -51,11 +52,11 @@ pub use pallet::*;
 
 #[frame_support::pallet]
 mod pallet {
-    use crate::{types::Proposal, weights::WeightInfoZeitgeist};
+    use crate::{traits::ProposalStorage, types::Proposal, weights::WeightInfoZeitgeist};
     use alloc::fmt::Debug;
     use core::marker::PhantomData;
     use frame_support::{
-        pallet_prelude::{IsType, StorageMap, StorageVersion, ValueQuery, Weight},
+        pallet_prelude::{IsType, StorageMap, StorageValue, StorageVersion, ValueQuery, Weight},
         traits::{schedule::v3::Anon as ScheduleAnon, Bounded, Hooks, OriginTrait},
         transactional, Blake2_128Concat, BoundedVec,
     };
@@ -65,10 +66,7 @@ mod pallet {
     };
     use parity_scale_codec::{Decode, Encode, MaxEncodedLen};
     use scale_info::TypeInfo;
-    use sp_runtime::{
-        traits::{ConstU32, Get},
-        DispatchResult,
-    };
+    use sp_runtime::{traits::Get, DispatchResult, SaturatedConversion};
     use zeitgeist_primitives::traits::FutarchyOracle;
 
     #[cfg(feature = "runtime-benchmarks")]
@@ -79,10 +77,13 @@ mod pallet {
         #[cfg(feature = "runtime-benchmarks")]
         type BenchmarkHelper: FutarchyBenchmarkHelper<Self::Oracle>;
 
+        /// The maximum number of proposals allowed to be in flight simultaneously.
+        type MaxProposals: Get<u32>;
+
         type MinDuration: Get<BlockNumberFor<Self>>;
 
         // The type used to define the oracle for each proposal.
-        type Oracle: FutarchyOracle
+        type Oracle: FutarchyOracle<BlockNumber = BlockNumberFor<Self>>
             + Clone
             + Debug
             + Decode
@@ -104,23 +105,21 @@ mod pallet {
     #[pallet::storage_version(STORAGE_VERSION)]
     pub struct Pallet<T>(PhantomData<T>);
 
-    pub(crate) type CacheSize = ConstU32<16>;
     pub(crate) type CallOf<T> = <T as frame_system::Config>::RuntimeCall;
     pub(crate) type BoundedCallOf<T> = Bounded<CallOf<T>>;
     pub(crate) type OracleOf<T> = <T as Config>::Oracle;
     pub(crate) type PalletsOriginOf<T> =
         <<T as frame_system::Config>::RuntimeOrigin as OriginTrait>::PalletsOrigin;
+    pub(crate) type ProposalsOf<T> = BoundedVec<Proposal<T>, <T as Config>::MaxProposals>;
 
     pub(crate) const STORAGE_VERSION: StorageVersion = StorageVersion::new(0);
 
     #[pallet::storage]
-    pub type Proposals<T: Config> = StorageMap<
-        _,
-        Blake2_128Concat,
-        BlockNumberFor<T>,
-        BoundedVec<Proposal<T>, CacheSize>,
-        ValueQuery,
-    >;
+    pub type Proposals<T: Config> =
+        StorageMap<_, Blake2_128Concat, BlockNumberFor<T>, ProposalsOf<T>, ValueQuery>;
+
+    #[pallet::storage]
+    pub type ProposalCount<T: Config> = StorageValue<_, u32, ValueQuery>;
 
     #[pallet::event]
     #[pallet::generate_deposit(pub(crate) fn deposit_event)]
@@ -148,6 +147,9 @@ mod pallet {
 
         /// The specified duration must be at least equal to `MinDuration`.
         DurationTooShort,
+
+        /// This is a logic error. You shouldn't see this.
+        UnexpectedStorageFailure,
     }
 
     #[pallet::call]
@@ -169,10 +171,34 @@ mod pallet {
     #[pallet::hooks]
     impl<T: Config> Hooks<BlockNumberFor<T>> for Pallet<T> {
         fn on_initialize(now: BlockNumberFor<T>) -> Weight {
-            let mut total_weight = Weight::zero(); // Add buffer.
+            let mut total_weight = Weight::zero();
+
+            // Update all oracles.
+            let mutate_all_result =
+                <Pallet<T> as ProposalStorage<T>>::mutate_all(|p| p.oracle.update(now));
+            if let Ok(block_to_weights) = mutate_all_result {
+                // We did one storage read per vector cached. Shouldn't saturate, but technically
+                // might.
+                let reads: u64 = block_to_weights.len().saturated_into();
+                total_weight = total_weight.saturating_add(T::DbWeight::get().reads(reads));
+
+                for weights in block_to_weights.values() {
+                    for &weight in weights.iter() {
+                        total_weight = total_weight.saturating_add(weight);
+                    }
+                }
+            } else {
+                // Unreachable!
+                return total_weight;
+            }
 
-            let proposals = Proposals::<T>::take(now);
+            //
             total_weight = total_weight.saturating_add(T::DbWeight::get().reads_writes(1, 1));
+            let proposals = if let Ok(proposals) = <Pallet<T> as ProposalStorage<T>>::take(now) {
+                proposals
+            } else {
+                return total_weight;
+            };
 
             for proposal in proposals.into_iter() {
                 let weight = Self::maybe_schedule_proposal(proposal);

zrml/futarchy/src/mock/runtime.rs

@@ -33,6 +33,7 @@ use crate::mock::types::MockBenchmarkHelper;
 
 parameter_types! {
     // zrml-futarchy
+    pub const MaxProposals: u32 = 16;
     pub const MinDuration: BlockNumber = 10;
 }
 
@@ -89,6 +90,7 @@ impl pallet_balances::Config for Runtime {
 impl zrml_futarchy::Config for Runtime {
     #[cfg(feature = "runtime-benchmarks")]
     type BenchmarkHelper = MockBenchmarkHelper;
+    type MaxProposals = MaxProposals;
     type MinDuration = MinDuration;
     type Oracle = MockOracle;
     type RuntimeEvent = RuntimeEvent;

zrml/futarchy/src/mock/types/oracle.rs

@@ -19,7 +19,8 @@ use alloc::fmt::Debug;
 use frame_support::pallet_prelude::Weight;
 use parity_scale_codec::{Decode, Encode, MaxEncodedLen};
 use scale_info::TypeInfo;
-use zeitgeist_primitives::traits::FutarchyOracle;
+use sp_runtime::traits::Zero;
+use zeitgeist_primitives::{traits::FutarchyOracle, types::BlockNumber};
 
 #[derive(Clone, Debug, Decode, Encode, Eq, MaxEncodedLen, PartialEq, TypeInfo)]
 pub struct MockOracle {
@@ -40,7 +41,13 @@ impl MockOracle {
 }
 
 impl FutarchyOracle for MockOracle {
+    type BlockNumber = BlockNumber;
+
     fn evaluate(&self) -> (Weight, bool) {
         (self.weight, self.value)
     }
+
+    fn update(&mut self, _: Self::BlockNumber) -> Weight {
+        Zero::zero()
+    }
 }

zrml/futarchy/src/proposal_storage.rs

@@ -0,0 +1,88 @@
+use crate::{
+    traits::ProposalStorage, types::Proposal, Config, Error, Pallet, ProposalCount, Proposals,
+    ProposalsOf,
+};
+use alloc::{collections::BTreeMap, vec, vec::Vec};
+use frame_support::{ensure, require_transactional, traits::Get};
+use frame_system::pallet_prelude::BlockNumberFor;
+use sp_runtime::{DispatchError, SaturatedConversion};
+use zeitgeist_primitives::math::checked_ops_res::{CheckedIncRes, CheckedSubRes};
+
+impl<T> ProposalStorage<T> for Pallet<T>
+where
+    T: Config,
+{
+    fn count() -> u32 {
+        ProposalCount::<T>::get()
+    }
+
+    #[require_transactional]
+    fn add(block_number: BlockNumberFor<T>, proposal: Proposal<T>) -> Result<(), DispatchError> {
+        let proposal_count = ProposalCount::<T>::get();
+        ensure!(proposal_count < T::MaxProposals::get(), Error::<T>::CacheFull);
+
+        let new_proposal_count = proposal_count.checked_inc_res()?;
+        ProposalCount::<T>::put(new_proposal_count);
+
+        // Can't error unless state is invalid.
+        let mutate_result = Proposals::<T>::try_mutate(block_number, |proposals| {
+            proposals.try_push(proposal).map_err(|_| Error::<T>::CacheFull)
+        });
+
+        Ok(mutate_result?)
+    }
+
+    /// Take all proposals scheduled at `block_number`.
+    fn take(block_number: BlockNumberFor<T>) -> Result<ProposalsOf<T>, DispatchError> {
+        let proposals = Proposals::<T>::take(block_number);
+
+        // Can't error unless state is invalid.
+        let proposal_count = ProposalCount::<T>::get();
+        let proposals_len: u32 = proposals.len().try_into().map_err(|_| Error::<T>::CacheFull)?;
+        let new_proposal_count = proposal_count.checked_sub_res(&proposals_len)?;
+        ProposalCount::<T>::put(new_proposal_count);
+
+        Ok(proposals)
+    }
+
+    /// Returns all proposals scheduled at `block_number`.
+    fn get(block_number: BlockNumberFor<T>) -> ProposalsOf<T> {
+        Proposals::<T>::get(block_number)
+    }
+
+    fn mutate_all<R, F>(
+        mut mutator: F,
+    ) -> Result<BTreeMap<BlockNumberFor<T>, Vec<R>>, DispatchError>
+    where
+        F: FnMut(&mut Proposal<T>) -> R,
+    {
+        // Collect keys to avoid iterating over the keys whilst modifying the map. Won't saturate
+        // unless `usize` has fewer bits than `u32` for some reason.
+        let keys: Vec<_> =
+            Proposals::<T>::iter_keys().take(T::MaxProposals::get().saturated_into()).collect();
+
+        let mut result_map = BTreeMap::new();
+
+        for k in keys.into_iter() {
+            let proposals = Self::get(k);
+
+            let mut results = vec![];
+
+            // If mutation goes out of bounds, we've clearly failed.
+            let proposals = proposals
+                .try_mutate(|v| {
+                    for p in v.iter_mut() {
+                        let r = mutator(p);
+                        results.push(r);
+                    }
+                })
+                .ok_or(Error::<T>::UnexpectedStorageFailure)?;
+
+            result_map.insert(k, results);
+
+            Proposals::<T>::insert(k, proposals);
+        }
+
+        Ok(result_map)
+    }
+}

zrml/futarchy/src/tests/mod.rs

@@ -27,14 +27,14 @@ use crate::{
         utility,
     },
     types::Proposal,
-    CacheSize, Config, Error, Event, Proposals,
+    Config, Error, Event, Proposals, ProposalsOf,
 };
 use frame_support::{
     assert_noop, assert_ok,
     dispatch::RawOrigin,
     traits::{schedule::DispatchTime, Bounded},
 };
-use sp_runtime::{traits::Get, BoundedVec, DispatchError};
+use sp_runtime::DispatchError;
 
 /// Utility struct for managing test accounts.
 pub(crate) struct Account {

zrml/futarchy/src/tests/submit_proposal.rs

@@ -134,9 +134,9 @@ fn submit_proposal_fails_if_cache_is_full() {
         // Mock up a full vector of proposals.
         let now = System::block_number();
         let to_be_scheduled_at = now + duration;
-        let cache_size: u32 = <CacheSize as Get<Option<u32>>>::get().unwrap();
-        let proposals_vec = vec![proposal.clone(); cache_size as usize];
-        let proposals: BoundedVec<_, CacheSize> = proposals_vec.try_into().unwrap();
+        let max_proposals: u32 = <Runtime as Config>::MaxProposals::get();
+        let proposals_vec = vec![proposal.clone(); max_proposals as usize];
+        let proposals: ProposalsOf<Runtime> = proposals_vec.try_into().unwrap();
         Proposals::<Runtime>::insert(to_be_scheduled_at, proposals);
 
         assert_noop!(

zrml/futarchy/src/traits/mod.rs

@@ -14,3 +14,7 @@
 //
 // You should have received a copy of the GNU General Public License
 // along with Zeitgeist. If not, see <https://www.gnu.org/licenses/>.
+
+mod proposal_storage;
+
+pub(crate) use proposal_storage::ProposalStorage;