From 7bd91e826e4b2d95f6d2ee7f6bb0f0f47d095859 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Sat, 21 Oct 2023 23:31:09 +0100 Subject: [PATCH] Update Polkadot ideal staking rate (#26) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update Polkadot inflation * fmr * add test to dispatch all proposals * Update relay/polkadot/src/lib.rs Co-authored-by: Gonçalo Pestana * Bump spec versions * Update changelog and make it release * Bump Kusama `transaction_version` The new society is at the same index as the old one, but has different `calls`. * Update `xcm-builder` * Update relay/polkadot/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * Be optimistic!! * Upgrade staging xcm to get the fix for the metadata * Add a test * Adds custom paras scheduler migration This is a backport from polkadot-sdk master that includes some fixes. * Also copy the migration over to Kusama * Update CHANGELOG --------- Co-authored-by: Gonçalo Pestana Co-authored-by: Bastian Köcher Co-authored-by: Bastian Köcher Co-authored-by: Oliver Tale-Yazdi --- CHANGELOG.md | 3 +- Cargo.lock | 17 +- relay/kusama/Cargo.toml | 4 +- relay/kusama/src/lib.rs | 8 +- relay/kusama/src/paras_scheduler_migration.rs | 218 ++++++++++++++++++ relay/polkadot/Cargo.toml | 6 +- relay/polkadot/src/lib.rs | 144 ++++++++++-- .../polkadot/src/paras_scheduler_migration.rs | 218 ++++++++++++++++++ .../asset-hubs/asset-hub-kusama/Cargo.toml | 4 +- .../asset-hubs/asset-hub-kusama/src/lib.rs | 4 +- .../asset-hubs/asset-hub-polkadot/Cargo.toml | 4 +- .../asset-hubs/asset-hub-polkadot/src/lib.rs | 2 +- .../bridge-hubs/bridge-hub-kusama/Cargo.toml | 4 +- .../bridge-hubs/bridge-hub-kusama/src/lib.rs | 2 +- .../bridge-hub-polkadot/Cargo.toml | 4 +- .../bridge-hub-polkadot/src/lib.rs | 2 +- .../collectives-polkadot/Cargo.toml | 4 +- .../collectives-polkadot/src/lib.rs | 2 +- .../gluttons/glutton-kusama/Cargo.toml | 4 +- .../gluttons/glutton-kusama/src/lib.rs | 2 +- 20 files changed, 604 insertions(+), 52 deletions(-) create mode 100644 relay/kusama/src/paras_scheduler_migration.rs create mode 100644 relay/polkadot/src/paras_scheduler_migration.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc6143dc8..bcfa459650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,11 @@ Changelog for the runtimes governed by the Polkadot Fellowship. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [Unreleased] +## [1.0.0] 22.10.2023 ### Changed +- Update Polkadot ideal staking rate ([polkadot-fellows/runtimes#26](https://github.com/polkadot-fellows/runtimes/pull/26)) - Treasury deprecate `propose_spend` dispatchable ([paritytech/substrate#14538](https://github.com/paritytech/substrate/pull/14538)) - Use benchmarked weights for `XCM` ([paritytech/polkadot#7077](https://github.com/paritytech/polkadot/pull/7077)) - Put HRMP Channel Management on General Admin Track ([paritytech/polkadot#7477](https://github.com/paritytech/polkadot/pull/7477)) diff --git a/Cargo.lock b/Cargo.lock index 9ce4f24cb9..c9865981cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7865,6 +7865,7 @@ dependencies = [ "pallet-session-benchmarking", "pallet-staking", "pallet-staking-reward-curve", + "pallet-staking-reward-fn", "pallet-staking-runtime-api", "pallet-timestamp", "pallet-tips", @@ -9789,9 +9790,9 @@ dependencies = [ [[package]] name = "scale-info" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0a159d0c45c12b20c5a844feb1fe4bea86e28f17b92a5f0c42193634d3782" +checksum = "7f7d66a1128282b7ef025a8ead62a4a9fcf017382ec53b8ffbf4d7bf77bd3c60" dependencies = [ "bitvec", "cfg-if", @@ -9803,9 +9804,9 @@ dependencies = [ [[package]] name = "scale-info-derive" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912e55f6d20e0e80d63733872b40e1227c0bce1e1ab81ba67d696339bfd7fd29" +checksum = "abf2c68b89cafb3b8d918dd07b42be0da66ff202cf1155c5739a4e0c1ea0dc19" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -11236,9 +11237,9 @@ dependencies = [ [[package]] name = "staging-xcm" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1712d3ab65f9f09c2f7c281106d45589182e71950279b98db83ce89125067d26" +checksum = "eec456fd5fcbc4dffe9c6042b452c1930eb1d5af5534d6ef36b8238b4517c9b7" dependencies = [ "bounded-collections", "derivative", @@ -11254,9 +11255,9 @@ dependencies = [ [[package]] name = "staging-xcm-builder" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd00b720a97ad1304684c07eea14fb3615dd2e26569d29876e9b40ff9c334eca" +checksum = "07c1ca6d8f2b7fcbfe8866c1a1cb8105b62c72a74e727dd8c9943e8ac0c410eb" dependencies = [ "frame-support", "frame-system", diff --git a/relay/kusama/Cargo.toml b/relay/kusama/Cargo.toml index 76eb17ea5e..7911a54974 100644 --- a/relay/kusama/Cargo.toml +++ b/relay/kusama/Cargo.toml @@ -109,9 +109,9 @@ runtime-common = { package = "polkadot-runtime-common", default-features = false runtime-parachains = { package = "polkadot-runtime-parachains", default-features = false , version = "2.0.0" } primitives = { package = "polkadot-primitives", default-features = false , version = "2.0.0" } -xcm = { package = "staging-xcm", default-features = false , version = "2.0.0" } +xcm = { package = "staging-xcm", default-features = false , version = "2.0.1" } xcm-executor = { package = "staging-xcm-executor", default-features = false , version = "2.0.0" } -xcm-builder = { package = "staging-xcm-builder", default-features = false , version = "2.0.0" } +xcm-builder = { package = "staging-xcm-builder", default-features = false , version = "2.0.1" } [dev-dependencies] tiny-keccak = { version = "2.0.2", features = ["keccak"] } diff --git a/relay/kusama/src/lib.rs b/relay/kusama/src/lib.rs index 659a7052d2..8900d60d0b 100644 --- a/relay/kusama/src/lib.rs +++ b/relay/kusama/src/lib.rs @@ -109,6 +109,8 @@ mod weights; // Voter bag threshold definitions. mod bag_thresholds; +mod paras_scheduler_migration; + // Historical information of society finances. mod past_payouts; @@ -137,10 +139,10 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("kusama"), impl_name: create_runtime_str!("parity-kusama"), authoring_version: 2, - spec_version: 9430, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, - transaction_version: 23, + transaction_version: 24, state_version: 1, }; @@ -1734,7 +1736,7 @@ pub mod migrations { >, pallet_im_online::migration::v1::Migration, parachains_configuration::migration::v7::MigrateToV7, - parachains_scheduler::migration::v1::MigrateToV1, + paras_scheduler_migration::v1::MigrateToV1, parachains_configuration::migration::v8::MigrateToV8, // Unlock/unreserve balances from Gov v1 pallets that hold them diff --git a/relay/kusama/src/paras_scheduler_migration.rs b/relay/kusama/src/paras_scheduler_migration.rs new file mode 100644 index 0000000000..d294861843 --- /dev/null +++ b/relay/kusama/src/paras_scheduler_migration.rs @@ -0,0 +1,218 @@ +//! A copy of the migration found in the polkadot sdk repo. +//! +//! It is copied as the version of the migration found in the crate used by this runtime is broken. + +use frame_support::{ + migrations::VersionedMigration, pallet_prelude::ValueQuery, storage_alias, + traits::OnRuntimeUpgrade, weights::Weight, +}; +use frame_system::pallet_prelude::BlockNumberFor; +use parity_scale_codec::{Decode, Encode}; +use primitives::{ + v5::{Assignment, ParasEntry}, + CoreIndex, CoreOccupied, GroupIndex, Id as ParaId, +}; +use runtime_parachains::scheduler::*; +use scale_info::TypeInfo; +use sp_core::{Get, RuntimeDebug}; +use sp_std::{ + collections::{btree_map::BTreeMap, vec_deque::VecDeque}, + prelude::*, +}; + +const LOG_TARGET: &str = "runtime::parachains::scheduler"; + +mod v0 { + use super::*; + + use primitives::{CollatorId, Id}; + + #[storage_alias] + pub(super) type Scheduled = StorageValue, Vec, ValueQuery>; + + #[derive(Clone, Encode, Decode)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub struct ParathreadClaim(pub Id, pub CollatorId); + + #[derive(Clone, Encode, Decode)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub struct ParathreadEntry { + /// The claim. + pub claim: ParathreadClaim, + /// Number of retries. + pub retries: u32, + } + + /// What is occupying a specific availability core. + #[derive(Clone, Encode, Decode)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub enum CoreOccupied { + /// A parathread. + Parathread(ParathreadEntry), + /// A parachain. + Parachain, + } + + /// The actual type isn't important, as we only delete the key in the state. + #[storage_alias] + pub(crate) type AvailabilityCores = + StorageValue, Vec>, ValueQuery>; + + /// The actual type isn't important, as we only delete the key in the state. + #[storage_alias] + pub(super) type ParathreadQueue = StorageValue, (), ValueQuery>; + + #[storage_alias] + pub(super) type ParathreadClaimIndex = StorageValue, (), ValueQuery>; + + /// The assignment type. + #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub enum AssignmentKind { + /// A parachain. + Parachain, + /// A parathread. + Parathread(CollatorId, u32), + } + + /// How a free core is scheduled to be assigned. + #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub struct CoreAssignment { + /// The core that is assigned. + pub core: CoreIndex, + /// The unique ID of the para that is assigned to the core. + pub para_id: ParaId, + /// The kind of the assignment. + pub kind: AssignmentKind, + /// The index of the validator group assigned to the core. + pub group_idx: GroupIndex, + } +} + +pub mod v1 { + use super::*; + + #[storage_alias] + pub(crate) type AvailabilityCores = + StorageValue, Vec>>, ValueQuery>; + + #[storage_alias] + pub(crate) type ClaimQueue = StorageValue< + Pallet, + BTreeMap>>>>, + ValueQuery, + >; + + #[allow(deprecated)] + pub type MigrateToV1 = VersionedMigration< + 0, + 1, + UncheckedMigrateToV1, + Pallet, + ::DbWeight, + >; + + #[deprecated(note = "Use MigrateToV1 instead")] + pub struct UncheckedMigrateToV1(sp_std::marker::PhantomData); + #[allow(deprecated)] + impl OnRuntimeUpgrade for UncheckedMigrateToV1 { + fn on_runtime_upgrade() -> Weight { + let weight_consumed = migrate_to_v1::(); + + log::info!(target: LOG_TARGET, "Migrating para scheduler storage to v1"); + + weight_consumed + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::DispatchError> { + let n: u32 = v0::Scheduled::::get().len() as u32 + + v0::AvailabilityCores::::get().iter().filter(|c| c.is_some()).count() as u32; + + log::info!( + target: LOG_TARGET, + "Number of scheduled and waiting for availability before: {n}", + ); + + Ok(n.encode()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(state: Vec) -> Result<(), sp_runtime::DispatchError> { + log::info!(target: LOG_TARGET, "Running post_upgrade()"); + + frame_support::ensure!( + v0::Scheduled::::get().is_empty(), + "Scheduled should be empty after the migration" + ); + + let expected_len = u32::decode(&mut &state[..]).unwrap(); + let availability_cores_waiting = AvailabilityCores::::get() + .iter() + .filter(|c| !matches!(c, CoreOccupied::Free)) + .count(); + + frame_support::ensure!( + ClaimQueue::::get().iter().map(|la_vec| la_vec.1.len()).sum::() as u32 + + availability_cores_waiting as u32 == + expected_len, + "ClaimQueue and AvailabilityCores should have the correct length", + ); + + Ok(()) + } + } +} + +pub fn migrate_to_v1() -> Weight { + let mut weight: Weight = Weight::zero(); + + v0::ParathreadQueue::::kill(); + v0::ParathreadClaimIndex::::kill(); + + let now = >::block_number(); + let scheduled = v0::Scheduled::::take(); + let sched_len = scheduled.len() as u64; + for core_assignment in scheduled { + let core_idx = core_assignment.core; + let assignment = Assignment::new(core_assignment.para_id); + let pe = ParasEntry::new(assignment, now); + + v1::ClaimQueue::::mutate(|la| { + la.entry(core_idx).or_default().push_back(Some(pe)); + }); + } + + let parachains = runtime_parachains::paras::Pallet::::parachains(); + let availability_cores = v0::AvailabilityCores::::take(); + let mut new_availability_cores = Vec::new(); + + for (core_index, core) in availability_cores.into_iter().enumerate() { + let new_core = if let Some(core) = core { + match core { + v0::CoreOccupied::Parachain => CoreOccupied::Paras(ParasEntry::new( + Assignment::new(parachains[core_index]), + now, + )), + v0::CoreOccupied::Parathread(entry) => + CoreOccupied::Paras(ParasEntry::new(Assignment::new(entry.claim.0), now)), + } + } else { + CoreOccupied::Free + }; + + new_availability_cores.push(new_core); + } + + v1::AvailabilityCores::::set(new_availability_cores); + + // 2x as once for Scheduled and once for Claimqueue + weight = weight.saturating_add(T::DbWeight::get().reads_writes(2 * sched_len, 2 * sched_len)); + // reading parachains + availability_cores, writing AvailabilityCores + weight = weight.saturating_add(T::DbWeight::get().reads_writes(2, 1)); + // 2x kill + weight = weight.saturating_add(T::DbWeight::get().writes(2)); + + weight +} diff --git a/relay/polkadot/Cargo.toml b/relay/polkadot/Cargo.toml index f8e7ded089..3c72f30e09 100644 --- a/relay/polkadot/Cargo.toml +++ b/relay/polkadot/Cargo.toml @@ -71,6 +71,7 @@ pallet-scheduler = { default-features = false , version = "24.0.0" } pallet-session = { default-features = false , version = "23.0.0" } frame-support = { default-features = false , version = "23.0.0" } pallet-staking = { default-features = false , version = "23.0.0" } +pallet-staking-reward-fn = { default-features = false, version = "14.0.0" } pallet-staking-reward-curve = { version = "8.0.0" } pallet-staking-runtime-api = { default-features = false , version = "9.0.0" } frame-system = { default-features = false , version = "23.0.0" } @@ -99,9 +100,9 @@ runtime-common = { package = "polkadot-runtime-common", default-features = false runtime-parachains = { package = "polkadot-runtime-parachains", default-features = false , version = "2.0.0" } primitives = { package = "polkadot-primitives", default-features = false , version = "2.0.0" } -xcm = { package = "staging-xcm", default-features = false , version = "2.0.0" } +xcm = { package = "staging-xcm", default-features = false , version = "2.0.1" } xcm-executor = { package = "staging-xcm-executor", default-features = false , version = "2.0.0" } -xcm-builder = { package = "staging-xcm-builder", default-features = false , version = "2.0.0" } +xcm-builder = { package = "staging-xcm-builder", default-features = false , version = "2.0.1" } [dev-dependencies] hex-literal = "0.4.1" @@ -171,6 +172,7 @@ std = [ "pallet-session-benchmarking?/std", "pallet-session/std", "pallet-staking-runtime-api/std", + "pallet-staking-reward-fn/std", "pallet-staking/std", "pallet-timestamp/std", "pallet-tips/std", diff --git a/relay/polkadot/src/lib.rs b/relay/polkadot/src/lib.rs index 45ea561b33..97ceedb0cd 100644 --- a/relay/polkadot/src/lib.rs +++ b/relay/polkadot/src/lib.rs @@ -106,6 +106,8 @@ mod weights; mod bag_thresholds; +mod paras_scheduler_migration; + // Governance configurations. pub mod governance; use governance::{ @@ -115,6 +117,8 @@ use governance::{ pub mod xcm_config; +pub const LOG_TARGET: &'static str = "runtime::polkadot"; + impl_runtime_weights!(polkadot_runtime_constants); // Make the WASM binary available. @@ -128,7 +132,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("polkadot"), impl_name: create_runtime_str!("parity-polkadot"), authoring_version: 0, - spec_version: 9430, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 24, @@ -537,6 +541,54 @@ parameter_types! { pub const MaxNominations: u32 = ::LIMIT as u32; } +/// Custom version of `runtime_commong::era_payout` somewhat tailored for Polkadot's crowdloan +/// unlock history. The only tweak should be +/// +/// ```diff +/// - let auction_proportion = Perquintill::from_rational(auctioned_slots.min(60), 200u64); +/// + let auction_proportion = Perquintill::from_rational(auctioned_slots.min(60), 300u64); +/// ``` +/// +/// See . +fn polkadot_era_payout( + total_staked: Balance, + total_stakable: Balance, + max_annual_inflation: Perquintill, + period_fraction: Perquintill, + auctioned_slots: u64, +) -> (Balance, Balance) { + use pallet_staking_reward_fn::compute_inflation; + use sp_runtime::traits::Saturating; + + let min_annual_inflation = Perquintill::from_rational(25u64, 1000u64); + let delta_annual_inflation = max_annual_inflation.saturating_sub(min_annual_inflation); + + // 20% reserved for up to 60 slots. + let auction_proportion = Perquintill::from_rational(auctioned_slots.min(60), 300u64); + + // Therefore the ideal amount at stake (as a percentage of total issuance) is 75% less the + // amount that we expect to be taken up with auctions. + let ideal_stake = Perquintill::from_percent(75).saturating_sub(auction_proportion); + + let stake = Perquintill::from_rational(total_staked, total_stakable); + let falloff = Perquintill::from_percent(5); + let adjustment = compute_inflation(stake, ideal_stake, falloff); + let staking_inflation = + min_annual_inflation.saturating_add(delta_annual_inflation * adjustment); + + let max_payout = period_fraction * max_annual_inflation * total_stakable; + let staking_payout = (period_fraction * staking_inflation) * total_stakable; + let rest = max_payout.saturating_sub(staking_payout); + + let other_issuance = total_stakable.saturating_sub(total_staked); + if total_staked > other_issuance { + let _cap_rest = Perquintill::from_rational(other_issuance, total_staked) * staking_payout; + // We don't do anything with this, but if we wanted to, we could introduce a cap on the + // treasury amount with: `rest = rest.min(cap_rest);` + } + (staking_payout, rest) +} + pub struct EraPayout; impl pallet_staking::EraPayout for EraPayout { fn era_payout( @@ -555,7 +607,7 @@ impl pallet_staking::EraPayout for EraPayout { const MAX_ANNUAL_INFLATION: Perquintill = Perquintill::from_percent(10); const MILLISECONDS_PER_YEAR: u64 = 1000 * 3600 * 24 * 36525 / 100; - runtime_common::impls::era_payout( + polkadot_era_payout( total_staked, total_issuance, MAX_ANNUAL_INFLATION, @@ -795,7 +847,7 @@ where ); let raw_payload = SignedPayload::new(call, extra) .map_err(|e| { - log::warn!("Unable to create signed payload: {:?}", e); + log::warn!(target: LOG_TARGET, "Unable to create signed payload: {:?}", e); }) .ok()?; let signature = raw_payload.using_encoded(|payload| C::sign(payload, public))?; @@ -1311,10 +1363,10 @@ impl frame_support::traits::OnRuntimeUpgrade for InitiateNominationPools { pallet_nomination_pools::MaxPoolMembersPerPool::::put(0); pallet_nomination_pools::MaxPoolMembers::::put(0); - log::info!(target: "runtime::polkadot", "pools config initiated 🎉"); + log::info!(target: LOG_TARGET, "pools config initiated 🎉"); ::DbWeight::get().reads_writes(1, 5) } else { - log::info!(target: "runtime::polkadot", "pools config already initiated 😏"); + log::info!(target: LOG_TARGET, "pools config already initiated 😏"); ::DbWeight::get().reads(1) } } @@ -1521,7 +1573,7 @@ pub mod migrations { pub type Unreleased = ( pallet_im_online::migration::v1::Migration, parachains_configuration::migration::v7::MigrateToV7, - parachains_scheduler::migration::v1::MigrateToV1, + crate::paras_scheduler_migration::v1::MigrateToV1, parachains_configuration::migration::v8::MigrateToV8, // Gov v1 storage migrations @@ -2045,7 +2097,7 @@ sp_api::impl_runtime_apis! { #[cfg(feature = "try-runtime")] impl frame_try_runtime::TryRuntime for Runtime { fn on_runtime_upgrade(checks: frame_try_runtime::UpgradeCheckSelect) -> (Weight, Weight) { - log::info!("try-runtime::on_runtime_upgrade polkadot."); + log::info!(target: LOG_TARGET, "try-runtime::on_runtime_upgrade polkadot."); let weight = Executive::try_runtime_upgrade(checks).unwrap(); (weight, BlockWeights::get().max_block) } @@ -2408,6 +2460,7 @@ mod multiplier_tests { use super::*; use frame_support::{dispatch::DispatchInfo, traits::OnFinalize}; use runtime_common::{MinimumMultiplier, TargetBlockFullness}; + use scale_info::TypeInfo; use separator::Separatable; use sp_runtime::traits::Convert; @@ -2530,6 +2583,13 @@ mod multiplier_tests { blocks += 1; } } + + #[test] + fn ensure_xcm_metadata_is_correct() { + let path = xcm::VersionedXcm::<()>::type_info().path; + // Ensure that the name doesn't include `staging` (from the pallet name) + assert_eq!(vec!["xcm", "VersionedXcm"], path.segments); + } } #[cfg(all(test, feature = "try-runtime"))] @@ -2537,21 +2597,15 @@ mod remote_tests { use super::*; use frame_try_runtime::{runtime_decl_for_try_runtime::TryRuntime, UpgradeCheckSelect}; use remote_externalities::{ - Builder, Mode, OfflineConfig, OnlineConfig, SnapshotConfig, Transport, + Builder, Mode, OfflineConfig, OnlineConfig, RemoteExternalities, SnapshotConfig, Transport, }; use std::env::var; - #[tokio::test] - async fn run_migrations() { - if var("RUN_MIGRATION_TESTS").is_err() { - return - } - - sp_tracing::try_init_simple(); + async fn remote_ext_test_setup() -> RemoteExternalities { let transport: Transport = var("WS").unwrap_or("wss://rpc.polkadot.io:443".to_string()).into(); let maybe_state_snapshot: Option = var("SNAP").map(|s| s.into()).ok(); - let mut ext = Builder::::default() + Builder::::default() .mode(if let Some(state_snapshot) = maybe_state_snapshot { Mode::OfflineOrElseOnline( OfflineConfig { state_snapshot: state_snapshot.clone() }, @@ -2566,7 +2620,63 @@ mod remote_tests { }) .build() .await - .unwrap(); + .unwrap() + } + + #[tokio::test] + async fn dispatch_all_proposals() { + if var("RUN_OPENGOV_TEST").is_err() { + return + } + + sp_tracing::try_init_simple(); + let mut ext = remote_ext_test_setup().await; + ext.execute_with(|| { + type Ref = pallet_referenda::ReferendumInfoOf; + type RefStatus = pallet_referenda::ReferendumStatusOf; + use sp_runtime::traits::Dispatchable; + let all_refs: Vec<(u32, RefStatus)> = + pallet_referenda::ReferendumInfoFor::::iter() + .filter_map(|(idx, reff): (_, Ref)| { + if let Ref::Ongoing(ref_status) = reff { + Some((idx, ref_status)) + } else { + None + } + }) + .collect::>(); + + for (ref_index, referenda) in all_refs { + log::info!(target: LOG_TARGET, "🚀 executing referenda #{}", ref_index); + let RefStatus { origin, proposal, .. } = referenda; + // we do more or less what the scheduler will do under the hood, as best as we can + // imitate: + let (call, _len) = match < + ::Preimages + as + frame_support::traits::QueryPreimage + >::peek(&proposal) { + Ok(x) => x, + Err(e) => { + log::error!(target: LOG_TARGET, "failed to get preimage: {:?}", e); + continue; + } + }; + + let dispatch_result = call.dispatch(origin.clone().into()); + log::info!(target: LOG_TARGET, "outcome of dispatch with origin {:?}: {:?}", origin, dispatch_result); + } + }); + } + + #[tokio::test] + async fn run_migrations() { + if var("RUN_MIGRATION_TESTS").is_err() { + return + } + + sp_tracing::try_init_simple(); + let mut ext = remote_ext_test_setup().await; ext.execute_with(|| Runtime::on_runtime_upgrade(UpgradeCheckSelect::PreAndPost)); } diff --git a/relay/polkadot/src/paras_scheduler_migration.rs b/relay/polkadot/src/paras_scheduler_migration.rs new file mode 100644 index 0000000000..d294861843 --- /dev/null +++ b/relay/polkadot/src/paras_scheduler_migration.rs @@ -0,0 +1,218 @@ +//! A copy of the migration found in the polkadot sdk repo. +//! +//! It is copied as the version of the migration found in the crate used by this runtime is broken. + +use frame_support::{ + migrations::VersionedMigration, pallet_prelude::ValueQuery, storage_alias, + traits::OnRuntimeUpgrade, weights::Weight, +}; +use frame_system::pallet_prelude::BlockNumberFor; +use parity_scale_codec::{Decode, Encode}; +use primitives::{ + v5::{Assignment, ParasEntry}, + CoreIndex, CoreOccupied, GroupIndex, Id as ParaId, +}; +use runtime_parachains::scheduler::*; +use scale_info::TypeInfo; +use sp_core::{Get, RuntimeDebug}; +use sp_std::{ + collections::{btree_map::BTreeMap, vec_deque::VecDeque}, + prelude::*, +}; + +const LOG_TARGET: &str = "runtime::parachains::scheduler"; + +mod v0 { + use super::*; + + use primitives::{CollatorId, Id}; + + #[storage_alias] + pub(super) type Scheduled = StorageValue, Vec, ValueQuery>; + + #[derive(Clone, Encode, Decode)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub struct ParathreadClaim(pub Id, pub CollatorId); + + #[derive(Clone, Encode, Decode)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub struct ParathreadEntry { + /// The claim. + pub claim: ParathreadClaim, + /// Number of retries. + pub retries: u32, + } + + /// What is occupying a specific availability core. + #[derive(Clone, Encode, Decode)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub enum CoreOccupied { + /// A parathread. + Parathread(ParathreadEntry), + /// A parachain. + Parachain, + } + + /// The actual type isn't important, as we only delete the key in the state. + #[storage_alias] + pub(crate) type AvailabilityCores = + StorageValue, Vec>, ValueQuery>; + + /// The actual type isn't important, as we only delete the key in the state. + #[storage_alias] + pub(super) type ParathreadQueue = StorageValue, (), ValueQuery>; + + #[storage_alias] + pub(super) type ParathreadClaimIndex = StorageValue, (), ValueQuery>; + + /// The assignment type. + #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub enum AssignmentKind { + /// A parachain. + Parachain, + /// A parathread. + Parathread(CollatorId, u32), + } + + /// How a free core is scheduled to be assigned. + #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub struct CoreAssignment { + /// The core that is assigned. + pub core: CoreIndex, + /// The unique ID of the para that is assigned to the core. + pub para_id: ParaId, + /// The kind of the assignment. + pub kind: AssignmentKind, + /// The index of the validator group assigned to the core. + pub group_idx: GroupIndex, + } +} + +pub mod v1 { + use super::*; + + #[storage_alias] + pub(crate) type AvailabilityCores = + StorageValue, Vec>>, ValueQuery>; + + #[storage_alias] + pub(crate) type ClaimQueue = StorageValue< + Pallet, + BTreeMap>>>>, + ValueQuery, + >; + + #[allow(deprecated)] + pub type MigrateToV1 = VersionedMigration< + 0, + 1, + UncheckedMigrateToV1, + Pallet, + ::DbWeight, + >; + + #[deprecated(note = "Use MigrateToV1 instead")] + pub struct UncheckedMigrateToV1(sp_std::marker::PhantomData); + #[allow(deprecated)] + impl OnRuntimeUpgrade for UncheckedMigrateToV1 { + fn on_runtime_upgrade() -> Weight { + let weight_consumed = migrate_to_v1::(); + + log::info!(target: LOG_TARGET, "Migrating para scheduler storage to v1"); + + weight_consumed + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::DispatchError> { + let n: u32 = v0::Scheduled::::get().len() as u32 + + v0::AvailabilityCores::::get().iter().filter(|c| c.is_some()).count() as u32; + + log::info!( + target: LOG_TARGET, + "Number of scheduled and waiting for availability before: {n}", + ); + + Ok(n.encode()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(state: Vec) -> Result<(), sp_runtime::DispatchError> { + log::info!(target: LOG_TARGET, "Running post_upgrade()"); + + frame_support::ensure!( + v0::Scheduled::::get().is_empty(), + "Scheduled should be empty after the migration" + ); + + let expected_len = u32::decode(&mut &state[..]).unwrap(); + let availability_cores_waiting = AvailabilityCores::::get() + .iter() + .filter(|c| !matches!(c, CoreOccupied::Free)) + .count(); + + frame_support::ensure!( + ClaimQueue::::get().iter().map(|la_vec| la_vec.1.len()).sum::() as u32 + + availability_cores_waiting as u32 == + expected_len, + "ClaimQueue and AvailabilityCores should have the correct length", + ); + + Ok(()) + } + } +} + +pub fn migrate_to_v1() -> Weight { + let mut weight: Weight = Weight::zero(); + + v0::ParathreadQueue::::kill(); + v0::ParathreadClaimIndex::::kill(); + + let now = >::block_number(); + let scheduled = v0::Scheduled::::take(); + let sched_len = scheduled.len() as u64; + for core_assignment in scheduled { + let core_idx = core_assignment.core; + let assignment = Assignment::new(core_assignment.para_id); + let pe = ParasEntry::new(assignment, now); + + v1::ClaimQueue::::mutate(|la| { + la.entry(core_idx).or_default().push_back(Some(pe)); + }); + } + + let parachains = runtime_parachains::paras::Pallet::::parachains(); + let availability_cores = v0::AvailabilityCores::::take(); + let mut new_availability_cores = Vec::new(); + + for (core_index, core) in availability_cores.into_iter().enumerate() { + let new_core = if let Some(core) = core { + match core { + v0::CoreOccupied::Parachain => CoreOccupied::Paras(ParasEntry::new( + Assignment::new(parachains[core_index]), + now, + )), + v0::CoreOccupied::Parathread(entry) => + CoreOccupied::Paras(ParasEntry::new(Assignment::new(entry.claim.0), now)), + } + } else { + CoreOccupied::Free + }; + + new_availability_cores.push(new_core); + } + + v1::AvailabilityCores::::set(new_availability_cores); + + // 2x as once for Scheduled and once for Claimqueue + weight = weight.saturating_add(T::DbWeight::get().reads_writes(2 * sched_len, 2 * sched_len)); + // reading parachains + availability_cores, writing AvailabilityCores + weight = weight.saturating_add(T::DbWeight::get().reads_writes(2, 1)); + // 2x kill + weight = weight.saturating_add(T::DbWeight::get().writes(2)); + + weight +} diff --git a/system-parachains/asset-hubs/asset-hub-kusama/Cargo.toml b/system-parachains/asset-hubs/asset-hub-kusama/Cargo.toml index fc0402a46c..13d150bc87 100644 --- a/system-parachains/asset-hubs/asset-hub-kusama/Cargo.toml +++ b/system-parachains/asset-hubs/asset-hub-kusama/Cargo.toml @@ -64,8 +64,8 @@ pallet-xcm-benchmarks = { default-features = false, optional = true , version = polkadot-core-primitives = { default-features = false, version = "2.0.0" } polkadot-parachain-primitives = { default-features = false, version = "1.0.0" } polkadot-runtime-common = { default-features = false, version = "2.0.0" } -xcm = { package = "staging-xcm", default-features = false, version = "2.0.0" } -xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.0" } +xcm = { package = "staging-xcm", default-features = false, version = "2.0.1" } +xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.1" } xcm-executor = { package = "staging-xcm-executor", default-features = false, version = "2.0.0" } # Cumulus diff --git a/system-parachains/asset-hubs/asset-hub-kusama/src/lib.rs b/system-parachains/asset-hubs/asset-hub-kusama/src/lib.rs index 12633ce5b9..7d2fafbafa 100644 --- a/system-parachains/asset-hubs/asset-hub-kusama/src/lib.rs +++ b/system-parachains/asset-hubs/asset-hub-kusama/src/lib.rs @@ -111,7 +111,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("statemine"), impl_name: create_runtime_str!("statemine"), authoring_version: 1, - spec_version: 10000, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 13, @@ -127,7 +127,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("statemine"), impl_name: create_runtime_str!("statemine"), authoring_version: 1, - spec_version: 10000, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 13, diff --git a/system-parachains/asset-hubs/asset-hub-polkadot/Cargo.toml b/system-parachains/asset-hubs/asset-hub-polkadot/Cargo.toml index 4898564c6c..973f527540 100644 --- a/system-parachains/asset-hubs/asset-hub-polkadot/Cargo.toml +++ b/system-parachains/asset-hubs/asset-hub-polkadot/Cargo.toml @@ -59,8 +59,8 @@ polkadot-core-primitives = { default-features = false, version = "2.0.0" } polkadot-parachain-primitives = { default-features = false, version = "1.0.0" } polkadot-runtime-common = { default-features = false, version = "2.0.0" } polkadot-runtime-constants = { path = "../../../relay/polkadot/constants", default-features = false} -xcm = { package = "staging-xcm", default-features = false, version = "2.0.0" } -xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.0" } +xcm = { package = "staging-xcm", default-features = false, version = "2.0.1" } +xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.1" } xcm-executor = { package = "staging-xcm-executor", default-features = false, version = "2.0.0" } # Cumulus diff --git a/system-parachains/asset-hubs/asset-hub-polkadot/src/lib.rs b/system-parachains/asset-hubs/asset-hub-polkadot/src/lib.rs index f5117490dd..2498ce7cc4 100644 --- a/system-parachains/asset-hubs/asset-hub-polkadot/src/lib.rs +++ b/system-parachains/asset-hubs/asset-hub-polkadot/src/lib.rs @@ -137,7 +137,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("statemint"), impl_name: create_runtime_str!("statemint"), authoring_version: 1, - spec_version: 10000, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 13, diff --git a/system-parachains/bridge-hubs/bridge-hub-kusama/Cargo.toml b/system-parachains/bridge-hubs/bridge-hub-kusama/Cargo.toml index 03d82da68c..11225c620b 100644 --- a/system-parachains/bridge-hubs/bridge-hub-kusama/Cargo.toml +++ b/system-parachains/bridge-hubs/bridge-hub-kusama/Cargo.toml @@ -57,8 +57,8 @@ pallet-xcm-benchmarks = { default-features = false, optional = true , version = polkadot-core-primitives = { default-features = false, version = "2.0.0" } polkadot-parachain-primitives = { default-features = false, version = "1.0.0" } polkadot-runtime-common = { default-features = false, version = "2.0.0" } -xcm = { package = "staging-xcm", default-features = false, version = "2.0.0" } -xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.0" } +xcm = { package = "staging-xcm", default-features = false, version = "2.0.1" } +xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.1" } xcm-executor = { package = "staging-xcm-executor", default-features = false, version = "2.0.0" } # Cumulus diff --git a/system-parachains/bridge-hubs/bridge-hub-kusama/src/lib.rs b/system-parachains/bridge-hubs/bridge-hub-kusama/src/lib.rs index 7a95e2e6b9..206f306843 100644 --- a/system-parachains/bridge-hubs/bridge-hub-kusama/src/lib.rs +++ b/system-parachains/bridge-hubs/bridge-hub-kusama/src/lib.rs @@ -154,7 +154,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("bridge-hub-kusama"), impl_name: create_runtime_str!("bridge-hub-kusama"), authoring_version: 1, - spec_version: 10000, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 3, diff --git a/system-parachains/bridge-hubs/bridge-hub-polkadot/Cargo.toml b/system-parachains/bridge-hubs/bridge-hub-polkadot/Cargo.toml index 587595bc98..cac2fcc358 100644 --- a/system-parachains/bridge-hubs/bridge-hub-polkadot/Cargo.toml +++ b/system-parachains/bridge-hubs/bridge-hub-polkadot/Cargo.toml @@ -57,8 +57,8 @@ pallet-xcm-benchmarks = { default-features = false, optional = true , version = polkadot-core-primitives = { default-features = false, version = "2.0.0" } polkadot-parachain-primitives = { default-features = false, version = "1.0.0" } polkadot-runtime-common = { default-features = false, version = "2.0.0" } -xcm = { package = "staging-xcm", default-features = false, version = "2.0.0" } -xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.0" } +xcm = { package = "staging-xcm", default-features = false, version = "2.0.1" } +xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.1" } xcm-executor = { package = "staging-xcm-executor", default-features = false, version = "2.0.0" } # Cumulus diff --git a/system-parachains/bridge-hubs/bridge-hub-polkadot/src/lib.rs b/system-parachains/bridge-hubs/bridge-hub-polkadot/src/lib.rs index dbfdc249a3..48d0744ff0 100644 --- a/system-parachains/bridge-hubs/bridge-hub-polkadot/src/lib.rs +++ b/system-parachains/bridge-hubs/bridge-hub-polkadot/src/lib.rs @@ -129,7 +129,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("bridge-hub-polkadot"), impl_name: create_runtime_str!("bridge-hub-polkadot"), authoring_version: 1, - spec_version: 10000, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 2, diff --git a/system-parachains/collectives/collectives-polkadot/Cargo.toml b/system-parachains/collectives/collectives-polkadot/Cargo.toml index f36b68a646..5af3c75bed 100644 --- a/system-parachains/collectives/collectives-polkadot/Cargo.toml +++ b/system-parachains/collectives/collectives-polkadot/Cargo.toml @@ -61,8 +61,8 @@ polkadot-core-primitives = { default-features = false, version = "2.0.0" } polkadot-parachain-primitives = { default-features = false, version = "1.0.0" } polkadot-runtime-common = { default-features = false, version = "2.0.0" } polkadot-runtime-constants = { path = "../../../relay/polkadot/constants", default-features = false} -xcm = { package = "staging-xcm", default-features = false, version = "2.0.0" } -xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.0" } +xcm = { package = "staging-xcm", default-features = false, version = "2.0.1" } +xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.1" } xcm-executor = { package = "staging-xcm-executor", default-features = false, version = "2.0.0" } # Cumulus diff --git a/system-parachains/collectives/collectives-polkadot/src/lib.rs b/system-parachains/collectives/collectives-polkadot/src/lib.rs index e79a5bb15b..31c6923e85 100644 --- a/system-parachains/collectives/collectives-polkadot/src/lib.rs +++ b/system-parachains/collectives/collectives-polkadot/src/lib.rs @@ -105,7 +105,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("collectives"), impl_name: create_runtime_str!("collectives"), authoring_version: 1, - spec_version: 10000, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 5, diff --git a/system-parachains/gluttons/glutton-kusama/Cargo.toml b/system-parachains/gluttons/glutton-kusama/Cargo.toml index 9bf6646ea6..c7552d0e2c 100644 --- a/system-parachains/gluttons/glutton-kusama/Cargo.toml +++ b/system-parachains/gluttons/glutton-kusama/Cargo.toml @@ -35,8 +35,8 @@ sp-transaction-pool = { default-features = false, version = "21.0.0" } sp-version = { default-features = false, version = "24.0.0" } # Polkadot -xcm = { package = "staging-xcm", default-features = false, version = "2.0.0" } -xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.0" } +xcm = { package = "staging-xcm", default-features = false, version = "2.0.1" } +xcm-builder = { package = "staging-xcm-builder", default-features = false, version = "2.0.1" } xcm-executor = { package = "staging-xcm-executor", default-features = false, version = "2.0.0" } # Cumulus diff --git a/system-parachains/gluttons/glutton-kusama/src/lib.rs b/system-parachains/gluttons/glutton-kusama/src/lib.rs index dde8f747d4..a6153952de 100644 --- a/system-parachains/gluttons/glutton-kusama/src/lib.rs +++ b/system-parachains/gluttons/glutton-kusama/src/lib.rs @@ -87,7 +87,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("glutton"), impl_name: create_runtime_str!("glutton"), authoring_version: 1, - spec_version: 10000, + spec_version: 1_000_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 1,