Migrations necessary for Edgeware Node Upgrade (#6)

* Revert "Remove balances migration. (paritytech#5224)"

This reverts commit e1fcc97.

* adjust type imports and remove test

* Assign unique storage names to pallets. (paritytech#5010)

* Assign unique storage names to pallets.

* Bump spec

* Upgrade logic for finality tracker (untested)

* Logic for migrating Identity (untested)

* Logic for migrating transaction-payment

* Fix tests

* Fix `decl_storage` build

* Contract -> Contracts

* Update Cargo.lock

* bump spec

* update migration

* Fix merge error

* Migration for contracts

* Remove serde

* Remove some illegal spaces and Options

* Fix types in identity.

* Minor variable rename

Co-authored-by: Gavin Wood <gavin@parity.io>

* add weights to runtime migrations

* use max block weight for balances migration

* temporarily remove failing contracts migration

* add more logging for migrations

* fix transaction-payment migration by converting multiplier type

* add more logging to balance migration

* more balance migration logging

* format balance migration

* even more balance migration logging

* remove logging in loop and change upgrade flag type to bool

* cherry-pick paritytech#5226 (dd97b14)

massage the code and fix anything failing compilation

* use test accounts for account migration

* move balances and accounts migrations out for custom calling

* make migration mods public

* fix transaction-payment migration based on paritytech#5673

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Gavin Wood <gavin@parity.io>

bin/node-template/pallets/template/src/mock.rs

@@ -45,7 +45,7 @@ impl system::Trait for Test {
 	type Version = ();
 	type ModuleToIndex = ();
 	type AccountData = ();
-	type OnNewAccount = ();
+	type MigrateAccount = (); type OnNewAccount = ();
 	type OnKilledAccount = ();
 }
 impl Trait for Test {

bin/node-template/runtime/src/lib.rs

@@ -182,7 +182,7 @@ impl system::Trait for Runtime {
 	/// This type is being generated by `construct_runtime!`.
 	type ModuleToIndex = ModuleToIndex;
 	/// What to do if a new account is created.
-	type OnNewAccount = ();
+	type MigrateAccount = (); type OnNewAccount = ();
 	/// What to do if an account is fully reaped from the system.
 	type OnKilledAccount = ();
 	/// The data to be stored in an account.

bin/node/runtime/src/lib.rs

@@ -164,6 +164,7 @@ impl frame_system::Trait for Runtime {
 	type Version = Version;
 	type ModuleToIndex = ModuleToIndex;
 	type AccountData = pallet_balances::AccountData<Balance>;
+	type MigrateAccount = (Balances, Identity, Democracy, Elections, ImOnline, Recovery, Session, Society, Staking, Vesting);
 	type OnNewAccount = ();
 	type OnKilledAccount = ();
 }

frame/assets/src/lib.rs

@@ -325,7 +325,7 @@ mod tests {
 		type Version = ();
 		type ModuleToIndex = ();
 		type AccountData = ();
-		type OnNewAccount = ();
+		type MigrateAccount = (); type OnNewAccount = ();
 		type OnKilledAccount = ();
 	}
 	impl Trait for Test {

frame/aura/src/mock.rs

@@ -67,7 +67,7 @@ impl frame_system::Trait for Test {
 	type Version = ();
 	type ModuleToIndex = ();
 	type AccountData = ();
-	type OnNewAccount = ();
+	type MigrateAccount = (); type OnNewAccount = ();
 	type OnKilledAccount = ();
 }
 

frame/authority-discovery/src/lib.rs

@@ -164,7 +164,7 @@ mod tests {
 		type Version = ();
 		type ModuleToIndex = ();
 		type AccountData = ();
-		type OnNewAccount = ();
+		type MigrateAccount = (); type OnNewAccount = ();
 		type OnKilledAccount = ();
 	}
 

frame/authorship/src/lib.rs

@@ -439,7 +439,7 @@ mod tests {
 		type Version = ();
 		type ModuleToIndex = ();
 		type AccountData = ();
-		type OnNewAccount = ();
+		type MigrateAccount = (); type OnNewAccount = ();
 		type OnKilledAccount = ();
 	}
 

frame/babe/src/lib.rs

@@ -208,6 +208,14 @@ decl_module! {
 			// remove temporary "environment" entry from storage
 			Lateness::<T>::kill();
 		}
+
+		fn on_runtime_upgrade() -> Weight {
+			for i in 0..=SegmentIndex::get() {
+				UnderConstruction::migrate_key_from_blake(i);
+			}
+			// TODO: determine weight
+			0
+		}
 	}
 }
 

frame/babe/src/mock.rs

@@ -78,7 +78,7 @@ impl frame_system::Trait for Test {
 	type MaximumBlockLength = MaximumBlockLength;
 	type ModuleToIndex = ();
 	type AccountData = ();
-	type OnNewAccount = ();
+	type MigrateAccount = (); type OnNewAccount = ();
 	type OnKilledAccount = ();
 }
 

frame/balances/Cargo.toml

@@ -16,6 +16,7 @@ serde = { version = "1.0.101", optional = true }
 codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] }
 sp-std = { version = "2.0.0-rc2", default-features = false, path = "../../primitives/std" }
 sp-runtime = { version = "2.0.0-rc2", default-features = false, path = "../../primitives/runtime" }
+sp-io = { version = "2.0.0-rc2", default-features = false, path = "../../primitives/io" }
 frame-benchmarking = { version = "2.0.0-rc2", default-features = false, path = "../benchmarking", optional = true }
 frame-support = { version = "2.0.0-rc2", default-features = false, path = "../support" }
 frame-system = { version = "2.0.0-rc2", default-features = false, path = "../system" }

frame/balances/src/lib.rs

@@ -154,6 +154,7 @@ mod tests;
 mod tests_local;
 mod tests_composite;
 mod benchmarking;
+pub mod migration;
 
 use sp_std::prelude::*;
 use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr, convert::Infallible};
@@ -164,8 +165,9 @@ use frame_support::{
 		Currency, OnKilledAccount, OnUnbalanced, TryDrop, StoredMap,
 		WithdrawReason, WithdrawReasons, LockIdentifier, LockableCurrency, ExistenceRequirement,
 		Imbalance, SignedImbalance, ReservableCurrency, Get, ExistenceRequirement::KeepAlive,
-		ExistenceRequirement::AllowDeath, IsDeadAccount, BalanceStatus as Status,
-	}
+		ExistenceRequirement::AllowDeath, IsDeadAccount, BalanceStatus as Status, MigrateAccount
+	},
+	weights::Weight,
 };
 use sp_runtime::{
 	RuntimeDebug, DispatchResult, DispatchError,
@@ -542,6 +544,36 @@ decl_module! {
 			let dest = T::Lookup::lookup(dest)?;
 			<Self as Currency<_>>::transfer(&transactor, &dest, value, KeepAlive)?;
 		}
+
+		fn on_runtime_upgrade() -> Weight {
+			// Moved to custom runtime upgrade
+			// migration::on_runtime_upgrade::<T, I>()
+			0
+		}
+	}
+}
+
+#[derive(Decode)]
+struct OldBalanceLock<Balance, BlockNumber> {
+	id: LockIdentifier,
+	amount: Balance,
+	until: BlockNumber,
+	reasons: WithdrawReasons,
+}
+
+impl<Balance, BlockNumber> OldBalanceLock<Balance, BlockNumber> {
+	fn upgraded(self) -> (BalanceLock<Balance>, BlockNumber) {
+		(BalanceLock {
+			id: self.id,
+			amount: self.amount,
+			reasons: self.reasons.into(),
+		}, self.until)
+	}
+}
+
+impl<T: Trait<I>, I: Instance> MigrateAccount<T::AccountId> for Module<T, I> {
+	fn migrate_account(account: &T::AccountId) {
+		Locks::<T, I>::migrate_key_from_blake(account);
 	}
 }
 
@@ -868,7 +900,7 @@ impl<T: Subtrait<I>, I: Instance> frame_system::Trait for ElevatedTrait<T, I> {
 	type AvailableBlockRatio = T::AvailableBlockRatio;
 	type Version = T::Version;
 	type ModuleToIndex = T::ModuleToIndex;
-	type OnNewAccount = T::OnNewAccount;
+	type MigrateAccount = (); type OnNewAccount = T::OnNewAccount;
 	type OnKilledAccount = T::OnKilledAccount;
 	type AccountData = T::AccountData;
 }

frame/balances/src/migration.rs

@@ -0,0 +1,145 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Temporary migrations of the balances module.
+
+use super::*;
+
+use frame_support::storage::migration::{
+	get_storage_value, have_storage_value, put_storage_value, take_storage_value, StorageIterator,
+};
+use frame_support::weights::Weight;
+use sp_io::hashing::twox_64;
+
+pub fn on_runtime_upgrade<T: Trait<I>, I: Instance>() -> Weight {
+	match StorageVersion::<I>::get() {
+		Releases::V2_0_0 => T::DbWeight::get().reads(1),
+		Releases::V1_0_0 => upgrade_v1_to_v2::<T, I>(),
+	}
+}
+
+// Upgrade from the pre-#4649 balances/vesting into the new balances.
+fn upgrade_v1_to_v2<T: Trait<I>, I: Instance>() -> Weight {
+	sp_runtime::print("Upgrading Account Balances...");
+	// First, migrate from old FreeBalance to new Account.
+	// We also move all locks across since only accounts with FreeBalance values have locks.
+	// FreeBalance: map T::AccountId => T::Balance
+	sp_runtime::print("Balances: FreeBalance -> Account");
+	for (hash, free) in StorageIterator::<T::Balance>::new(b"Balances", b"FreeBalance").drain() {
+		let mut account = AccountData {
+			free,
+			..Default::default()
+		};
+		// Locks: map T::AccountId => Vec<BalanceLock>
+		let old_locks = get_storage_value::<Vec<OldBalanceLock<T::Balance, T::BlockNumber>>>(
+			b"Balances",
+			b"Locks",
+			&hash,
+		);
+		if let Some(locks) = old_locks {
+			let locks = locks
+				.into_iter()
+				.map(|i| {
+					let (result, expiry) = i.upgraded();
+					if expiry != T::BlockNumber::max_value() {
+						// Any `until`s that are not T::BlockNumber::max_value come from
+						// democracy and need to be migrated over there.
+						// Democracy: Locks get(locks): map T::AccountId => Option<T::BlockNumber>;
+						put_storage_value(b"Democracy", b"Locks", &hash, expiry);
+					}
+					result
+				})
+				.collect::<Vec<_>>();
+			for l in locks.iter() {
+				if l.reasons == Reasons::All || l.reasons == Reasons::Misc {
+					account.misc_frozen = account.misc_frozen.max(l.amount);
+				}
+				if l.reasons == Reasons::All || l.reasons == Reasons::Fee {
+					account.fee_frozen = account.fee_frozen.max(l.amount);
+				}
+			}
+			put_storage_value(b"Balances", b"Locks", &hash, locks);
+		}
+		put_storage_value(b"Balances", b"Account", &hash, account);
+	}
+	// Second, migrate old ReservedBalance into new Account.
+	// ReservedBalance: map T::AccountId => T::Balance
+	sp_runtime::print("Balances: ReservedBalance -> Account");
+	for (hash, reserved) in StorageIterator::<T::Balance>::new(b"Balances", b"ReservedBalance").drain() {
+		let mut account =
+			get_storage_value::<AccountData<T::Balance>>(b"Balances", b"Account", &hash).unwrap_or_default();
+		account.reserved = reserved;
+		put_storage_value(b"Balances", b"Account", &hash, account);
+	}
+
+	// Finally, migrate vesting and ensure locks are in place. We will be lazy and just lock
+	// for the maximum amount (i.e. at genesis). Users will need to call "vest" to reduce the
+	// lock to something sensible.
+	// pub Vesting: map T::AccountId => Option<VestingSchedule>;
+	sp_runtime::print("Balances: Vesting");
+	for (hash, vesting) in
+		StorageIterator::<(T::Balance, T::Balance, T::BlockNumber)>::new(b"Balances", b"Vesting").drain()
+	{
+		let mut account =
+			get_storage_value::<AccountData<T::Balance>>(b"Balances", b"Account", &hash).unwrap_or_default();
+		let mut locks = get_storage_value::<Vec<BalanceLock<T::Balance>>>(b"Balances", b"Locks", &hash)
+			.unwrap_or_default();
+		locks.push(BalanceLock {
+			id: *b"vesting ",
+			amount: vesting.0.clone(),
+			reasons: Reasons::Misc,
+		});
+		account.misc_frozen = account.misc_frozen.max(vesting.0.clone());
+		put_storage_value(b"Vesting", b"Vesting", &hash, vesting);
+		put_storage_value(b"Balances", b"Locks", &hash, locks);
+		put_storage_value(b"Balances", b"Account", &hash, account);
+	}
+
+	let prefix = {
+		let encoded_prefix_key_hash = b":session:keys".to_vec().encode();
+		let mut h = twox_64(&encoded_prefix_key_hash[..]).to_vec();
+		h.extend(&encoded_prefix_key_hash[..]);
+		h
+	};
+
+	sp_runtime::print("Balances: Balance::Account -> System::Account");
+	for (hash, balances) in StorageIterator::<AccountData<T::Balance>>::new(b"Balances", b"Account").drain() {
+		let nonce = take_storage_value::<T::Index>(b"System", b"AccountNonce", &hash).unwrap_or_default();
+		let mut refs: system::RefCount = 0;
+		// The items in Kusama that would result in a ref count being incremented.
+		if have_storage_value(b"Democracy", b"Proxy", &hash) {
+			refs += 1
+		}
+		// We skip Recovered since it's being replaced anyway.
+		let mut prefixed_hash = prefix.clone();
+		prefixed_hash.extend(&hash[..]);
+		if have_storage_value(b"Session", b"NextKeys", &prefixed_hash) {
+			refs += 1
+		}
+		if have_storage_value(b"Staking", b"Bonded", &hash) {
+			refs += 1
+		}
+		put_storage_value(b"System", b"Account", &hash, (nonce, refs, &balances));
+	}
+
+	take_storage_value::<bool>(b"Balances", b"IsUpgraded", &[]);
+
+	StorageVersion::<I>::put(Releases::V2_0_0);
+
+	sp_runtime::print("Done Account Balances.");
+	// TODO determine actual weight
+	T::MaximumBlockWeight::get()
+}

frame/balances/src/tests_composite.rs

@@ -77,7 +77,7 @@ impl frame_system::Trait for Test {
 	type Version = ();
 	type ModuleToIndex = ();
 	type AccountData = super::AccountData<u64>;
-	type OnNewAccount = ();
+	type MigrateAccount = (); type OnNewAccount = ();
 	type OnKilledAccount = ();
 }
 parameter_types! {

frame/balances/src/tests_local.rs

@@ -77,7 +77,7 @@ impl frame_system::Trait for Test {
 	type Version = ();
 	type ModuleToIndex = ();
 	type AccountData = super::AccountData<u64>;
-	type OnNewAccount = ();
+	type MigrateAccount = (); type OnNewAccount = ();
 	type OnKilledAccount = Module<Test>;
 }
 parameter_types! {

frame/benchmarking/src/tests.rs

@@ -88,7 +88,7 @@ impl frame_system::Trait for Test {
 	type Version = ();
 	type ModuleToIndex = ();
 	type AccountData = ();
-	type OnNewAccount = ();
+	type MigrateAccount = (); type OnNewAccount = ();
 	type OnKilledAccount = ();
 }
 

frame/collective/src/lib.rs

@@ -334,6 +334,16 @@ fn get_result_weight(result: DispatchResultWithPostInfo) -> Option<Weight> {
 	}
 }
 
+mod migration {
+	use super::*;
+
+	pub fn migrate<T: Trait<I>, I: Instance>() {
+		for p in Proposals::<T, I>::get().into_iter() {
+			ProposalOf::<T, I>::migrate_key_from_blake(&p);
+			Voting::<T, I>::migrate_key_from_blake(&p);
+		}
+	}
+}
 
 // Note that councillor operations are assigned to the operational class.
 decl_module! {
@@ -342,6 +352,12 @@ decl_module! {
 
 		fn deposit_event() = default;
 
+		fn on_runtime_upgrade() -> Weight {
+			migration::migrate::<T, I>();
+			// TODO: determine actual weight
+			0
+		}
+
 		/// Set the collective's membership.
 		///
 		/// - `new_members`: The new member list. Be nice to the chain and provide it sorted.
@@ -990,7 +1006,7 @@ mod tests {
 		type Version = ();
 		type ModuleToIndex = ();
 		type AccountData = ();
-		type OnNewAccount = ();
+		type MigrateAccount = (); type OnNewAccount = ();
 		type OnKilledAccount = ();
 	}
 	impl Trait<Instance1> for Test {

frame/contracts/src/lib.rs

@@ -89,6 +89,7 @@ mod rent;
 
 #[cfg(test)]
 mod tests;
+mod migration;
 
 use crate::exec::ExecutionContext;
 use crate::account_db::{AccountDb, DirectAccountDb};
@@ -578,6 +579,10 @@ decl_module! {
 				T::Currency::deposit_into_existing(&rewarded, T::SurchargeReward::get())?;
 			}
 		}
+
+		fn on_runtime_upgrade() -> Weight {
+			migration::on_runtime_upgrade::<T>()
+		}
 	}
 }