Inherent filtering follow up (#4305)

* Add feature more feature gating for benchmarking + tests

* New line

* cargo run --quiet --release --features=runtime-benchmarks -- benchmark --chain=kusama-dev --steps=50 --repeat=20 --pallet=runtime_parachains::paras_inherent --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --header=./file_header.txt --output=./runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs

* cargo run --quiet --release --features=runtime-benchmarks -- benchmark --chain=polkadot-dev --steps=50 --repeat=20 --pallet=runtime_parachains::paras_inherent --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --header=./file_header.txt --output=./runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs

* Do not assume we use max validators per core

* Use kusama weights for rococo (hopefully temp)

* cargo run --quiet --release --features=runtime-benchmarks -- benchmark --chain=westend-dev --steps=50 --repeat=20 --pallet=runtime_parachains::paras_inherent --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --header=./file_header.txt --output=./runtime/westend/src/weights/runtime_parachains_paras_inherent.rs

* Add more validity votes when neccesary

* Some fixes for the last commit

* Restore westend weights

* cargo run --quiet --release --features=runtime-benchmarks -- benchmark --chain=westend-dev --steps=50 --repeat=20 --pallet=runtime_parachains::paras_inherent --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --header=./file_header.txt --output=./runtime/westend/src/weights/runtime_parachains_paras_inherent.rs

* Revert bad westend weights write

* Make sure to update val idx before skipping

* Fix validity vote range to max at group size'

* Temp setup for rococo

* cargo run --quiet --release --features runtime-benchmarks -- benchmark --chain=rococo-dev --steps=50 --repeat=20 --pallet=runtime_parachains::paras_inherent --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs --header=./file_header.txt

* Augment generated Rococo weights

* Make it compile

* Revert range for enter_backed_candidates_variable

* Delete runtime/kusama/src/weights/runtime_paras_paras_inherent.rs

Co-authored-by: Parity Bot <admin@parity.io>

primitives/Cargo.toml

@@ -55,4 +55,5 @@ std = [
 	"polkadot-core-primitives/std",
 	"bitvec/std",
 	"frame-system/std",
-]
+]
+runtime-benchmarks = []

primitives/src/v1/signed.rs

@@ -262,9 +262,7 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> UncheckedSigned<Payloa
 	}
 
 	/// Sign this payload with the given context and pair.
-	///
-	/// # WARNING
-	/// Only meant for usage in tests and and benchmarks.
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
 	pub fn benchmark_sign<H: Encode>(
 		public: &crate::v0::ValidatorId,
 		payload: Payload,
@@ -279,9 +277,7 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> UncheckedSigned<Payloa
 	}
 
 	/// Immutably access the signature.
-	///
-	/// # WARNING
-	/// Only meant for usage in tests and and benchmarks.
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
 	pub fn benchmark_signature(&self) -> ValidatorSignature {
 		self.signature.clone()
 	}

runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs

@@ -16,15 +16,15 @@
 //! Autogenerated weights for `runtime_parachains::paras_inherent`
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
-//! DATE: 2021-11-14, STEPS: `50`, REPEAT: 3, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! DATE: 2021-11-16, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
 //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
 
 // Executed Command:
 // target/release/polkadot
 // benchmark
 // --chain=kusama-dev
 // --steps=50
-// --repeat=3
+// --repeat=20
 // --pallet=runtime_parachains::paras_inherent
 // --extrinsic=*
 // --execution=wasm
@@ -70,9 +70,9 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_variable_disputes(v: u32, ) -> Weight {
-		(316_331_000 as Weight)
-			// Standard Error: 112_000
-			.saturating_add((325_000 as Weight).saturating_mul(v as Weight))
+		(202_257_000 as Weight)
+			// Standard Error: 2_000
+			.saturating_add((311_000 as Weight).saturating_mul(v as Weight))
 			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
@@ -103,7 +103,7 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		(352_749_000 as Weight)
+		(247_226_000 as Weight)
 			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
@@ -136,9 +136,9 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
-		(88_047_000 as Weight)
-			// Standard Error: 3_275_000
-			.saturating_add((68_499_000 as Weight).saturating_mul(v as Weight))
+		(322_535_000 as Weight)
+			// Standard Error: 24_000
+			.saturating_add((49_098_000 as Weight).saturating_mul(v as Weight))
 			.saturating_add(T::DbWeight::get().reads(26 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
@@ -171,7 +171,7 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidate_code_upgrade() -> Weight {
-		(53_728_168_000 as Weight)
+		(41_671_398_000 as Weight)
 			.saturating_add(T::DbWeight::get().reads(26 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}

runtime/parachains/Cargo.toml

@@ -86,6 +86,7 @@ runtime-benchmarks = [
 	"frame-benchmarking",
 	"frame-support/runtime-benchmarks",
 	"frame-system/runtime-benchmarks",
+	"primitives/runtime-benchmarks",
 ]
 try-runtime = [
 	"frame-support/try-runtime",

runtime/parachains/src/builder.rs

@@ -54,7 +54,7 @@ pub(crate) struct BenchBuilder<T: paras_inherent::Config> {
 }
 
 /// Paras inherent `enter` benchmark scenario.
-#[allow(dead_code)]
+#[cfg(any(feature = "runtime-benchmarks", test))]
 pub(crate) struct Bench<T: paras_inherent::Config> {
 	pub(crate) data: ParachainsInherentData<T::Header>,
 	pub(crate) _session: u32,
@@ -113,7 +113,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		self.max_validators.unwrap_or(Self::fallback_max_validators())
 	}
 
-	#[allow(dead_code)]
+	#[cfg(not(feature = "runtime-benchmarks"))]
 	pub(crate) fn set_max_validators(mut self, n: u32) -> Self {
 		self.max_validators = Some(n);
 		self
@@ -129,7 +129,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	/// corresponding dispute statement set. Note that if the number of disputes is not specified it fallbacks
 	/// to having a dispute per every validator. Additionally, an entry is not guaranteed to have a dispute - it
 	/// must line up with the cores marked as disputed as defined in `Self::Build`.
-	#[allow(dead_code)]
+	#[cfg(not(feature = "runtime-benchmarks"))]
 	pub(crate) fn set_dispute_statements(mut self, m: BTreeMap<u32, u32>) -> Self {
 		self.dispute_statements = m;
 		self
@@ -140,7 +140,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	}
 
 	/// Set maximum number of validators per core.
-	#[allow(dead_code)]
+	#[cfg(not(feature = "runtime-benchmarks"))]
 	pub(crate) fn set_max_validators_per_core(mut self, n: u32) -> Self {
 		self.max_validators_per_core = Some(n);
 		self
@@ -152,7 +152,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	}
 
 	/// Minimum number of validity votes in order for a backed candidate to be included.
-	#[allow(dead_code)]
+	#[cfg(feature = "runtime-benchmarks")]
 	pub(crate) fn fallback_min_validity_votes() -> u32 {
 		(Self::fallback_max_validators() / 2) + 1
 	}
@@ -302,7 +302,6 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 			Default::default(),
 		);
 
-		assert_eq!(scheduler::ValidatorGroups::<T>::get().len(), total_cores as usize);
 		assert_eq!(<shared::Pallet<T>>::session_index(), target_session);
 
 		// We need to refetch validators since they have been shuffled.

runtime/parachains/src/paras_inherent/benchmarking.rs

@@ -85,11 +85,18 @@ benchmarks! {
 	// Variant over `v`, the amount of validity votes for a backed candidate. This gives the weight
 	// of a single backed candidate.
 	enter_backed_candidates_variable {
-		// NOTE: the starting value must be over half of `max_validators` so the backed candidate is
-		// not rejected.
-		let v
-			in (BenchBuilder::<T>::fallback_min_validity_votes())
-				..BenchBuilder::<T>::fallback_max_validators();
+		// NOTE: the starting value must be over half of the max validators per group so the backed
+		// candidate is not rejected. Also, we cannot have more validity votes than validators in
+		// the group.
+
+		// Do not use this range for Rococo because it only has 1 validator per backing group,
+		// which causes issues when trying to create slopes with the benchmarking analysis. Instead
+		// use v = 1 for running Rococo benchmarks
+		let v in (BenchBuilder::<T>::fallback_min_validity_votes())
+			..(BenchBuilder::<T>::fallback_max_validators());
+
+		// Comment in for running rococo benchmarks
+		// let v = 1;
 
 		let cores_with_backed: BTreeMap<_, _>
 			= vec![(0, v)] // The backed candidate will have `v` validity votes.

runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs

@@ -16,22 +16,22 @@
 //! Autogenerated weights for `runtime_parachains::paras_inherent`
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
-//! DATE: 2021-11-14, STEPS: `50`, REPEAT: 3, LOW RANGE: `[]`, HIGH RANGE: `[]`
-//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
+//! DATE: 2021-11-17, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("polkadot-dev"), DB CACHE: 128
 
 // Executed Command:
 // target/release/polkadot
 // benchmark
-// --chain=kusama-dev
+// --chain=polkadot-dev
 // --steps=50
-// --repeat=3
+// --repeat=20
 // --pallet=runtime_parachains::paras_inherent
 // --extrinsic=*
 // --execution=wasm
 // --wasm-execution=compiled
 // --heap-pages=4096
 // --header=./file_header.txt
-// --output=./runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
+// --output=./runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
 
 
 #![cfg_attr(rustfmt, rustfmt_skip)]
@@ -70,9 +70,9 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_variable_disputes(v: u32, ) -> Weight {
-		(316_331_000 as Weight)
-			// Standard Error: 112_000
-			.saturating_add((325_000 as Weight).saturating_mul(v as Weight))
+		(201_282_000 as Weight)
+			// Standard Error: 2_000
+			.saturating_add((309_000 as Weight).saturating_mul(v as Weight))
 			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
@@ -103,7 +103,7 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		(352_749_000 as Weight)
+		(245_511_000 as Weight)
 			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
@@ -136,9 +136,9 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
-		(88_047_000 as Weight)
-			// Standard Error: 3_275_000
-			.saturating_add((68_499_000 as Weight).saturating_mul(v as Weight))
+		(318_776_000 as Weight)
+			// Standard Error: 24_000
+			.saturating_add((49_071_000 as Weight).saturating_mul(v as Weight))
 			.saturating_add(T::DbWeight::get().reads(26 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
@@ -171,7 +171,7 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidate_code_upgrade() -> Weight {
-		(53_728_168_000 as Weight)
+		(42_284_487_000 as Weight)
 			.saturating_add(T::DbWeight::get().reads(26 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}