Implement pallet view function queries (#4722)

Closes #216.

This PR allows pallets to define a `view_functions` impl like so:

```rust
#[pallet::view_functions]
impl<T: Config> Pallet<T>
where
	T::AccountId: From<SomeType1> + SomeAssociation1,
{
	/// Query value no args.
	pub fn get_value() -> Option<u32> {
		SomeValue::<T>::get()
	}

	/// Query value with args.
	pub fn get_value_with_arg(key: u32) -> Option<u32> {
		SomeMap::<T>::get(key)
	}
}
```
### `QueryId`

Each view function is uniquely identified by a `QueryId`, which for this
implementation is generated by:

```twox_128(pallet_name) ++ twox_128("fn_name(fnarg_types) -> return_ty")```

The prefix `twox_128(pallet_name)` is the same as the storage prefix for pallets and take into account multiple instances of the same pallet.

The suffix is generated from the fn type signature so is guaranteed to be unique for that pallet impl. For one of the view fns in the example above it would be `twox_128("get_value_with_arg(u32) -> Option<u32>")`. It is a known limitation that only the type names themselves are taken into account: in the case of type aliases the signature may have the same underlying types but a different id; for generics the concrete types may be different but the signatures will remain the same.

The existing Runtime `Call` dispatchables are addressed by their concatenated indices `pallet_index ++ call_index`, and the dispatching is handled by the SCALE decoding of the `RuntimeCallEnum::PalletVariant(PalletCallEnum::dispatchable_variant(payload))`. For `view_functions` the runtime/pallet generated enum structure is replaced by implementing the `DispatchQuery` trait on the outer (runtime) scope, dispatching to a pallet based on the id prefix, and the inner (pallet) scope dispatching to the specific function based on the id suffix.

Future implementations could also modify/extend this scheme and routing to pallet agnostic queries.

### Executing externally

These view functions can be executed externally via the system runtime api:

```rust
pub trait ViewFunctionsApi<QueryId, Query, QueryResult, Error> where
	QueryId: codec::Codec,
	Query: codec::Codec,
	QueryResult: codec::Codec,
	Error: codec::Codec,
{
	/// Execute a view function query.
fn execute_query(query_id: QueryId, query: Query) -> Result<QueryResult,
Error>;
}
```
### `XCQ`
Currently there is work going on by @xlc to implement [`XCQ`](https://github.com/open-web3-stack/XCQ/) which may eventually supersede this work.

It may be that we still need the fixed function local query dispatching in addition to XCQ, in the same way that we have chain specific runtime dispatchables and XCM.

I have kept this in mind and the high level query API is agnostic to the underlying query dispatch and execution. I am just providing the implementation for the `view_function` definition.

### Metadata
Currently I am utilizing the `custom` section of the frame metadata, to avoid modifying the official metadata format until this is standardized.

### vs `runtime_api`
There are similarities with `runtime_apis`, some differences being:
- queries can be defined directly on pallets, so no need for boilerplate declarations and implementations
- no versioning, the `QueryId` will change if the signature changes. 
- possibility for queries to be executed from smart contracts (see below)

### Calling from contracts
Future work would be to add `weight` annotations to the view function queries, and a host function to `pallet_contracts` to allow executing these queries from contracts.

### TODO

- [x] Consistent naming (view functions pallet impl, queries, high level api?)
- [ ] End to end tests via `runtime_api`
- [ ] UI tests
- [x] Mertadata tests
- [ ] Docs

---------

Co-authored-by: kianenigma <kian@parity.io>
Co-authored-by: James Wilson <james@jsdw.me>
Co-authored-by: Giuseppe Re <giuseppe.re@parity.io>
Co-authored-by: Guillaume Thiolliere <guillaume.thiolliere@parity.io>

.github/workflows/check-semver.yml

@@ -76,6 +76,7 @@ jobs:
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'R0-silent') }}
         run: |
           rustup default $TOOLCHAIN
+          rustup target add wasm32-unknown-unknown --toolchain $TOOLCHAIN
           rustup component add rust-src --toolchain $TOOLCHAIN
 
       - name: install parity-publish

Cargo.toml

@@ -363,6 +363,7 @@ members = [
 	"substrate/frame/examples/single-block-migrations",
 	"substrate/frame/examples/split",
 	"substrate/frame/examples/tasks",
+	"substrate/frame/examples/view-functions",
 	"substrate/frame/executive",
 	"substrate/frame/fast-unstake",
 	"substrate/frame/glutton",
@@ -941,6 +942,7 @@ pallet-example-offchain-worker = { path = "substrate/frame/examples/offchain-wor
 pallet-example-single-block-migrations = { path = "substrate/frame/examples/single-block-migrations", default-features = false }
 pallet-example-split = { path = "substrate/frame/examples/split", default-features = false }
 pallet-example-tasks = { path = "substrate/frame/examples/tasks", default-features = false }
+pallet-example-view-functions = { path = "substrate/frame/examples/view-functions", default-features = false }
 pallet-examples = { path = "substrate/frame/examples" }
 pallet-fast-unstake = { path = "substrate/frame/fast-unstake", default-features = false }
 pallet-glutton = { path = "substrate/frame/glutton", default-features = false }

cumulus/pallets/weight-reclaim/src/tests.rs

@@ -89,7 +89,8 @@ mod runtime {
 		RuntimeHoldReason,
 		RuntimeSlashReason,
 		RuntimeLockId,
-		RuntimeTask
+		RuntimeTask,
+		RuntimeViewFunction
 	)]
 	pub struct Test;
 

docs/sdk/src/reference_docs/chain_spec_runtime/src/runtime.rs

@@ -57,7 +57,14 @@ type SignedExtra = ();
 mod runtime {
 	/// The main runtime type.
 	#[runtime::runtime]
-	#[runtime::derive(RuntimeCall, RuntimeEvent, RuntimeError, RuntimeOrigin, RuntimeTask)]
+	#[runtime::derive(
+		RuntimeCall,
+		RuntimeEvent,
+		RuntimeError,
+		RuntimeOrigin,
+		RuntimeTask,
+		RuntimeViewFunction
+	)]
 	pub struct Runtime;
 
 	/// Mandatory system pallet that should always be included in a FRAME runtime.

polkadot/runtime/westend/src/lib.rs

@@ -1613,7 +1613,8 @@ mod runtime {
 		RuntimeHoldReason,
 		RuntimeSlashReason,
 		RuntimeLockId,
-		RuntimeTask
+		RuntimeTask,
+		RuntimeViewFunction
 	)]
 	pub struct Runtime;
 
@@ -1975,6 +1976,12 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
+	impl frame_support::view_functions::runtime_api::RuntimeViewFunction<Block> for Runtime {
+		fn execute_view_function(id: frame_support::view_functions::ViewFunctionId, input: Vec<u8>) -> Result<Vec<u8>, frame_support::view_functions::ViewFunctionDispatchError> {
+			Runtime::execute_view_function(id, input)
+		}
+	}
+
 	impl sp_block_builder::BlockBuilder<Block> for Runtime {
 		fn apply_extrinsic(extrinsic: <Block as BlockT>::Extrinsic) -> ApplyExtrinsicResult {
 			Executive::apply_extrinsic(extrinsic)

prdoc/pr_4722.prdoc

@@ -0,0 +1,33 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: Implement pallet view functions
+
+doc:
+  - audience: Runtime Dev
+    description: |
+      Read-only view functions can now be defined on pallets. These functions provide an interface for querying state,
+      from both outside and inside the runtime. Common queries can be defined on pallets, without users having to
+      access the storage directly.
+
+  - audience: Runtime User
+    description: |
+      Querying the runtime state is now easier with the introduction of pallet view functions. Clients can call commonly
+      defined view functions rather than accessing the storage directly. These are similar to the Runtime APIs, but
+      are defined within the runtime itself.
+
+crates:
+  - name: frame-support
+    bump: minor
+  - name: sp-metadata-ir
+    bump: major
+  - name: frame-support-procedural
+    bump: patch
+  - name: pallet-example-view-functions
+    bump: patch
+  - name: cumulus-pov-validator
+    bump: none
+  - name: cumulus-pallet-weight-reclaim
+    bump: patch
+  - name: westend-runtime
+    bump: minor

substrate/bin/node/runtime/src/lib.rs

@@ -2459,7 +2459,8 @@ mod runtime {
 		RuntimeHoldReason,
 		RuntimeSlashReason,
 		RuntimeLockId,
-		RuntimeTask
+		RuntimeTask,
+		RuntimeViewFunction
 	)]
 	pub struct Runtime;
 
@@ -3013,6 +3014,12 @@ impl_runtime_apis! {
 		}
 	}
 
+	impl frame_support::view_functions::runtime_api::RuntimeViewFunction<Block> for Runtime {
+		fn execute_view_function(id: frame_support::view_functions::ViewFunctionId, input: Vec<u8>) -> Result<Vec<u8>, frame_support::view_functions::ViewFunctionDispatchError> {
+			Runtime::execute_view_function(id, input)
+		}
+	}
+
 	impl sp_block_builder::BlockBuilder<Block> for Runtime {
 		fn apply_extrinsic(extrinsic: <Block as BlockT>::Extrinsic) -> ApplyExtrinsicResult {
 			Executive::apply_extrinsic(extrinsic)

substrate/frame/examples/Cargo.toml

@@ -26,6 +26,7 @@ pallet-example-offchain-worker = { workspace = true }
 pallet-example-single-block-migrations = { workspace = true }
 pallet-example-split = { workspace = true }
 pallet-example-tasks = { workspace = true }
+pallet-example-view-functions = { workspace = true }
 
 [features]
 default = ["std"]
@@ -40,6 +41,7 @@ std = [
 	"pallet-example-single-block-migrations/std",
 	"pallet-example-split/std",
 	"pallet-example-tasks/std",
+	"pallet-example-view-functions/std",
 ]
 try-runtime = [
 	"pallet-default-config-example/try-runtime",
@@ -51,4 +53,5 @@ try-runtime = [
 	"pallet-example-single-block-migrations/try-runtime",
 	"pallet-example-split/try-runtime",
 	"pallet-example-tasks/try-runtime",
+	"pallet-example-view-functions/try-runtime",
 ]

substrate/frame/examples/src/lib.rs

@@ -48,6 +48,9 @@
 //!
 //! - [`pallet_example_tasks`]: This pallet demonstrates the use of `Tasks` to execute service work.
 //!
+//! - [`pallet_example_view_functions`]: This pallet demonstrates the use of view functions to query
+//!   pallet state.
+//!
 //! - [`pallet_example_authorization_tx_extension`]: An example `TransactionExtension` that
 //!   authorizes a custom origin through signature validation, along with two support pallets to
 //!   showcase the usage.

substrate/frame/examples/view-functions/Cargo.toml

@@ -0,0 +1,61 @@
+[package]
+name = "pallet-example-view-functions"
+version = "1.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+description = "Pallet to demonstrate the usage of view functions to query pallet state"
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false, workspace = true }
+frame-metadata = { features = ["current"], workspace = true }
+log = { workspace = true }
+scale-info = { version = "2.11.1", default-features = false, features = ["derive"], workspace = true }
+
+frame-support = { path = "../../support", default-features = false, workspace = true }
+frame-system = { path = "../../system", default-features = false, workspace = true }
+
+sp-core = { default-features = false, path = "../../../primitives/core", workspace = true }
+sp-io = { path = "../../../primitives/io", default-features = false, workspace = true }
+sp-metadata-ir = { path = "../../../primitives/metadata-ir", default-features = false, workspace = true }
+sp-runtime = { path = "../../../primitives/runtime", default-features = false, workspace = true }
+
+frame-benchmarking = { path = "../../benchmarking", default-features = false, optional = true, workspace = true }
+
+[dev-dependencies]
+pretty_assertions = { version = "1.3.0" }
+
+[features]
+default = ["std"]
+std = [
+	"codec/std",
+	"frame-benchmarking?/std",
+	"frame-metadata/std",
+	"frame-support/std",
+	"frame-system/std",
+	"log/std",
+	"scale-info/std",
+	"sp-core/std",
+	"sp-io/std",
+	"sp-metadata-ir/std",
+	"sp-runtime/std",
+]
+runtime-benchmarks = [
+	"frame-benchmarking/runtime-benchmarks",
+	"frame-support/runtime-benchmarks",
+	"frame-system/runtime-benchmarks",
+	"sp-runtime/runtime-benchmarks",
+]
+try-runtime = [
+	"frame-support/try-runtime",
+	"frame-system/try-runtime",
+	"sp-runtime/try-runtime",
+]

substrate/frame/examples/view-functions/src/lib.rs

@@ -0,0 +1,114 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! This pallet demonstrates the use of the `pallet::view_functions_experimental` api for service
+//! work.
+#![cfg_attr(not(feature = "std"), no_std)]
+
+pub mod tests;
+
+use frame_support::Parameter;
+use scale_info::TypeInfo;
+
+pub struct SomeType1;
+impl From<SomeType1> for u64 {
+	fn from(_t: SomeType1) -> Self {
+		0u64
+	}
+}
+
+pub trait SomeAssociation1 {
+	type _1: Parameter + codec::MaxEncodedLen + TypeInfo;
+}
+impl SomeAssociation1 for u64 {
+	type _1 = u64;
+}
+
+#[frame_support::pallet]
+pub mod pallet {
+	use super::*;
+	use frame_support::pallet_prelude::*;
+
+	#[pallet::error]
+	pub enum Error<T> {}
+
+	#[pallet::config]
+	pub trait Config: frame_system::Config {}
+
+	#[pallet::pallet]
+	pub struct Pallet<T>(_);
+
+	#[pallet::storage]
+	pub type SomeValue<T: Config> = StorageValue<_, u32>;
+
+	#[pallet::storage]
+	pub type SomeMap<T: Config> = StorageMap<_, Twox64Concat, u32, u32, OptionQuery>;
+
+	#[pallet::view_functions_experimental]
+	impl<T: Config> Pallet<T>
+	where
+		T::AccountId: From<SomeType1> + SomeAssociation1,
+	{
+		/// Query value no args.
+		pub fn get_value() -> Option<u32> {
+			SomeValue::<T>::get()
+		}
+
+		/// Query value with args.
+		pub fn get_value_with_arg(key: u32) -> Option<u32> {
+			SomeMap::<T>::get(key)
+		}
+	}
+}
+
+#[frame_support::pallet]
+pub mod pallet2 {
+	use super::*;
+	use frame_support::pallet_prelude::*;
+
+	#[pallet::error]
+	pub enum Error<T, I = ()> {}
+
+	#[pallet::config]
+	pub trait Config<I: 'static = ()>: frame_system::Config {}
+
+	#[pallet::pallet]
+	pub struct Pallet<T, I = ()>(PhantomData<(T, I)>);
+
+	#[pallet::storage]
+	pub type SomeValue<T: Config<I>, I: 'static = ()> = StorageValue<_, u32>;
+
+	#[pallet::storage]
+	pub type SomeMap<T: Config<I>, I: 'static = ()> =
+		StorageMap<_, Twox64Concat, u32, u32, OptionQuery>;
+
+	#[pallet::view_functions_experimental]
+	impl<T: Config<I>, I: 'static> Pallet<T, I>
+	where
+		T::AccountId: From<SomeType1> + SomeAssociation1,
+	{
+		/// Query value no args.
+		pub fn get_value() -> Option<u32> {
+			SomeValue::<T, I>::get()
+		}
+
+		/// Query value with args.
+		pub fn get_value_with_arg(key: u32) -> Option<u32> {
+			SomeMap::<T, I>::get(key)
+		}
+	}
+}