Merge #3684

3684: Improve support for cycles access r=zhangsoledad a=zhangsoledad

<!--
Thank you for contributing to nervosnetwork/ckb!

If you haven't already, please read [CONTRIBUTING](https://github.com/nervosnetwork/ckb/blob/develop/CONTRIBUTING.md) document.

If you're unsure about anything, just ask; somebody should be along to answer within a day or two.

PR Title Format:
1. module [, module2, module3]: what's changed
2. *: what's changed
-->

### What problem does this PR solve?

Improve ckb's support for recording and accessing cycles as described in [3613](#3673), while implementing the fee_rate estimate short-term scheme.


### What is changed and how it works?

* For cycles and tx_size, the appending of records is done in a compatible way, so that when a new version of the node is run, new data is written, cycles and tx_size are appended, and old data is read in a compatible way. RPC returns data that is not recorded as `null`.
* `get_block`/`get_block_by_number` adds a parameter `with_cycles` to choose whether to return cycles or not, mainly because some scenarios, such as indexer, do not need cycles and are performance sensitive, so an additional parameter is added to avoid the extra overhead needed to return cycles.


### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

- Unit test
- Integration test


### Release note <!-- Choose from None, Title Only and Note. Bugfixes or new features need a release note. -->

```release-note
Title Only: Include only the PR title in the release note.
```



Co-authored-by: zhangsoledad <787953403@qq.com>

chain/src/chain.rs

@@ -22,7 +22,7 @@ use ckb_types::{
             ResolvedTransaction,
         },
         service::{Request, DEFAULT_CHANNEL_SIZE, SIGNAL_CHANNEL_SIZE},
-        BlockExt, BlockNumber, BlockView, Capacity, Cycle, HeaderView,
+        BlockExt, BlockNumber, BlockView, Cycle, HeaderView,
     },
     packed::{Byte32, ProposalShortId},
     utilities::merkle_mountain_range::ChainRootMMR,
@@ -412,6 +412,8 @@ impl ChainService {
             total_uncles_count: parent_ext.total_uncles_count + block.data().uncles().len() as u64,
             verified: None,
             txs_fees: vec![],
+            cycles: None,
+            txs_sizes: None,
         };
 
         db_txn.insert_block_epoch_index(
@@ -755,8 +757,12 @@ impl ChainService {
                             };
                             match verified {
                                 Ok((cycles, cache_entries)) => {
-                                    let txs_fees =
-                                        cache_entries.iter().map(|entry| entry.fee).collect();
+                                    let txs_sizes = resolved
+                                        .iter()
+                                        .map(|rtx| {
+                                            rtx.transaction.data().serialized_size_in_block() as u64
+                                        })
+                                        .collect();
                                     txn.attach_block(b)?;
                                     attach_block_cell(txn, b)?;
                                     mmr.push(b.digest())
@@ -766,7 +772,8 @@ impl ChainService {
                                         txn,
                                         &b.header().hash(),
                                         ext.clone(),
-                                        Some(txs_fees),
+                                        Some(&cache_entries),
+                                        Some(txs_sizes),
                                     )?;
 
                                     if !switch.disable_script() && b.transactions().len() > 1 {
@@ -798,7 +805,7 @@ impl ChainService {
                 attach_block_cell(txn, b)?;
                 mmr.push(b.digest())
                     .map_err(|e| InternalErrorKind::MMR.other(e))?;
-                self.insert_ok_ext(txn, &b.header().hash(), ext.clone(), None)?;
+                self.insert_ok_ext(txn, &b.header().hash(), ext.clone(), None, None)?;
             }
         }
 
@@ -835,12 +842,19 @@ impl ChainService {
         txn: &StoreTransaction,
         hash: &Byte32,
         mut ext: BlockExt,
-        txs_fees: Option<Vec<Capacity>>,
+        cache_entries: Option<&[Completed]>,
+        txs_sizes: Option<Vec<u64>>,
     ) -> Result<(), Error> {
         ext.verified = Some(true);
-        if let Some(txs_fees) = txs_fees {
+        if let Some(entries) = cache_entries {
+            let (txs_fees, cycles) = entries
+                .iter()
+                .map(|entry| (entry.fee, entry.cycles))
+                .unzip();
             ext.txs_fees = txs_fees;
+            ext.cycles = Some(cycles);
         }
+        ext.txs_sizes = txs_sizes;
         txn.insert_block_ext(hash, &ext)
     }
 

chain/src/tests/find_fork.rs

@@ -66,6 +66,8 @@ fn test_find_fork_case1() {
         // if txs in parent is invalid, txs in block is also invalid
         verified: None,
         txs_fees: vec![],
+        cycles: None,
+        txs_sizes: None,
     };
 
     let mut fork = ForkChanges::default();
@@ -136,6 +138,8 @@ fn test_find_fork_case2() {
         // if txs in parent is invalid, txs in block is also invalid
         verified: None,
         txs_fees: vec![],
+        cycles: None,
+        txs_sizes: None,
     };
 
     let mut fork = ForkChanges::default();
@@ -207,6 +211,8 @@ fn test_find_fork_case3() {
         // if txs in parent is invalid, txs in block is also invalid
         verified: None,
         txs_fees: vec![],
+        cycles: None,
+        txs_sizes: None,
     };
     let mut fork = ForkChanges::default();
 
@@ -277,6 +283,8 @@ fn test_find_fork_case4() {
         // if txs in parent is invalid, txs in block is also invalid
         verified: None,
         txs_fees: vec![],
+        cycles: None,
+        txs_sizes: None,
     };
 
     let mut fork = ForkChanges::default();

chain/src/tests/load_code_with_snapshot.rs

@@ -108,7 +108,10 @@ fn test_load_code() {
     let ret = tx_pool.submit_local_tx(tx.clone()).unwrap();
     assert!(ret.is_ok(), "ret {:?}", ret);
     let tx_status = tx_pool.get_tx_status(tx.hash());
-    assert_eq!(tx_status.unwrap().unwrap(), TxStatus::Pending);
+    assert_eq!(
+        tx_status.unwrap().unwrap(),
+        (TxStatus::Pending, Some(11174))
+    );
 }
 
 #[test]
@@ -177,7 +180,7 @@ fn test_load_code_with_snapshot() {
         let mut counter = 0;
         loop {
             let tx_status = tx_pool.get_tx_status(tx.hash());
-            if let Ok(Ok(status)) = tx_status {
+            if let Ok(Ok((status, _))) = tx_status {
                 if status == TxStatus::Pending {
                     break;
                 }
@@ -265,7 +268,7 @@ fn _test_load_code_with_snapshot_after_hardfork(script_type: ScriptHashType) {
         let mut counter = 0;
         loop {
             let tx_status = tx_pool.get_tx_status(tx.hash());
-            if let Ok(Ok(status)) = tx_status {
+            if let Ok(Ok((status, _))) = tx_status {
                 if status == TxStatus::Pending {
                     break;
                 }

devtools/doc/rpc.py

@@ -256,6 +256,8 @@ def handle_starttag(self, tag, attrs):
                 self.ty = '`null`'
             if self.ty == RUST_DOC_PREFIX + '/std/primitive.bool.html':
                 self.ty = '`boolean`'
+            if self.ty == RUST_DOC_PREFIX + '/std/primitive.f64.html':
+                self.ty = '`64-bit floating point`'
             if self.ty == RUST_DOC_PREFIX + '/alloc/string/struct.String.html':
                 self.ty = '`string`'
             elif self.ty == RUST_DOC_PREFIX + '/core/option/enum.Option.html':

rpc/Cargo.toml

@@ -43,6 +43,7 @@ ckb-tx-pool = { path = "../tx-pool", version = "= 0.106.0-pre" }
 ckb-memory-tracker = { path = "../util/memory-tracker", version = "= 0.106.0-pre" }
 ckb-pow = { path = "../pow", version = "= 0.106.0-pre" }
 ckb-indexer = { path = "../util/indexer", version = "= 0.106.0-pre" }
+itertools = "0.10.5"
 tokio = "1"
 
 [dev-dependencies]

rpc/README.md

@@ -52,6 +52,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.61.0.
         * [Method `get_consensus`](#method-get_consensus)
         * [Method `get_block_median_time`](#method-get_block_median_time)
         * [Method `estimate_cycles`](#method-estimate_cycles)
+        * [Method `get_fee_rate_statics`](#method-get_fee_rate_statics)
     * [Module Experiment](#module-experiment)
         * [Method `dry_run_transaction`](#method-dry_run_transaction)
         * [Method `calculate_dao_maximum_withdraw`](#method-calculate_dao_maximum_withdraw)
@@ -105,6 +106,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.61.0.
     * [Type `BlockEconomicState`](#type-blockeconomicstate)
     * [Type `BlockIssuance`](#type-blockissuance)
     * [Type `BlockNumber`](#type-blocknumber)
+    * [Type `BlockResponse`](#type-blockresponse)
     * [Type `BlockTemplate`](#type-blocktemplate)
     * [Type `BlockView`](#type-blockview)
     * [Type `Byte32`](#type-byte32)
@@ -130,6 +132,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.61.0.
     * [Type `EpochNumberWithFraction`](#type-epochnumberwithfraction)
     * [Type `EpochView`](#type-epochview)
     * [Type `EstimateCycles`](#type-estimatecycles)
+    * [Type `FeeRateStatics`](#type-feeratestatics)
     * [Type `H256`](#type-h256)
     * [Type `HardForkFeature`](#type-hardforkfeature)
     * [Type `Header`](#type-header)
@@ -281,10 +284,11 @@ A cell is live if
 *   it is not found as an input in any transaction in the canonical chain.
 
 #### Method `get_block`
-* `get_block(block_hash, verbosity)`
+* `get_block(block_hash, verbosity, with_cycles)`
     * `block_hash`: [`H256`](#type-h256)
     * `verbosity`: [`Uint32`](#type-uint32) `|` `null`
-* result: [`BlockView`](#type-blockview) `|` [`SerializedBlock`](#type-serializedblock) `|` `null`
+    * `with_cycles`: `boolean` `|` `null`
+* result: [`BlockResponse`](#type-blockresponse) `|` `null`
 
 Returns the information about a block by hash.
 
@@ -294,6 +298,8 @@ Returns the information about a block by hash.
 
 *   `verbosity` - result format which allows 0 and 2. (**Optional**, the default is 2.)
 
+*   `with_cycles` - whether the return cycles of block transactions. (**Optional**, default false.)
+
 ###### Returns
 
 The RPC returns a block or null. When the RPC returns a block, the block hash must equal to the parameter `block_hash`.
@@ -315,7 +321,9 @@ Request
   "jsonrpc": "2.0",
   "method": "get_block",
   "params": [
-    "0xa5f5c85987a15de25661e5a214f2c1449cd803f071acc7999820f25246471f40"
+    "0xa5f5c85987a15de25661e5a214f2c1449cd803f071acc7999820f25246471f40",
+     null,
+     true
   ]
 }
 ```
@@ -378,7 +386,8 @@ Response
         ]
       }
     ],
-    "uncles": []
+    "uncles": [],
+    "cycles": []
   }
 }
 ```
@@ -397,10 +406,11 @@ The response looks like below when `verbosity` is 0.
 
 
 #### Method `get_block_by_number`
-* `get_block_by_number(block_number, verbosity)`
+* `get_block_by_number(block_number, verbosity, with_cycles)`
     * `block_number`: [`BlockNumber`](#type-blocknumber)
     * `verbosity`: [`Uint32`](#type-uint32) `|` `null`
-* result: [`BlockView`](#type-blockview) `|` [`SerializedBlock`](#type-serializedblock) `|` `null`
+    * `with_cycles`: `boolean` `|` `null`
+* result: [`BlockResponse`](#type-blockresponse) `|` `null`
 
 Returns the block in the [canonical chain](#canonical-chain) with the specific block number.
 
@@ -410,6 +420,8 @@ Returns the block in the [canonical chain](#canonical-chain) with the specific b
 
 *   `verbosity` - result format which allows 0 and 2. (**Optional**, the default is 2.)
 
+*   `with_cycles` - whether the return cycles of block transactions. (**Optional**, default false.)
+
 ###### Returns
 
 The RPC returns the block when `block_number` is less than or equal to the tip block number returned by [`get_tip_block_number`](#method-get_tip_block_number) and returns null otherwise.
@@ -500,7 +512,8 @@ Response
         ]
       }
     ],
-    "uncles": []
+    "uncles": [],
+    "cycles": null
   }
 }
 ```
@@ -827,6 +840,7 @@ Response
       "version": "0x0",
       "witnesses": []
     },
+    "cycles": "0x219",
     "tx_status": {
       "block_hash": null,
       "status": "pending",
@@ -846,6 +860,7 @@ The response looks like below when `verbosity` is 0.
   "jsonrpc": "2.0",
   "result": {
     "transaction": "0x.....",
+    "cycles": "0x219",
     "tx_status": {
       "block_hash": null,
       "status": "pending",
@@ -1661,6 +1676,51 @@ Response
 ```
 
 
+#### Method `get_fee_rate_statics`
+* `get_fee_rate_statics(target)`
+    * `target`: [`Uint64`](#type-uint64) `|` `null`
+* result: [`FeeRateStatics`](#type-feeratestatics) `|` `null`
+
+Returns the fee_rate statistics of confirmed blocks on the chain
+
+###### Params
+
+*   `target` - Specify the number (1 - 101) of confirmed blocks to be counted. If the number is even, automatically add one. If not specified, defaults to 21
+
+###### Returns
+
+If the query has data records, it returns statistics, if not, it returns null.
+
+###### Examples
+
+Request
+
+
+```
+{
+  "id": 42,
+  "jsonrpc": "2.0",
+  "method": "get_fee_rate_statics",
+  "params": []
+}
+```
+
+
+Response
+
+
+```
+{
+  "id": 42,
+  "jsonrpc": "2.0",
+  "result": {
+    "mean":59.29387293275573,
+    "median":5.288207297726071
+   }
+}
+```
+
+
 ### Module Experiment
 
 RPC Module Experiment for experimenting methods.
@@ -4946,6 +5006,19 @@ Consecutive block number starting from 0.
 
 This is a 64-bit unsigned integer type encoded as the 0x-prefixed hex string in JSON. See examples of [Uint64](#type-uint64).
 
+### Type `BlockResponse`
+
+The wrapper represent response of `get_block` | `get_block_by_number`, return a Block with cycles.
+
+#### Fields
+
+`BlockResponse` is a JSON object with the following fields.
+
+*   `block`: [`ResponseFormat`](#type-responseformat) - The block structure
+
+*   `cycles`: `Array<` [`Cycle`](#type-cycle) `>` `|` `null` - The block transactions consumed cycles.
+
+
 ### Type `BlockTemplate`
 
 A block template for miners.
@@ -5559,6 +5632,19 @@ Response result of the RPC method `estimate_cycles`.
 *   `cycles`: [`Cycle`](#type-cycle) - The count of cycles that the VM has consumed to verify this transaction.
 
 
+### Type `FeeRateStatics`
+
+The fee_rate statistics information, includes mean and median
+
+#### Fields
+
+`FeeRateStatics` is a JSON object with the following fields.
+
+*   `mean`: `64-bit floating point` - mean
+
+*   `median`: `64-bit floating point` - median
+
+
 ### Type `H256`
 
 The 32-byte fixed-length binary data.
@@ -6573,6 +6659,8 @@ The JSON view of a transaction as well as its status.
 
 *   `transaction`: [`ResponseFormat`](#type-responseformat) `|` `null` - The transaction.
 
+*   `cycles`: [`Cycle`](#type-cycle) `|` `null` - The transaction consumed cycles.
+
 *   `tx_status`: [`TxStatus`](#type-txstatus) - The Transaction status.