Merge branch 'master' into dependabot/cargo/tracing-0.1.33

.github/workflows/build_test.yml

@@ -50,7 +50,7 @@ jobs:
           command: test
           args: -p starcoin-move-prover
       - name: check changed files
-        run: bash ./scripts/changed-files.sh
+        run: bash ./scripts/changed_files.sh
       - name: integration test dev environment
         env:
           RUST_LOG: info

contrib-contracts/src/starcoin_merkle_test.rs

@@ -61,7 +61,7 @@ fn test_starcoin_merkle() -> Result<()> {
     }
 
     {
-        // change to previout state root.
+        // change to previous state root.
         let old_chain_state = chain_state.fork_at(state_root);
         // let state_root = chain_state.state_root();
         let _expected_root = MoveValue::vector_u8(state_root.to_vec());

executor/src/block_executor.rs

@@ -63,11 +63,6 @@ pub fn block_execute(
                 let txn_state_root = chain_state
                     .commit()
                     .map_err(BlockExecutorError::BlockChainStateErr)?;
-                //every transaction's state tree root and tree nodes should save to storage
-                //TODO merge database flush.
-                chain_state
-                    .flush()
-                    .map_err(BlockExecutorError::BlockChainStateErr)?;
 
                 executed_data.txn_infos.push(TransactionInfo::new(
                     txn_hash,

scripts/auto_rerun_test.ps1

@@ -0,0 +1,31 @@
+$STARCOIN_DIR = (get-item $PSScriptRoot).parent.FullName
+$TEST_RESULT_FILE="$STARCOIN_DIR/target/debug/test_result.txt"
+$TEST_RESULT_FAILED_FILE="$STARCOIN_DIR/target/debug/test_result_failed.txt"
+Write-Host "Starcoin root dir: $STARCOIN_DIR"
+
+$env:RUSTFLAGS="-Ccodegen-units=1 -Copt-level=0"
+$env:RUSTC_BOOTSTRAP=1
+$env:RUST_MIN_STACK=8*1024*1024
+$env:RUST_LOG="OFF"
+$env:RUST_BACKTRACE=0
+cargo xtest --exclude starcoin-move-prover -j 15 -- --test-threads=10 --color never --format pretty | Tee-Object "$TEST_RESULT_FILE"
+
+$failed_tests=Select-String -Pattern 'test .* \.\.\. FAILED' -Path "./target/debug/test_result.txt" -AllMatche
+Write-Host "All failed tests are redirected to file: $TEST_RESULT_FAILED_FILE" -ForegroundColor Green
+$failed_tests > "$TEST_RESULT_FAILED_FILE"
+
+Write-Host "Retring failed test cases" -ForegroundColor Green
+$env:RUST_LOG="DEBUG"
+$env:RUST_BACKTRACE="FULL"
+$case_status=0
+$failed_tests | ForEach-Object {
+    $test=$_ -split ' '
+    $test=$test[1]
+    Write-Host "Rerunning test failed case: $test" -ForegroundColor Red
+    cargo xtest -j 15 "$test" -- --test-threads=1 --nocapture
+    if ($LASTEXITCODE -ne 0) {
+        $case_status=$LASTEXITCODE
+        Write-Host "Test case $test failed with $case_status" -ForegroundColor Red
+    }
+}
+exit $case_status

scripts/changed_files.ps1

@@ -0,0 +1,5 @@
+$diff=$(git diff)
+Write-Host "$diff"
+
+$changed_files=$(git status --porcelain)
+Write-Host "$changed_files"

scripts/check_commit.ps1

@@ -0,0 +1,16 @@
+$STARCOIN_DIR = (get-item $PSScriptRoot).parent.FullName
+
+# cargo fmt check
+cargo fmt -- --check
+# cargo clippy check
+cargo clippy --all-targets -- -D warnings
+# generate stdlib
+cargo run -p stdlib
+# generate genesis
+cargo run -p starcoin-genesis
+# generate rpc schema document
+cargo run -p starcoin-rpc-api -- -d ./rpc/generated_rpc_schema
+# test config file
+cargo test -p starcoin-config  test_example_config_compact
+# check changed files
+. "${STARCOIN_DIR}/scripts/changed_files.ps1"

scripts/check_commit.sh

@@ -19,4 +19,4 @@ cargo run -p starcoin-rpc-api -- -d ./rpc/generated_rpc_schema
 # test config file
 cargo test -p starcoin-config  test_example_config_compact
 # check changed files
-"${STARCOIN_DIR}"/scripts/changed-files.sh
+"${STARCOIN_DIR}"/scripts/changed_files.sh

scripts/dev_setup.ps1

@@ -0,0 +1,133 @@
+<#
+.SYNOPSIS
+Usage:
+Installs or updates necessary dev tools for starcoinorg/starcoin.
+-p update environment variables only
+-t install build tools
+-y installs or updates Move prover tools: z3, cvc5, dotnet, boogie
+-dir <path> - directory to install to
+.DESCRIPTION
+helper for setting up starcoin development environment
+#>
+param(
+[Parameter()]
+[Alias('p')]
+[switch]$INSTALL_PROFILE,
+
+[Parameter()]
+[Alias('t')]
+[switch]$INSTALL_BUILD_TOOLS,
+
+[Parameter()]
+[Alias('y')]
+[switch]$INSTALL_PROVER,
+
+[Parameter()]
+[Alias('dir')]
+[string]$INSTALL_DIR="${HOME}\.starcoin_deps"
+)
+
+
+
+$Z3_VERSION="4.8.13"
+$CVC5_VERSION="1.0.0"
+$BOOGIE_VERSION="2.9.6"
+
+Write-Host "INSTALL_PROFILE=$INSTALL_PROFILE"
+Write-Host "INSTALL_BUILD_TOOLS=$INSTALL_BUILD_TOOLS"
+Write-Host "INSTALL_PROVER=$INSTALL_PROVER"
+Write-Host "INSTALL_DIR=$INSTALL_DIR"
+
+# check environment if exist
+# $key env $value value
+function check_set_env {
+    param(
+    [Parameter(Mandatory=$true)]
+    [string]$key,
+    [Parameter(Mandatory=$true)]
+    [string]$value
+    )
+    [string] $env_value = [Environment]::GetEnvironmentVariable($key, 'User')
+    if($env_value -ne $value){
+        Write-Host "set $key=$value"
+        [Environment]::SetEnvironmentVariable($key,$value,'User')
+    }else{
+        Write-Host "Environment variable $key is set"
+    }
+}
+
+# set env and path variables
+function set_env_path {
+    Write-Host "Setting environment variables for profile"
+    check_set_env "Z3_EXE" "$INSTALL_DIR\z3\z3.exe"
+    check_set_env "CVC5_EXE" "$INSTALL_DIR\cvc5.exe"
+    check_set_env "BOOGIE_EXE" "$INSTALL_DIR\tools\boogie\"
+}
+
+function install_z3 {
+    $z3pkg = "z3-$Z3_VERSION-x64-win"
+    # download z3
+    Invoke-WebRequest -Uri "https://github.com/Z3Prover/z3/releases/download/z3-$z3_version/$z3pkg.zip" -OutFile "$INSTALL_DIR\z3\z3.zip"
+    # unzip z3
+    Write-Host "Unzipping z3"
+    Expand-Archive "$INSTALL_DIR\z3\z3.zip" -DestinationPath "$INSTALL_DIR\z3"
+    # remove z3.zip
+    Remove-Item "$INSTALL_DIR\z3\z3.zip" -Force
+    # mv z3.exe into z3 path
+    Copy-Item "$INSTALL_DIR\z3\$z3pkg\bin\z3.exe" "$INSTALL_DIR\z3\z3.exe"
+}
+
+function install_cvc5 {
+    Invoke-WebRequest -Uri "https://github.com/cvc5/cvc5/releases/download/cvc5-$CVC5_VERSION/cvc5-Win64.exe" -OutFile "$INSTALL_DIR\cvc5.exe"
+}
+
+function install_boogie {
+    dotnet tool update --tool-path "$INSTALL_DIR\tools\boogie" Boogie --version $BOOGIE_VERSION
+}
+
+function install_build_tools {
+    try {
+        clang 2>&1>$null
+    } catch {
+        Write-Error "Clang not found, installing llvm and clang"
+        $llvmVersion = "12.0.0"
+        Write-Host "Installing LLVM $llvmVersion ..." -ForegroundColor Cyan
+        Write-Host "Downloading..."
+        $exePath = "$env:temp\LLVM-$llvmVersion-win64.exe"
+        Invoke-WebRequest -Uri "https://github.com/llvm/llvm-project/releases/download/llvmorg-$llvmVersion/LLVM-$llvmVersion-win64.exe" -OutFile $exePath
+        Write-Host "Installing..."
+        cmd /c start $exePath
+        Write-Host "Installed" -ForegroundColor Green
+        [Environment]::SetEnvironmentVariable("Path", $env:Path + ";C:\Program Files\LLVM\bin", 'User')
+    }
+    try {
+        dotnet 2>&1>$null
+    } catch {
+        Write-Error "Dotnet sdk not found, installing dotnet sdk!"
+        $exePath = "$env:temp\dotnet-sdk-6.0.202-win-x64.exe"
+        Invoke-WebRequest -Uri "https://download.visualstudio.microsoft.com/download/pr/e4f4bbac-5660-45a9-8316-0ffc10765179/8ade57de09ce7f12d6411ed664f74eca/dotnet-sdk-6.0.202-win-x64.exe" -OutFile $exePath
+        Write-Host "Installing..."
+        cmd /c start $exePath
+        Write-Host "Installed" -ForegroundColor Green
+    }
+    try {
+        cargo 2>&1>$null
+    } catch {
+        throw "install rust by yourself please"
+    }
+}
+
+if ($INSTALL_PROFILE -eq $true) {
+    Write-Host "Installing profile"
+    set_env_path
+}
+if ($INSTALL_BUILD_TOOLS -eq $true) {
+    Write-Host "Installing build tools"
+    install_build_tools
+}
+if ($INSTALL_PROVER -eq $true) {
+    Write-Host "Installing prover"
+    install_z3
+    install_cvc5
+    install_boogie
+}

state/state-tree/Cargo.toml

@@ -17,6 +17,7 @@ serde = { version = "1.0.130" }
 forkable-jellyfish-merkle = { path = "../../commons/forkable-jellyfish-merkle"}
 starcoin-state-store-api = {path = "../state-store-api"}
 bcs-ext = { package="bcs-ext", path = "../../commons/bcs_ext" }
+logger = { path = "../../commons/logger", package="starcoin-logger"}
 
 [dev-dependencies]
 starcoin-config= { path = "../../config"}

state/state-tree/src/state_tree.rs

@@ -6,6 +6,7 @@ use forkable_jellyfish_merkle::proof::SparseMerkleProof;
 use forkable_jellyfish_merkle::{
     JellyfishMerkleTree, RawKey, StaleNodeIndex, TreeReader, TreeUpdateBatch,
 };
+use logger::prelude::debug;
 use parking_lot::{Mutex, RwLock};
 use starcoin_crypto::hash::*;
 use starcoin_state_store_api::*;
@@ -23,7 +24,8 @@ use std::sync::Arc;
 #[derive(Clone)]
 pub struct StateCache<K: RawKey> {
     root_hash: HashValue,
-    change_set: TreeUpdateBatch<K>,
+    change_set_list: Vec<(HashValue, TreeUpdateBatch<K>)>,
+    split_off_idx: Option<usize>,
 }
 
 impl<K> StateCache<K>
@@ -33,17 +35,22 @@ where
     pub fn new(initial_root: HashValue) -> Self {
         Self {
             root_hash: initial_root,
-            change_set: TreeUpdateBatch::default(),
+            change_set_list: Vec::new(),
+            split_off_idx: None,
         }
     }
 
     fn reset(&mut self, root_hash: HashValue) {
         self.root_hash = root_hash;
-        self.change_set = TreeUpdateBatch::default();
+        self.change_set_list = if let Some(split_idx) = self.split_off_idx {
+            self.change_set_list.split_off(split_idx)
+        } else {
+            Vec::new()
+        };
     }
 
     fn add_changeset(&mut self, root_hash: HashValue, cs: TreeUpdateBatch<K>) {
-        let cur_change_set = &mut self.change_set;
+        let mut cur_change_set = TreeUpdateBatch::default();
         let mut cs_num_stale_leaves = cs.num_stale_leaves;
         for stale_node in cs.stale_node_index_batch.iter() {
             match cur_change_set.node_batch.remove(&stale_node.node_key) {
@@ -70,7 +77,7 @@ where
                 cur_change_set.num_new_leaves += 1;
             }
         }
-
+        self.change_set_list.push((root_hash, cur_change_set));
         self.root_hash = root_hash;
     }
 }
@@ -196,11 +203,25 @@ where
 
     /// commit the state change into underline storage.
     pub fn flush(&self) -> Result<()> {
-        let (root_hash, change_sets) = self.change_sets();
+        let change_set_list = {
+            let mut cache_guard = self.cache.lock();
+            cache_guard.split_off_idx = Some(cache_guard.change_set_list.len());
+            cache_guard.change_set_list.clone()
+        };
 
+        debug!("change_set_list len {}", change_set_list.len());
+        // when self::commit call self::updates(&self, updates: Vec<(K, Option<Blob>)>)
+        // the param updates is empty cause this situation
+        if change_set_list.is_empty() {
+            return Ok(());
+        }
+        let mut root_hash = HashValue::default();
         let mut node_map = BTreeMap::new();
-        for (nk, n) in change_sets.node_batch.into_iter() {
-            node_map.insert(nk, n.try_into()?);
+        for (hash, change_sets) in change_set_list.into_iter() {
+            for (nk, n) in change_sets.node_batch.into_iter() {
+                node_map.insert(nk, n.try_into()?);
+            }
+            root_hash = hash;
         }
         self.storage.write_nodes(node_map)?;
         // and then advance the storage root hash
@@ -259,9 +280,6 @@ where
         };
         let tree = JellyfishMerkleTree::new(&reader);
         let (new_state_root, change_set) = tree.updates(Some(cur_root_hash), updates)?;
-        // cache.root_hashes.push(new_state_root);
-        // cache.change_sets.push(change_set);
-        // cache.root_hash = new_state_root;
         cache.add_changeset(new_state_root, change_set);
         Ok(new_state_root)
     }
@@ -296,10 +314,18 @@ where
     // }
 
     /// get all changes so far based on initial root_hash.
+    /*
     pub fn change_sets(&self) -> (HashValue, TreeUpdateBatch<K>) {
         let cache_guard = self.cache.lock();
         (cache_guard.root_hash, cache_guard.change_set.clone())
+    } */
+
+    /// get last changes root_hash
+    pub fn last_change_sets(&self) -> Option<(HashValue, TreeUpdateBatch<K>)> {
+        let cache_gurad = self.cache.lock();
+        cache_gurad.change_set_list.last().cloned()
     }
+
     // TODO: to keep atomic with other commit.
     // TODO: think about the WriteBatch trait position.
     // pub fn save<T>(&self, batch: &mut T) -> Result<()>
@@ -327,8 +353,10 @@ where
         if node_key == &*SPARSE_MERKLE_PLACEHOLDER_HASH {
             return Ok(Some(Node::new_null()));
         }
-        if let Some(n) = self.cache.change_set.node_batch.get(node_key).cloned() {
-            return Ok(Some(n));
+        for change_set in self.cache.change_set_list.iter().rev() {
+            if let Some(n) = change_set.1.node_batch.get(node_key).cloned() {
+                return Ok(Some(n));
+            }
         }
         match self.store.get(node_key) {
             Ok(Some(n)) => Ok(Some(n.try_into()?)),
@@ -350,8 +378,10 @@ where
         if node_key == &*SPARSE_MERKLE_PLACEHOLDER_HASH {
             return Ok(Some(Node::new_null()));
         }
-        if let Some(n) = self.cache.change_set.node_batch.get(node_key).cloned() {
-            return Ok(Some(n));
+        for change_set in self.cache.change_set_list.iter().rev() {
+            if let Some(n) = change_set.1.node_batch.get(node_key).cloned() {
+                return Ok(Some(n));
+            }
         }
         match self.store.get(node_key) {
             Ok(Some(n)) => Ok(Some(n.try_into()?)),