Merge pull request #157 from infinity-swap/EPROD-381-investigate-the-…

…evmc-canister-upgrade-problems

EPROD 381 Stable structs get corrupted after upgrade

.github/workflows/build_and_tests.yml

@@ -57,4 +57,29 @@ jobs:
 
       - name: Generate code coverage
         run: |
-          cargo +nightly tarpaulin --verbose --timeout 120 --out Xml --workspace --exclude canister-a --exclude canister-b --exclude canister-c --exclude canister-d --exclude canister-e --exclude test-payment-canister
+          cargo +nightly tarpaulin --verbose --timeout 120 --out Xml --workspace  --exclude canister-a --exclude canister-b --exclude canister-c --exclude canister-d --exclude canister-e --exclude test-payment-canister
+
+  integration-test:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Machine setup
+        run: |
+          sudo apt update 
+          sudo apt install -y libunwind-dev cmake protobuf-compiler
+          sh -ci "$(curl -fsSL https://internetcomputer.org/install.sh)"
+          rustup target add wasm32-unknown-unknown
+          cargo install ic-wasm
+          dfx start --background
+          dfx canister create dummy_canister
+          dfx build dummy_canister
+          ic-payments/build_payments_canister.sh
+
+      - name: Run integration tests
+        run: |
+          export WASMS_DIR="../target/wasm32-unknown-unknown/release/"
+          cd ./ic-stable-structures
+          cargo test --features state-machine

Cargo.toml

@@ -17,6 +17,8 @@ members = [
     "ic-payments",
     "ic-payments/test-payment-canister",
     "ic-stable-structures",
+    "ic-stable-structures/tests/did",
+    "ic-stable-structures/tests/dummy_canister",
     "ic-storage",
     "ic-storage/ic-storage-derive",
 ]

dfx.json

@@ -1,41 +1,47 @@
 {
-      "canisters": {
-            "canister-a": {
-                  "build": "bash scripts/build.sh",
-                  "candid": "ic-canister/tests/canister_a/canister_a.did",
-                  "wasm": "target/wasm32-unknown-unknown/release/canister_a.wasm",
-                  "type": "custom"
-            },
-            "canister-b": {
-                  "build": "bash scripts/build.sh",
-                  "candid": "ic-canister/tests/canister_b/canister_b.did",
-                  "wasm": "target/wasm32-unknown-unknown/release/canister_b.wasm",
-                  "type": "custom"
-            },
-            "canister-c": {
-                  "build": "bash scripts/build.sh",
-                  "candid": "ic-canister/tests/canister-c/canister-c.did",
-                  "wasm": "target/wasm32-unknown-unknown/release/canister-c.wasm",
-                  "type": "custom"
-            },
-            "canister-d": {
-                  "build": "bash scripts/build.sh",
-                  "candid": "ic-canister/tests/canister-d/canister-d.did",
-                  "wasm": "target/wasm32-unknown-unknown/release/canister-d.wasm",
-                  "type": "custom"
-            },
-            "log_canister": {
-              "candid": "ic-log/examples/log_canister.did",
-              "build": "bash scripts/build_log_canister.sh",
-              "wasm": "target/wasm32-unknown-unknown/release/examples/log_canister.wasm",
-              "type": "custom"
-            }
-      },
-      "networks": {
-            "local": {
-                  "bind": "127.0.0.1:8000",
-                  "type": "ephemeral"
-            }
-      },
-      "version": 1
-}
+  "canisters": {
+    "canister-a": {
+      "build": "bash scripts/build.sh",
+      "candid": "ic-canister/tests/canister_a/canister_a.did",
+      "wasm": "target/wasm32-unknown-unknown/release/canister_a.wasm",
+      "type": "custom"
+    },
+    "canister-b": {
+      "build": "bash scripts/build.sh",
+      "candid": "ic-canister/tests/canister_b/canister_b.did",
+      "wasm": "target/wasm32-unknown-unknown/release/canister_b.wasm",
+      "type": "custom"
+    },
+    "canister-c": {
+      "build": "bash scripts/build.sh",
+      "candid": "ic-canister/tests/canister-c/canister-c.did",
+      "wasm": "target/wasm32-unknown-unknown/release/canister-c.wasm",
+      "type": "custom"
+    },
+    "canister-d": {
+      "build": "bash scripts/build.sh",
+      "candid": "ic-canister/tests/canister-d/canister-d.did",
+      "wasm": "target/wasm32-unknown-unknown/release/canister-d.wasm",
+      "type": "custom"
+    },
+    "dummy_canister": {
+      "candid": "ic-stable-structures/tests/dummy_canister/dummy_canister.did",
+      "build": "bash scripts/build_dummy_canister.sh",
+      "wasm": "target/wasm32-unknown-unknown/release/dummy_canister.wasm",
+      "type": "custom"
+    },
+    "log_canister": {
+      "candid": "ic-log/examples/log_canister.did",
+      "build": "bash scripts/build_log_canister.sh",
+      "wasm": "target/wasm32-unknown-unknown/release/examples/log_canister.wasm",
+      "type": "custom"
+    }
+  },
+  "networks": {
+    "local": {
+      "bind": "127.0.0.1:8000",
+      "type": "ephemeral"
+    }
+  },
+  "version": 1
+}

ic-payments/build_payments_canister.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+
+export PROTOC_INCLUDE=${PWD}/../
+
+# Get example icrc1 canister
+if [ ! -f ic-payments/tests/common/ic-icrc1-ledger.wasm ]; then
+    export IC_VERSION=b43543ce7365acd1720294e701e8e8361fa30c8f
+    curl -o ic-icrc1-ledger.wasm.gz https://download.dfinity.systems/ic/${IC_VERSION}/canisters/ic-icrc1-ledger.wasm.gz
+    gunzip ic-icrc1-ledger.wasm.gz
+    mv ic-icrc1-ledger.wasm ic-payments/tests/common/
+fi
+
+# Build test payment canister
+cargo build --target wasm32-unknown-unknown --features export-api -p test-payment-canister --release
+ic-wasm target/wasm32-unknown-unknown/release/test_payment_canister.wasm -o ic-payments/tests/common/payment_canister.wasm shrink

ic-payments/state_machine_test.sh

@@ -1,22 +1,10 @@
+#!/bin/sh
+
 # Example script to run the state_machine tests. To make it work, replace
 # the location of google protobuf repo in your system. You can find the
 # needed repo at https://github.com/protocolbuffers/protobuf/tree/main/src/google/protobuf
 
-set -e
-
-export PROTOC_INCLUDE=${PWD}/../
-
-# Get example icrc1 canister
-if [ ! -f ic-payments/tests/common/ic-icrc1-ledger.wasm ]; then
-    export IC_VERSION=b43543ce7365acd1720294e701e8e8361fa30c8f
-    curl -o ic-icrc1-ledger.wasm.gz https://download.dfinity.systems/ic/${IC_VERSION}/canisters/ic-icrc1-ledger.wasm.gz
-    gunzip ic-icrc1-ledger.wasm.gz
-    mv ic-icrc1-ledger.wasm ic-payments/tests/common/
-fi
-
-# Build test payment canister
-cargo build --target wasm32-unknown-unknown --features export-api -p test-payment-canister --release
-ic-wasm target/wasm32-unknown-unknown/release/test_payment_canister.wasm -o ic-payments/tests/common/payment_canister.wasm shrink
+./build_payments_canister.sh
 
 # Run the test
 cargo +nightly test -p ic-payments --features state-machine

ic-stable-structures/Cargo.toml

@@ -6,3 +6,16 @@ edition.workspace = true
 [dependencies]
 ic-exports = { path = "../ic-exports" }
 thiserror = { workspace = true }
+
+[dev-dependencies]
+anyhow = "1"
+candid = { workspace = true }
+did = { path = "./tests/did" }
+ic-cdk = { workspace = true }
+ic-cdk-macros = { workspace = true }
+once_cell = "1.18"
+serde = { workspace = true }
+
+
+[features]
+state-machine = ["ic-exports/state-machine"]

ic-stable-structures/src/error.rs

@@ -1,4 +1,4 @@
-use ic_exports::stable_structures::{btreemap, cell, log, GrowFailed};
+use ic_exports::stable_structures::{btreemap, cell, log, vec, GrowFailed};
 use thiserror::Error;
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -11,6 +11,10 @@ pub enum Error {
     ValueTooLarge(u64),
     #[error("memory manager and stable structure has incompatible versions")]
     IncompatibleVersions,
+    #[error("the vector type is not compatible with the current vector")]
+    IncompatibleElementType,
+    #[error("bad magic number: {0:?}")]
+    BadMagic([u8; 3]),
 }
 
 impl From<cell::InitError> for Error {
@@ -46,6 +50,17 @@ impl From<log::InitError> for Error {
     }
 }
 
+impl From<vec::InitError> for Error {
+    fn from(e: vec::InitError) -> Self {
+        match e {
+            vec::InitError::IncompatibleVersion(_) => Self::IncompatibleVersions,
+            vec::InitError::IncompatibleElementType => Self::IncompatibleElementType,
+            vec::InitError::OutOfMemory => Self::OutOfStableMemory,
+            vec::InitError::BadMagic(magic) => Self::BadMagic(magic),
+        }
+    }
+}
+
 impl From<GrowFailed> for Error {
     fn from(_: GrowFailed) -> Self {
         Self::OutOfStableMemory

ic-stable-structures/src/storage/structures/vec.rs

@@ -29,7 +29,7 @@ impl<T: BoundedStorable> StableVec<T> {
         self.get_inner().map_or(true, InnerVec::is_empty)
     }
 
-    /// Removes al the values from the vector
+    /// Removes all the values from the vector
     pub fn clear(&mut self) -> Result<()> {
         let memory_id = self.memory_id;
         if let Some(vec) = self.mut_inner() {

ic-stable-structures/src/storage_wasm/structures_wasm.rs

@@ -192,7 +192,7 @@ impl<T: Storable> StableLog<T> {
         let index_memory = crate::get_memory_by_id(index_memory_id);
         let data_memory = crate::get_memory_by_id(data_memory_id);
 
-        let inner = log::Log::new(index_memory, data_memory);
+        let inner = log::Log::init(index_memory, data_memory)?;
         Ok(Self(Some(inner)))
     }
 
@@ -308,7 +308,7 @@ impl<T: BoundedStorable> StableVec<T> {
     /// Creates new `StableVec`
     pub fn new(memory_id: MemoryId) -> Result<Self> {
         Ok(Self(
-            vec::Vec::<T, Memory>::new(get_memory_by_id(memory_id))?,
+            vec::Vec::<T, Memory>::init(get_memory_by_id(memory_id))?,
             memory_id,
         ))
     }

ic-stable-structures/tests/did/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "did"
+version.workspace = true
+edition.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+
+[dependencies]
+candid = { workspace = true }
+ic-stable-structures = { path = "../../../ic-stable-structures" }
+serde = { workspace = true }

ic-stable-structures/tests/did/src/lib.rs

@@ -0,0 +1,36 @@
+use candid::{CandidType, Decode, Deserialize, Encode};
+use ic_stable_structures::{BoundedStorable, ChunkSize, SlicedStorable, Storable};
+
+pub fn encode(item: &impl CandidType) -> Vec<u8> {
+    Encode!(item).expect("failed to encode item to candid")
+}
+
+pub fn decode<'a, T: CandidType + Deserialize<'a>>(bytes: &'a [u8]) -> T {
+    Decode!(bytes, T).expect("failed to decode item from candid")
+}
+
+#[derive(Debug, Default, Clone, Copy, CandidType, Deserialize)]
+pub struct Transaction {
+    pub from: u8,
+    pub to: u8,
+    pub value: u8,
+}
+
+impl Storable for Transaction {
+    fn to_bytes(&self) -> std::borrow::Cow<[u8]> {
+        encode(self).into()
+    }
+
+    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
+        decode(&bytes)
+    }
+}
+
+impl SlicedStorable for Transaction {
+    const CHUNK_SIZE: ChunkSize = 8;
+}
+
+impl BoundedStorable for Transaction {
+    const MAX_SIZE: u32 = 32;
+    const IS_FIXED_SIZE: bool = false;
+}

ic-stable-structures/tests/dummy_canister/.gitignore

@@ -0,0 +1 @@
+!dummy_canister.did

ic-stable-structures/tests/dummy_canister/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "dummy_canister"
+version.workspace = true
+edition.workspace = true
+
+[features]
+default = []
+export-api = []
+
+[dependencies]
+candid = { workspace = true }
+did = { path = "../did" }
+ic-canister = { path = "../../../ic-canister/ic-canister" }
+ic-cdk = { workspace = true }
+ic-exports = { path = "../../../ic-exports" }
+ic-stable-structures = { path = "../../../ic-stable-structures" }
+serde = { workspace = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["rt", "macros"] }

ic-stable-structures/tests/dummy_canister/dummy_canister.did

@@ -0,0 +1,15 @@
+type Transaction = record { to : nat8; value : nat8; from : nat8 };
+service : () -> {
+  get_tx_from_btreemap : (nat64) -> (opt Transaction) query;
+  get_tx_from_cell : () -> (Transaction) query;
+  get_tx_from_log : (nat64) -> (opt Transaction) query;
+  get_tx_from_map : (nat64) -> (opt Transaction) query;
+  get_tx_from_multimap : (nat64) -> (opt Transaction) query;
+  get_tx_from_vec : (nat64) -> (opt Transaction) query;
+  insert_tx_to_btreemap : (Transaction) -> (nat64);
+  insert_tx_to_cell : (Transaction) -> ();
+  insert_tx_to_map : (Transaction) -> (nat64);
+  insert_tx_to_multimap : (Transaction) -> (nat64);
+  push_tx_to_log : (Transaction) -> (nat64);
+  push_tx_to_vec : (Transaction) -> (nat64);
+}