feat: introduce candid-extractor for candid export (#424)

* remove wasi feature

* export_candid

* examples and CI

* speed up ic-wasm install

* fix

* add candid-extractor crate

* update readme and ic0.txt

* update CI

* fix CI

* try again

* fix ci

* fix

* bump versions and update changelog

* update date

* Update src/ic-cdk-timers/CHANGELOG.md

Co-authored-by: Adam Spofford <93943719+adamspofford-dfinity@users.noreply.github.com>

---------

Co-authored-by: Adam Spofford <93943719+adamspofford-dfinity@users.noreply.github.com>

.github/workflows/ci.yml

@@ -40,8 +40,8 @@ jobs:
           target: wasm32-unknown-unknown
       - name: Run builds
         run: |
-          cargo build --workspace --exclude ic-cdk-e2e-tests --target wasm32-unknown-unknown
-          cargo build --workspace --exclude ic-cdk-e2e-tests --target wasm32-unknown-unknown --release
+          cargo build --workspace --exclude ic-cdk-e2e-tests --exclude candid-extractor --target wasm32-unknown-unknown
+          cargo build --workspace --exclude ic-cdk-e2e-tests --exclude candid-extractor --target wasm32-unknown-unknown --release
           cargo build --example=work
 
   test:

.github/workflows/examples.yml

@@ -13,11 +13,45 @@ concurrency:
 env:
   rust-version: 1.65.0
   dfx-version: 0.14.1
-  wasmtime-version: 10.0.1
+  ic-wasm-version: 0.4.0
 
 jobs:
+  build-candid-extractor:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-candid-extractor-${{ hashFiles('**/Cargo.toml', 'rust-toolchain.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-candid-extractor-
+            ${{ runner.os }}-
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: ${{ env.rust-version }}
+
+      - name: Build candid-extractor
+        run: cargo build -p candid-extractor --release
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: candid-extractor
+          path: target/release/candid-extractor
+
   test:
     runs-on: ubuntu-latest
+    needs: build-candid-extractor
     strategy:
       fail-fast: false
       matrix:
@@ -52,15 +86,19 @@ jobs:
       - name: Install ic-wasm
         # might already in cache
         run: |
-          if ! [ -x "$(command -v ic-wasm)" ]; then
-            cargo install ic-wasm
-          fi
+          wget https://github.com/dfinity/ic-wasm/releases/download/${{env.ic-wasm-version }}/ic-wasm-linux64
+          chmod 755 ic-wasm-linux64
+          mv ic-wasm-linux64 /usr/local/bin/ic-wasm
+
+      - name: Download candid-extractor
+        uses: actions/download-artifact@v3
+        with:
+          name: candid-extractor
 
-      - name: Install wasmtime
+      - name: Install candid-extractor
         run: |
-          wget https://github.com/bytecodealliance/wasmtime/releases/download/v${{env.wasmtime-version}}/wasmtime-v${{env.wasmtime-version}}-x86_64-linux.tar.xz
-          tar xf wasmtime-v${{env.wasmtime-version}}-x86_64-linux.tar.xz
-          mv wasmtime-v${{env.wasmtime-version}}-x86_64-linux/wasmtime /usr/local/bin/
+          chmod 755 candid-extractor
+          mv candid-extractor /usr/local/bin/candid-extractor
 
       - name: Install DFX
         run: |

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "src/ic-cdk-bindgen",
     "src/ic-cdk-macros",
     "src/ic-cdk-timers",
+    "src/candid-extractor",
     "library/ic-certified-map",
     "library/ic-ledger-types",
     "e2e-tests",
@@ -30,11 +31,11 @@ lto = true
 opt-level = 'z'
 
 [workspace.dependencies]
-ic0 = { path = "src/ic0", version = "0.18.11" }
-ic-cdk = { path = "src/ic-cdk", version = "0.10" }
-ic-cdk-timers = { path = "src/ic-cdk-timers", version = "0.4.0" }
+ic0 = { path = "src/ic0", version = "0.18.12" }
+ic-cdk = { path = "src/ic-cdk", version = "0.11" }
+ic-cdk-timers = { path = "src/ic-cdk-timers", version = "0.5.0" }
 
-candid = "0.9"
+candid = "0.9.6"
 futures = "0.3"
 hex = "0.4"
 quote = "1"

README.md

@@ -36,6 +36,8 @@ Generate Rust bindings from Candid to make inter-canister calls.
 Annotate functions with attribute macros to make them exposed public interfaces.
 - [`ic-cdk-timers`](src/ic-cdk-timers):
 The library implements multiple and periodic timers.
+- [`candid-extractor`](src/candid-extractor/):
+A CLI tool to extract candid definition from canister WASM.
 - [`ic-certified-map`](library/ic-certified-map):
 An implementation of map which support *certified queries*.
 - [`ic-ledger-types`](library/ic-ledger-types):
@@ -51,7 +53,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 candid = "0.9" # this version is required if you want to define Candid data types
-ic-cdk = "0.10"
+ic-cdk = "0.11"
 ```
 
 Then in your rust source code:

examples/build.sh

@@ -7,24 +7,17 @@ root="$(dirname "$0")/.."
 example_root="$(dirname "$0")/$name"
 did_file="/tmp/a.did"
 
-# This script generates the did file, build the project (passed as $1) and then run the ic-wasm to shrink and attach metadata.
-cargo build --manifest-path="$example_root/Cargo.toml" \
-    --target wasm32-unknown-unknown \
-    --release \
-    --package "$package" --features "ic-cdk/wasi"
-
-wasmtime "$example_root/target/wasm32-unknown-unknown/release/$package.wasm" > $did_file
-
 cargo build --manifest-path="$example_root/Cargo.toml" \
     --target wasm32-unknown-unknown \
     --release \
     --package "$package"
 
+candid-extractor "$example_root/target/wasm32-unknown-unknown/release/$package.wasm" 2>/dev/null > $did_file || true
+
 ic-wasm "$example_root/target/wasm32-unknown-unknown/release/$package.wasm" \
     -o "$example_root/target/wasm32-unknown-unknown/release/$package.wasm" \
     metadata candid:service -v public -f $did_file
 
 ic-wasm "$example_root/target/wasm32-unknown-unknown/release/$package.wasm" \
     -o "$example_root/target/wasm32-unknown-unknown/release/$package-opt.wasm" \
     shrink
-

examples/counter/src/inter2_rs/build.rs

@@ -4,8 +4,8 @@ use std::path::PathBuf;
 fn main() {
     let manifest_dir =
         PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("Cannot find manifest dir"));
-    let counter = Config::new("inter_mo");
+    let inter_mo = Config::new("inter_mo");
     let mut builder = Builder::new();
-    builder.add(counter);
+    builder.add(inter_mo);
     builder.build(Some(manifest_dir.join("declarations")));
 }

examples/profile/src/profile_inter_rs/build.rs

@@ -4,8 +4,8 @@ use std::path::PathBuf;
 fn main() {
     let manifest_dir =
         PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("Cannot find manifest dir"));
-    let counter = Config::new("profile_rs");
+    let profile_rs = Config::new("profile_rs");
     let mut builder = Builder::new();
-    builder.add(counter);
+    builder.add(profile_rs);
     builder.build(Some(manifest_dir.join("declarations")));
 }

examples/profile/src/profile_inter_rs/lib.rs

@@ -5,7 +5,7 @@ use declarations::profile_rs::{profile_rs, Profile};
 
 #[update(name = "getSelf")]
 async fn get_self() -> Profile {
-    profile_rs.getSelf().await.unwrap().0
+    profile_rs.get_self().await.unwrap().0
 }
 
 #[update]

ic0.txt

@@ -0,0 +1,68 @@
+ic0.msg_arg_data_size : () -> i32;                                          // I U Q CQ Ry CRy F
+ic0.msg_arg_data_copy : (dst : i32, offset : i32, size : i32) -> ();        // I U Q CQ Ry CRy F
+ic0.msg_caller_size : () -> i32;                                            // *
+ic0.msg_caller_copy : (dst : i32, offset: i32, size : i32) -> ();           // *
+ic0.msg_reject_code : () -> i32;                                            // Ry Rt CRy CRt
+ic0.msg_reject_msg_size : () -> i32;                                        // Rt CRt
+ic0.msg_reject_msg_copy : (dst : i32, offset : i32, size : i32) -> ();      // Rt CRt
+
+ic0.msg_reply_data_append : (src : i32, size : i32) -> ();                  // U Q CQ Ry Rt CRy CRt
+ic0.msg_reply : () -> ();                                                   // U Q CQ Ry Rt CRy CRt
+ic0.msg_reject : (src : i32, size : i32) -> ();                             // U Q CQ Ry Rt CRy CRt
+
+ic0.msg_cycles_available : () -> i64;                                       // U Rt Ry
+ic0.msg_cycles_available128 : (dst : i32) -> ();                            // U Rt Ry
+ic0.msg_cycles_refunded : () -> i64;                                        // Rt Ry
+ic0.msg_cycles_refunded128 : (dst : i32) -> ();                             // Rt Ry
+ic0.msg_cycles_accept : (max_amount : i64) -> (amount : i64);               // U Rt Ry
+ic0.msg_cycles_accept128 : (max_amount_high : i64, max_amount_low: i64, dst : i32)
+                       -> ();                                               // U Rt Ry
+
+ic0.canister_self_size : () -> i32;                                         // *
+ic0.canister_self_copy : (dst : i32, offset : i32, size : i32) -> ();       // *
+ic0.canister_cycle_balance : () -> i64;                                     // *
+ic0.canister_cycle_balance128 : (dst : i32) -> ();                          // *
+ic0.canister_status : () -> i32;                                            // *
+ic0.canister_version : () -> i64;                                           // *
+
+ic0.msg_method_name_size : () -> i32;                                       // F
+ic0.msg_method_name_copy : (dst : i32, offset : i32, size : i32) -> ();     // F
+ic0.accept_message : () -> ();                                              // F
+
+ic0.call_new :                                                              // U CQ Ry Rt CRy CRt T
+  ( callee_src  : i32,
+    callee_size : i32,
+    name_src : i32,
+    name_size : i32,
+    reply_fun : i32,
+    reply_env : i32,
+    reject_fun : i32,
+    reject_env : i32
+  ) -> ();
+ic0.call_on_cleanup : (fun : i32, env : i32) -> ();                         // U CQ Ry Rt CRy CRt T
+ic0.call_data_append : (src : i32, size : i32) -> ();                       // U CQ Ry Rt CRy CRt T
+ic0.call_cycles_add : (amount : i64) -> ();                                 // U Ry Rt T
+ic0.call_cycles_add128 : (amount_high : i64, amount_low: i64) -> ();        // U Ry Rt T
+ic0.call_perform : () -> ( err_code : i32 );                                // U CQ Ry Rt CRy CRt T
+
+ic0.stable_size : () -> (page_count : i32);                                 // * s
+ic0.stable_grow : (new_pages : i32) -> (old_page_count : i32);              // * s
+ic0.stable_write : (offset : i32, src : i32, size : i32) -> ();             // * s
+ic0.stable_read : (dst : i32, offset : i32, size : i32) -> ();              // * s
+ic0.stable64_size : () -> (page_count : i64);                               // * s
+ic0.stable64_grow : (new_pages : i64) -> (old_page_count : i64);            // * s
+ic0.stable64_write : (offset : i64, src : i64, size : i64) -> ();           // * s
+ic0.stable64_read : (dst : i64, offset : i64, size : i64) -> ();            // * s
+
+ic0.certified_data_set : (src: i32, size: i32) -> ();                       // I G U Ry Rt T
+ic0.data_certificate_present : () -> i32;                                   // *
+ic0.data_certificate_size : () -> i32;                                      // *
+ic0.data_certificate_copy : (dst: i32, offset: i32, size: i32) -> ();       // *
+
+ic0.time : () -> (timestamp : i64);                                         // *
+ic0.global_timer_set : (timestamp : i64) -> i64;                            // I G U Ry Rt C T
+ic0.performance_counter : (counter_type : i32) -> (counter : i64);          // * s
+ic0.is_controller: (src: i32, size: i32) -> ( result: i32);                 // * s
+
+ic0.debug_print : (src : i32, size : i32) -> ();                            // * s
+ic0.trap : (src : i32, size : i32) -> ();                                   // * s

library/ic-ledger-types/CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 
+## [0.8.0] - 2023-09-18
+
+### Changed
+- Upgrade `ic-cdk` to v0.11.
+
 ## [0.7.0] - 2023-07-13
 
 ### Added

library/ic-ledger-types/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ic-ledger-types"
-version = "0.7.0"
+version = "0.8.0"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true

src/candid-extractor/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "candid-extractor"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+description = "CLI tool to extract candid definition from canister WASM."
+readme = "README.md"
+categories = ["development-tools"]
+keywords = ["internet-computer", "wasm", "dfinity", "canister", "cdk"]
+include = ["src", "Cargo.toml", "LICENSE", "README.md"]
+
+[dependencies]
+anyhow = "1.0.72"
+wasmtime = "12"
+
+[dev-dependencies]
+quote.workspace = true
+syn = { workspace = true, features = ["parsing", "full", "extra-traits"] }
+
+[[example]]
+name = "generate_mock_wat"
+path = "util/generate_mock_wat.rs"

src/candid-extractor/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

src/candid-extractor/README.md

@@ -0,0 +1,30 @@
+# candid-extractor
+
+A CLI tool to extract candid definition from canister WASM.
+
+## Installation
+
+```
+cargo install candid-extractor
+```
+
+## Usage
+
+```
+candid-extractor path/to/canister.wasm
+```
+
+## Update ic_mock.wat
+
+`candid-extractor` requires a mock WASM (`ic_mock.wat`) which provides ic0 imports.
+
+Such `ic_mock.wat` is directly generated from the [system API][1].
+
+When interface-spec releases a new version that modify ic0 system API:
+
+1. replace `ic0.txt` in the root of this project;
+2. execute `cargo run --example=generate_mock_wat`;
+
+`ic_mock.wat` should be updated.
+
+[1]: https://internetcomputer.org/docs/current/references/ic-interface-spec/#system-api-imports

src/candid-extractor/ic0.txt

@@ -0,0 +1 @@
+../../ic0.txt

src/candid-extractor/ic_mock.wat

@@ -0,0 +1,53 @@
+(module
+    ;; This file is generated from ic0.txt.
+    ;; Don't manually modify it.
+    (func (export "msg_arg_data_size")  (result i32) i32.const 0)
+    (func (export "msg_arg_data_copy") (param i32 i32 i32) )
+    (func (export "msg_caller_size")  (result i32) i32.const 0)
+    (func (export "msg_caller_copy") (param i32 i32 i32) )
+    (func (export "msg_reject_code")  (result i32) i32.const 0)
+    (func (export "msg_reject_msg_size")  (result i32) i32.const 0)
+    (func (export "msg_reject_msg_copy") (param i32 i32 i32) )
+    (func (export "msg_reply_data_append") (param i32 i32) )
+    (func (export "msg_reply")  )
+    (func (export "msg_reject") (param i32 i32) )
+    (func (export "msg_cycles_available")  (result i64) i64.const 0)
+    (func (export "msg_cycles_available128") (param i32) )
+    (func (export "msg_cycles_refunded")  (result i64) i64.const 0)
+    (func (export "msg_cycles_refunded128") (param i32) )
+    (func (export "msg_cycles_accept") (param i64) (result i64) i64.const 0)
+    (func (export "msg_cycles_accept128") (param i64 i64 i32) )
+    (func (export "canister_self_size")  (result i32) i32.const 0)
+    (func (export "canister_self_copy") (param i32 i32 i32) )
+    (func (export "canister_cycle_balance")  (result i64) i64.const 0)
+    (func (export "canister_cycle_balance128") (param i32) )
+    (func (export "canister_status")  (result i32) i32.const 0)
+    (func (export "canister_version")  (result i64) i64.const 0)
+    (func (export "msg_method_name_size")  (result i32) i32.const 0)
+    (func (export "msg_method_name_copy") (param i32 i32 i32) )
+    (func (export "accept_message")  )
+    (func (export "call_new") (param i32 i32 i32 i32 i32 i32 i32 i32) )
+    (func (export "call_on_cleanup") (param i32 i32) )
+    (func (export "call_data_append") (param i32 i32) )
+    (func (export "call_cycles_add") (param i64) )
+    (func (export "call_cycles_add128") (param i64 i64) )
+    (func (export "call_perform")  (result i32) i32.const 0)
+    (func (export "stable_size")  (result i32) i32.const 0)
+    (func (export "stable_grow") (param i32) (result i32) i32.const 0)
+    (func (export "stable_write") (param i32 i32 i32) )
+    (func (export "stable_read") (param i32 i32 i32) )
+    (func (export "stable64_size")  (result i64) i64.const 0)
+    (func (export "stable64_grow") (param i64) (result i64) i64.const 0)
+    (func (export "stable64_write") (param i64 i64 i64) )
+    (func (export "stable64_read") (param i64 i64 i64) )
+    (func (export "certified_data_set") (param i32 i32) )
+    (func (export "data_certificate_present")  (result i32) i32.const 0)
+    (func (export "data_certificate_size")  (result i32) i32.const 0)
+    (func (export "data_certificate_copy") (param i32 i32 i32) )
+    (func (export "time")  (result i64) i64.const 0)
+    (func (export "global_timer_set") (param i64) (result i64) i64.const 0)
+    (func (export "performance_counter") (param i32) (result i64) i64.const 0)
+    (func (export "is_controller") (param i32 i32) (result i32) i32.const 0)
+    (func (export "debug_print") (param i32 i32) )
+    (func (export "trap") (param i32 i32) )
+)