feat(db): add some basic db tests (#567)

* Add some basic db tests

* Configure CI for database tests

* Fix double CI

* Add more output tests

* fmt

* Add integration tests inner crate

* fmt

* fix dependencies

* Revert cargo file formatting

* Fix toml and gitignore

* one more thing

* `bee` -> `chronicle`

* Use integration tests rather than a separate crate

* rename

* clippy

* Re-separate dev deps

* Revert MongoClient

* docs

* gitignore

Co-authored-by: Jochen Görtler <jochen.goertler@iota.org>

.cargo/config.toml

@@ -11,7 +11,8 @@ ci-clippy-api = "clippy --all-targets --no-default-features --features api,stard
 ci-doctest = "test --doc --all-features"
 ci-doc = "doc --all-features --no-deps --document-private-items"
 ci-fmt = "fmt --all -- --check"
-ci-test = "test --all-targets --all-features"
+ci-test = "test --all-features --bins --lib"
+ci-test-int = "test --all-features --test *"
 ci-toml = "sort --grouped --check"
 ci-udeps = "udeps --all-targets --all-features --backend=depinfo"
 

.github/workflows/_test_int.yml

@@ -0,0 +1,45 @@
+name: Integration Tests
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: true
+        type: string
+      rust:
+        required: true
+        type: string
+      mongodb:
+        required: true
+        type: string
+
+jobs:
+  test-int:
+    name: "${{ inputs.os }}, ${{ inputs.rust }}"
+    runs-on: ${{ inputs.os }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install Rust (${{ inputs.rust }})
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ inputs.rust }}
+          override: true
+
+      - uses: Swatinem/rust-cache@v1
+
+      - name: Start MongoDB
+        uses: supercharge/mongodb-github-action@1.7.0
+        with:
+          mongodb-version: ${{ inputs.mongodb }}
+          mongodb-replica-set: test-rs
+
+      - name: Test DB
+        uses: actions-rs/cargo@v1
+        with:
+          command: ci-test-int

.github/workflows/ci.yml

@@ -2,15 +2,17 @@ name: ci
 
 on:
   push:
-    paths-ignore: 
-      - '**.md'
-      - 'bin/inx-chronicle/docker/**'
+    paths-ignore:
+      - "**.md"
+      - "bin/inx-chronicle/docker/**"
+      - "bin/inx-chronicle/docs/**"
     branches:
       - main
   pull_request:
-    paths-ignore: 
-      - '**.md'
-      - 'bin/inx-chronicle/docker/**'
+    paths-ignore:
+      - "**.md"
+      - "bin/inx-chronicle/docker/**"
+      - "bin/inx-chronicle/docs/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -22,14 +24,19 @@ jobs:
     uses: ./.github/workflows/_check.yml
     with: { os: ubuntu-latest, rust: stable }
 
+  test-int:
+    name: "integration tests"
+    uses: ./.github/workflows/_test_int.yml
+    with: { os: ubuntu-latest, rust: stable, mongodb: "5.0" }
+
   format:
     uses: ./.github/workflows/_fmt.yml
     with: { os: ubuntu-latest, rust: nightly }
 
   clippy:
     uses: ./.github/workflows/_clippy.yml
     with: { os: ubuntu-latest, rust: stable }
-  
+
   check-toml:
     runs-on: ubuntu-latest
     steps:
@@ -58,9 +65,9 @@ jobs:
 
       - uses: actions-rs/toolchain@v1
         with:
-            profile: minimal
-            toolchain: nightly
-            override: true
+          profile: minimal
+          toolchain: nightly
+          override: true
 
       - uses: Swatinem/rust-cache@v1
 

.github/workflows/scripts/coverage.sh

@@ -7,9 +7,9 @@ mkdir coverage
 
 # Run tests with profiling instrumentation
 echo "Running instrumented unit tests..."
-RUSTFLAGS="-Zinstrument-coverage" LLVM_PROFILE_FILE="bee-%m.profraw" cargo +nightly test --tests --all --all-features
+RUSTFLAGS="-Zinstrument-coverage" LLVM_PROFILE_FILE="chronicle-%m.profraw" cargo +nightly test --tests --all --all-features
 
-# Merge all .profraw files into "bee.profdata"
+# Merge all .profraw files into "chronicle.profdata"
 echo "Merging coverage data..."
 PROFRAW=""
 for file in $(find . -type f -name "*.profraw");
@@ -18,7 +18,7 @@ do
   PROFRAW="${PROFRAW} $file"
 done
 
-cargo +nightly profdata -- merge ${PROFRAW} -o bee.profdata
+cargo +nightly profdata -- merge ${PROFRAW} -o chronicle.profdata
 
 # List the test binaries
 echo "Locating test binaries..."
@@ -39,7 +39,7 @@ done
 # Generate and export the coverage report to lcov format
 echo "Generating lcov file..."
 cargo +nightly cov -- export ${BINARIES} \
-  --instr-profile=bee.profdata \
+  --instr-profile=chronicle.profdata \
   --ignore-filename-regex="/.cargo|rustc|target|tests|/.rustup" \
   --format=lcov --Xdemangler=rustfilt \
   >> coverage/coverage.info

.gitignore

@@ -2,7 +2,7 @@
 
 **/.env
 
-/bin/inx-chronicle/config.toml
+config.toml
 
 .vscode
 

Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "chronicle"
 version = "1.0.0-beta.13"
-authors = ["IOTA Stiftung"]
+authors = [ "IOTA Stiftung" ]
 edition = "2021"
 description = "IOTA permanode implemented as an IOTA Node Extension (INX)."
 readme = "README.md"
 repository = "https://github.com/iotaledger/inx-chronicle"
 license = "Apache-2.0"
-keywords = ["iota", "storage", "permanode", "chronicle", "inx"]
+keywords = [ "iota", "storage", "permanode", "chronicle", "inx" ]
 homepage = "https://www.iota.org"
 rust-version = "1.60"
 
@@ -32,68 +32,67 @@ dyn-clone = { version = "1.0", default-features = false }
 futures = { version = "0.3", default-features = false }
 humantime = { version = "2.1.0", default-features = false }
 humantime-serde = { version = "1.1", default-features = false }
-mongodb = { version = "2.2", default-features = false, features = ["tokio-runtime"] }
+mongodb = { version = "2.2", default-features = false, features = [ "tokio-runtime" ] }
 pin-project = { version = "1.0", default-features = false }
 prefix-hex = { version = "0.4.0", default-features = false, features = [ "primitive-types" ] }
 primitive-types = { version = "0.11", default-features = false }
-serde = { version = "1.0", features = ["derive"], default-features = false }
+serde = { version = "1.0", features = [ "derive" ], default-features = false }
 serde_bytes = { version = "0.11", default-features = false }
-serde_json = { version = "1.0", default-features = false, features = ["std"] }
+serde_json = { version = "1.0", default-features = false, features = [ "std" ] }
 thiserror = { version = "1.0", default-features = false }
-time = { version = "0.3", default-features = false, features = ["std"] }
+time = { version = "0.3", default-features = false, features = [ "std" ] }
 tokio = { version = "1.19", default-features = false, features = [ "macros", "rt-multi-thread", "signal" ] }
 tokio-stream = { version = "0.1", default-features = false }
 toml = { version = "0.5", default-features = false }
-tracing = { version = "0.1", default-features = false, features = ["std", "attributes"] }
-tracing-futures = { version = "0.2", default-features = false, features = ["std", "std-future"] }
-tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "fmt", "ansi", "smallvec", "tracing-log", "local-time", "env-filter"] }
+tracing = { version = "0.1", default-features = false, features = [ "std", "attributes" ] }
+tracing-futures = { version = "0.2", default-features = false, features = [ "std", "std-future" ] }
+tracing-subscriber = { version = "0.3", default-features = false, features = [ "std", "fmt", "ansi", "smallvec", "tracing-log", "local-time", "env-filter" ] }
 uint = { version = "0.9", default-features = false }
 url = { version = "2.2", default-features = false }
-uuid = { version = "1.1", default-features = false, features = ["v4"] }
+uuid = { version = "1.1", default-features = false, features = [ "v4" ] }
 
 # API
 auth-helper = { version = "0.3", default-features = false, optional = true }
-axum = { version = "0.5", default-features = false, features = ["http1", "json", "query", "original-uri", "headers"], optional = true }
-ed25519 = { version = "1.5", default-features = false, features = ["alloc", "pkcs8", "pem"], optional = true }
-ed25519-dalek = { version = "1.0", default-features = false, features = ["u64_backend"], optional = true }
+axum = { version = "0.5", default-features = false, features = [ "http1", "json", "query", "original-uri", "headers" ], optional = true }
+ed25519 = { version = "1.5", default-features = false, features = [ "alloc", "pkcs8", "pem" ], optional = true }
+ed25519-dalek = { version = "1.0", default-features = false, features = [ "u64_backend" ], optional = true }
 hex = { version = "0.4", default-features = false, optional = true }
-hyper = { version = "0.14", default-features = false, features = ["server", "tcp", "stream"], optional = true }
+hyper = { version = "0.14", default-features = false, features = [ "server", "tcp", "stream" ], optional = true }
 lazy_static = { version = "1.4", default-features = false, optional = true }
 packable = { version = "0.5", default-features = false, optional = true }
-rand = { version = "0.8", default-features = false, features = ["std"], optional = true }
-regex = { version = "1.5", default-features = false, features = ["std"], optional = true }
+rand = { version = "0.8", default-features = false, features = [ "std" ], optional = true }
+regex = { version = "1.5", default-features = false, features = [ "std" ], optional = true }
 serde_urlencoded = { version = "0.7", default-features = false, optional = true }
 tower = { version = "0.4", default-features = false, optional = true }
-tower-http = { version = "0.3", default-features = false, features = ["cors", "catch-panic", "trace"], optional = true }
-zeroize = { version = "1.5", default-features = false, features = ["std"], optional = true }
+tower-http = { version = "0.3", default-features = false, features = [ "cors", "catch-panic", "trace" ], optional = true }
+zeroize = { version = "1.5", default-features = false, features = [ "std" ], optional = true }
 
 # INX
 bee-inx = { version = "1.0.0-beta.4", default-features = false, optional = true }
 tonic = { version = "0.8", default-features = false, optional = true }
 
 # Metrics
 metrics = { version = "0.20.0", default-features = false }
-metrics-exporter-prometheus = { version = "0.11.0", default-features = false, features = ["http-listener", "tokio"] }
+metrics-exporter-prometheus = { version = "0.11.0", default-features = false, features = [ "http-listener", "tokio" ] }
 metrics-util = { version = "0.14.0", default-features = false }
 
 # Stardust types
 bee-api-types-stardust = { package = "bee-api-types", version = "1.0.0-beta.4", default-features = false, optional = true }
-bee-block-stardust = { package = "bee-block", version = "1.0.0-beta.5", default-features = false, features = [ "dto", "std", "serde", ], optional = true }
+bee-block-stardust = { package = "bee-block", version = "1.0.0-beta.5", default-features = false, features = [ "dto", "rand", "std", "serde" ], optional = true }
 
 # Tokio Console
 console-subscriber = { version = "0.1", default-features = false, optional = true }
 
 [dev-dependencies]
-bee-block-stardust = { package = "bee-block", version = "1.0.0-beta.5", default-features = false, features = [ "dto", "rand", "std", "serde", ] }
 packable = { version = "0.5", default-features = false }
 
 [features]
-default = [
+default = [ 
     "api",
     "inx",
     "stardust",
 ]
-api = [ 
+api = [
     "dep:auth-helper",
     "dep:axum",
     "dep:ed25519",
@@ -111,15 +110,15 @@ api = [
     "dep:zeroize",
     "stardust",
 ]
-console = [
+console = [ 
     "dep:console-subscriber",
     "tokio/tracing",
 ]
-inx = [
+inx = [ 
     "dep:bee-inx",
     "dep:tonic",
 ]
-stardust = [
+stardust = [ 
     "dep:bee-block-stardust",
     "dep:bee-api-types-stardust",
 ]

src/bin/inx-chronicle/api/stardust/analytics/routes.rs

@@ -9,8 +9,8 @@ use chronicle::{
         MongoDb,
     },
     types::stardust::block::{
-        AliasOutput, BasicOutput, FoundryOutput, MilestonePayload, NftOutput, TaggedDataPayload, TransactionPayload,
-        TreasuryTransactionPayload,
+        output::{AliasOutput, BasicOutput, FoundryOutput, NftOutput},
+        payload::{MilestonePayload, TaggedDataPayload, TransactionPayload, TreasuryTransactionPayload},
     },
 };
 

src/bin/inx-chronicle/api/stardust/core/routes.rs

@@ -29,7 +29,11 @@ use chronicle::{
         MongoDb,
     },
     types::{
-        stardust::block::{BlockId, MilestoneId, OutputId, TransactionId},
+        stardust::block::{
+            output::OutputId,
+            payload::{milestone::MilestoneId, transaction::TransactionId},
+            BlockId,
+        },
         tangle::MilestoneIndex,
     },
 };

src/bin/inx-chronicle/api/stardust/explorer/extractors.rs

@@ -10,7 +10,7 @@ use axum::{
 };
 use chronicle::{
     db::collections::SortOrder,
-    types::{stardust::block::OutputId, tangle::MilestoneIndex},
+    types::{stardust::block::output::OutputId, tangle::MilestoneIndex},
 };
 use serde::Deserialize;
 

src/bin/inx-chronicle/api/stardust/explorer/routes.rs

@@ -6,7 +6,7 @@ use std::str::FromStr;
 use axum::{extract::Path, routing::get, Extension, Router};
 use chronicle::{
     db::MongoDb,
-    types::stardust::block::{Address, BlockId, MilestoneId},
+    types::stardust::block::{payload::milestone::MilestoneId, Address, BlockId},
 };
 use futures::{StreamExt, TryStreamExt};
 

src/bin/inx-chronicle/api/stardust/indexer/extractors.rs

@@ -11,7 +11,7 @@ use axum::{
 use chronicle::{
     db::collections::{AliasOutputsQuery, BasicOutputsQuery, FoundryOutputsQuery, NftOutputsQuery, SortOrder},
     types::{
-        stardust::block::{Address, OutputId},
+        stardust::block::{output::OutputId, Address},
         tangle::MilestoneIndex,
     },
 };

src/bin/inx-chronicle/api/stardust/indexer/routes.rs

@@ -9,7 +9,7 @@ use chronicle::{
         collections::{AliasOutputsQuery, BasicOutputsQuery, FoundryOutputsQuery, IndexedId, NftOutputsQuery},
         MongoDb,
     },
-    types::stardust::block::{AliasId, FoundryId, NftId},
+    types::stardust::block::output::{AliasId, FoundryId, NftId},
 };
 use mongodb::bson;
 

src/db/collections/block.rs

@@ -16,7 +16,7 @@ use crate::{
     db::MongoDb,
     types::{
         ledger::{BlockMetadata, LedgerInclusionState},
-        stardust::block::{Block, BlockId, OutputId, Payload, TransactionId},
+        stardust::block::{output::OutputId, payload::transaction::TransactionId, Block, BlockId, Payload},
         tangle::MilestoneIndex,
     },
 };

src/db/collections/ledger_update.rs

@@ -20,7 +20,7 @@ use crate::{
     types::{
         ledger::{LedgerOutput, LedgerSpent, MilestoneIndexTimestamp},
         stardust::{
-            block::{Address, OutputId},
+            block::{output::OutputId, Address},
             milestone::MilestoneTimestamp,
         },
         tangle::MilestoneIndex,

src/db/collections/milestone.rs

@@ -18,7 +18,7 @@ use crate::{
     types::{
         ledger::MilestoneIndexTimestamp,
         stardust::{
-            block::{MilestoneId, MilestoneOption, MilestonePayload},
+            block::payload::milestone::{MilestoneId, MilestoneOption, MilestonePayload},
             milestone::MilestoneTimestamp,
         },
         tangle::MilestoneIndex,
@@ -146,8 +146,7 @@ impl MongoDb {
                 doc! { "at.milestone_index": index },
                 FindOneOptions::builder()
                     .projection(doc! {
-                        "milestone_id": "$milestone_id",
-
+                        "milestone_id": "$_id",
                     })
                     .build(),
             )
@@ -173,12 +172,9 @@ impl MongoDb {
             payload,
         };
 
-        let mut doc = bson::to_document(&milestone_document)?;
-        doc.insert("_id", milestone_document.milestone_id.to_hex());
-
         self.db
-            .collection::<bson::Document>(MilestoneDocument::COLLECTION)
-            .insert_one(doc, None)
+            .collection::<MilestoneDocument>(MilestoneDocument::COLLECTION)
+            .insert_one(milestone_document, None)
             .await?;
 
         Ok(())

src/db/collections/mod.rs

@@ -24,8 +24,8 @@ pub use self::{
     treasury::TreasuryResult,
 };
 use crate::types::stardust::block::{
-    AliasOutput, BasicOutput, FoundryOutput, MilestonePayload, NftOutput, TaggedDataPayload, TransactionPayload,
-    TreasuryTransactionPayload,
+    output::{AliasOutput, BasicOutput, FoundryOutput, NftOutput},
+    payload::{MilestonePayload, TaggedDataPayload, TransactionPayload, TreasuryTransactionPayload},
 };
 
 /// Batch size for `insert_many` operations.