define Address as u128, Identity as u256

Co-authored-by: James Gilles <jameshgilles@gmail.com>

Fix C#?

Fix codegen tests

Wipe old bench data before running bench tests

Fix csharp?

Revert U128/U256 formatting changes, they were printing numbers backwards

Fix failing test

Fix some passing tests, add a failing test

Bump serde_json version and add arbitrary_precision feature.
This allows passing u128 through serde_json::Value, which is done in the CLI crate.

Cargo.toml

@@ -203,7 +203,7 @@ scopeguard = "1.1.0"
 second-stack = "0.3"
 sendgrid = "0.21"
 serde = { version = "1.0.136", features = ["derive"] }
-serde_json = { version = "1.0.87", features = ["raw_value"] }
+serde_json = { version = "1.0.128", features = ["raw_value", "arbitrary_precision"] }
 serde_path_to_error = "0.1.9"
 serde_with = { version = "3.3.0", features = ["base64", "hex"] }
 serial_test = "2.0.0"

crates/bench/src/lib.rs

@@ -16,7 +16,7 @@ mod tests {
         sqlite::SQLite,
         ResultBench,
     };
-    use std::{io, sync::Once};
+    use std::{io, path::Path, sync::Once};
     use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
 
     static INIT: Once = Once::new();
@@ -39,6 +39,15 @@ mod tests {
                 .with(fmt_layer)
                 .with(env_filter_layer)
                 .init();
+
+            // Remove cached data from previous runs.
+            // This directory is only reused to speed up runs with Callgrind. In tests, it's fine to wipe it.
+            let mut bench_dot_spacetime = Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf();
+            bench_dot_spacetime.push(".spacetime");
+            if std::fs::metadata(&bench_dot_spacetime).is_ok() {
+                std::fs::remove_dir_all(bench_dot_spacetime)
+                    .expect("failed to wipe Spacetimedb/crates/bench/.spacetime");
+            }
         });
     }
 

crates/bindings-csharp/BSATN.Runtime/Builtins.cs

@@ -6,6 +6,15 @@ namespace SpacetimeDB;
 using SpacetimeDB.BSATN;
 using SpacetimeDB.Internal;
 
+internal static class Util
+{
+    // Same as `Convert.ToHexString`, but that method is not available in .NET Standard
+    // which we need to target for Unity support.
+    public static string ToHex<T>(T val)
+        where T : struct =>
+        BitConverter.ToString(MemoryMarshal.AsBytes([val]).ToArray()).Replace("-", "");
+}
+
 public readonly partial struct Unit
 {
     // Custom BSATN that returns an inline empty product type that can be recognised by SpacetimeDB.
@@ -63,69 +72,69 @@ string wrapperPropertyName
         );
 }
 
-public record Address : BytesWrapper
+public readonly record struct Address
 {
-    protected override int SIZE => 16;
-
-    public Address() { }
+    private readonly U128 value;
 
-    private Address(byte[] bytes)
-        : base(bytes) { }
+    internal Address(U128 v) => value = v;
 
     public static Address? From(byte[] bytes)
     {
-        if (bytes.All(b => b == 0))
-        {
-            return null;
-        }
-        return new(bytes);
+        Debug.Assert(bytes.Length == 16);
+        var addr = new Address(MemoryMarshal.Read<U128>(bytes));
+        return addr == default ? null : addr;
     }
 
     public static Address Random()
     {
         var random = new Random();
-        var addr = new Address();
-        random.NextBytes(addr.bytes);
-        return addr;
+        var bytes = new byte[16];
+        random.NextBytes(bytes);
+        return Address.From(bytes) ?? default;
     }
 
     public readonly struct BSATN : IReadWrite<Address>
     {
-        public Address Read(BinaryReader reader) => new(ReadRaw(reader));
+        public Address Read(BinaryReader reader) =>
+            new(new SpacetimeDB.BSATN.U128Stdb().Read(reader));
 
-        public void Write(BinaryWriter writer, Address value) => value.Write(writer);
+        public void Write(BinaryWriter writer, Address value) =>
+            new SpacetimeDB.BSATN.U128Stdb().Write(writer, value.value);
 
         public AlgebraicType GetAlgebraicType(ITypeRegistrar registrar) =>
-            BytesWrapper.GetAlgebraicType(registrar, "__address_bytes");
+            new AlgebraicType.Product([new("__address__", new AlgebraicType.U128(default))]);
     }
 
-    // This must be explicitly forwarded to base, otherwise record will generate a new implementation.
-    public override string ToString() => base.ToString();
+    public override string ToString() => Util.ToHex(value);
 }
 
-public record Identity : BytesWrapper
+public readonly record struct Identity
 {
-    protected override int SIZE => 32;
+    private readonly U256 value;
 
-    public Identity() { }
+    internal Identity(U256 val) => value = val;
 
     public Identity(byte[] bytes)
-        : base(bytes) { }
+    {
+        Debug.Assert(bytes.Length == 32);
+        value = MemoryMarshal.Read<U256>(bytes);
+    }
 
     public static Identity From(byte[] bytes) => new(bytes);
 
     public readonly struct BSATN : IReadWrite<Identity>
     {
-        public Identity Read(BinaryReader reader) => new(ReadRaw(reader));
+        public Identity Read(BinaryReader reader) => new(new SpacetimeDB.BSATN.U256().Read(reader));
 
-        public void Write(BinaryWriter writer, Identity value) => value.Write(writer);
+        public void Write(BinaryWriter writer, Identity value) =>
+            new SpacetimeDB.BSATN.U256().Write(writer, value.value);
 
         public AlgebraicType GetAlgebraicType(ITypeRegistrar registrar) =>
-            BytesWrapper.GetAlgebraicType(registrar, "__identity_bytes");
+            new AlgebraicType.Product([new("__identity__", new AlgebraicType.U256(default))]);
     }
 
     // This must be explicitly forwarded to base, otherwise record will generate a new implementation.
-    public override string ToString() => base.ToString();
+    public override string ToString() => Util.ToHex(value);
 }
 
 // We store time information in microseconds in internal usages.

crates/bindings/src/rt.rs

@@ -348,7 +348,7 @@ extern "C" fn __describe_module__(description: BytesSink) {
 /// - `sender_3` contains bytes `[24..32]`.
 ///
 /// The `address_{0-1}` are the pieces of a `[u8; 16]` (`u128`) representing the callers's `Address`.
-/// They are encoded as follows (assuming `identity.__address_bytes: [u8; 16]`):
+/// They are encoded as follows (assuming `address.__address__: u128`):
 /// - `address_0` contains bytes `[0 ..8 ]`.
 /// - `address_1` contains bytes `[8 ..16]`.
 ///

crates/cli/src/subcommands/subscribe.rs

@@ -99,6 +99,7 @@ fn reformat_update<'a>(
             let table_ty = schema.typespace.resolve(table_schema.product_type_ref);
 
             let reformat_row = |row: &str| -> anyhow::Result<Value> {
+                // TODO: can the following two calls be merged into a single call to reduce allocations?
                 let row = serde_json::from_str::<Value>(row)?;
                 let row = serde::de::DeserializeSeed::deserialize(SeedWrapper(table_ty), row)?;
                 let row = table_ty.with_value(&row);

crates/core/src/client/messages.rs

@@ -157,7 +157,7 @@ impl ToProtocol for TransactionUpdateMessage {
                 },
                 energy_quanta_used: event.energy_quanta_used,
                 host_execution_duration_micros: event.host_execution_duration.as_micros() as u64,
-                caller_address: event.caller_address.unwrap_or(Address::zero()),
+                caller_address: event.caller_address.unwrap_or(Address::ZERO),
             };
 
             ws::ServerMessage::TransactionUpdate(tx_update)

crates/core/src/db/datastore/locking_tx_datastore/datastore.rs

@@ -1042,7 +1042,7 @@ mod tests {
     }
 
     fn get_datastore() -> Result<Locking> {
-        Locking::bootstrap(Address::zero())
+        Locking::bootstrap(Address::ZERO)
     }
 
     fn col(col: u16) -> ColList {
@@ -1358,15 +1358,15 @@ mod tests {
             ColRow { table: ST_CONSTRAINT_ID.into(), pos: 2, name: "table_id", ty: TableId::get_type() },
             ColRow { table: ST_CONSTRAINT_ID.into(), pos: 3, name: "constraint_data", ty: resolved_type_via_v9::<StConstraintData>() },
 
-            ColRow { table: ST_MODULE_ID.into(), pos: 0, name: "database_address", ty: AlgebraicType::bytes() },
-            ColRow { table: ST_MODULE_ID.into(), pos: 1, name: "owner_identity", ty: AlgebraicType::bytes() },
+            ColRow { table: ST_MODULE_ID.into(), pos: 0, name: "database_address", ty: AlgebraicType::U128 },
+            ColRow { table: ST_MODULE_ID.into(), pos: 1, name: "owner_identity", ty: AlgebraicType::U256 },
             ColRow { table: ST_MODULE_ID.into(), pos: 2, name: "program_kind", ty: AlgebraicType::U8 },
-            ColRow { table: ST_MODULE_ID.into(), pos: 3, name: "program_hash", ty: AlgebraicType::bytes() },
+            ColRow { table: ST_MODULE_ID.into(), pos: 3, name: "program_hash", ty: AlgebraicType::U256 },
             ColRow { table: ST_MODULE_ID.into(), pos: 4, name: "program_bytes", ty: AlgebraicType::bytes() },
             ColRow { table: ST_MODULE_ID.into(), pos: 5, name: "module_version", ty: AlgebraicType::String },
 
-            ColRow { table: ST_CLIENT_ID.into(), pos: 0, name: "identity", ty: AlgebraicType::bytes()},
-            ColRow { table: ST_CLIENT_ID.into(), pos: 1, name: "address", ty: AlgebraicType::bytes()},
+            ColRow { table: ST_CLIENT_ID.into(), pos: 0, name: "identity", ty: AlgebraicType::U256},
+            ColRow { table: ST_CLIENT_ID.into(), pos: 1, name: "address", ty: AlgebraicType::U128},
 
             ColRow { table: ST_VAR_ID.into(), pos: 0, name: "name", ty: AlgebraicType::String },
             ColRow { table: ST_VAR_ID.into(), pos: 1, name: "value", ty: resolved_type_via_v9::<StVarValue>() },

crates/core/src/db/datastore/system_tables.rs

@@ -27,7 +27,7 @@ use spacetimedb_sats::algebraic_value::ser::value_serialize;
 use spacetimedb_sats::hash::Hash;
 use spacetimedb_sats::product_value::InvalidFieldError;
 use spacetimedb_sats::{
-    impl_deserialize, impl_serialize, impl_st, AlgebraicType, AlgebraicValue, ArrayValue, SumValue,
+    impl_deserialize, impl_serialize, impl_st, u256, AlgebraicType, AlgebraicValue, ArrayValue, SumValue,
 };
 use spacetimedb_schema::def::{BTreeAlgorithm, ConstraintData, IndexAlgorithm, ModuleDef, UniqueConstraintData};
 use spacetimedb_schema::schema::{
@@ -804,23 +804,23 @@ impl_st!([] ModuleKind, AlgebraicType::U8);
 
 /// A wrapper for `Address` that acts like `AlgebraicType::bytes()` for serialization purposes.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub struct AddressViaBytes(pub Address);
-impl_serialize!([] AddressViaBytes, (self, ser) => self.0.as_slice().serialize(ser));
-impl_deserialize!([] AddressViaBytes, de => <[u8; 16]>::deserialize(de).map(Address::from_slice).map(AddressViaBytes));
-impl_st!([] AddressViaBytes, AlgebraicType::bytes());
-impl From<Address> for AddressViaBytes {
+pub struct AddressViaU128(pub Address);
+impl_serialize!([] AddressViaU128, (self, ser) => self.0.to_u128().serialize(ser));
+impl_deserialize!([] AddressViaU128, de => <u128>::deserialize(de).map(Address::from_u128).map(AddressViaU128));
+impl_st!([] AddressViaU128, AlgebraicType::U128);
+impl From<Address> for AddressViaU128 {
     fn from(addr: Address) -> Self {
         Self(addr)
     }
 }
 
 /// A wrapper for `Identity` that acts like `AlgebraicType::bytes()` for serialization purposes.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub struct IdentityViaBytes(pub Identity);
-impl_serialize!([] IdentityViaBytes, (self, ser) => self.0.as_bytes().serialize(ser));
-impl_deserialize!([] IdentityViaBytes, de => <[u8; 32]>::deserialize(de).map(|arr| Identity::from_slice(&arr[..])).map(IdentityViaBytes));
-impl_st!([] IdentityViaBytes, AlgebraicType::bytes());
-impl From<Identity> for IdentityViaBytes {
+pub struct IdentityViaU256(pub Identity);
+impl_serialize!([] IdentityViaU256, (self, ser) => self.0.to_u256().serialize(ser));
+impl_deserialize!([] IdentityViaU256, de => <u256>::deserialize(de).map(Identity::from_u256).map(IdentityViaU256));
+impl_st!([] IdentityViaU256, AlgebraicType::U256);
+impl From<Identity> for IdentityViaU256 {
     fn from(id: Identity) -> Self {
         Self(id)
     }
@@ -843,8 +843,8 @@ impl From<Identity> for IdentityViaBytes {
 #[derive(Clone, Debug, Eq, PartialEq, SpacetimeType)]
 #[sats(crate = spacetimedb_lib)]
 pub struct StModuleRow {
-    pub(crate) database_address: AddressViaBytes,
-    pub(crate) owner_identity: IdentityViaBytes,
+    pub(crate) database_address: AddressViaU128,
+    pub(crate) owner_identity: IdentityViaU256,
     pub(crate) program_kind: ModuleKind,
     pub(crate) program_hash: Hash,
     pub(crate) program_bytes: Box<[u8]>,
@@ -867,23 +867,24 @@ pub fn read_bytes_from_col(row: RowRef<'_>, col: impl StFields) -> Result<Box<[u
 
 /// Read an [`Address`] directly from the column `col` in `row`.
 ///
-/// The [`Address`] is assumed to be stored as a flat byte array.
+/// The [`Address`] is assumed to be stored as an u128.
 pub fn read_addr_from_col(row: RowRef<'_>, col: impl StFields) -> Result<Address, DBError> {
-    read_bytes_from_col(row, col).map(Address::from_slice)
+    let val: u128 = row.read_col(col.col_id())?;
+    Ok(val.into())
 }
 
 /// Read an [`Identity`] directly from the column `col` in `row`.
 ///
 /// The [`Identity`] is assumed to be stored as a flat byte array.
 pub fn read_identity_from_col(row: RowRef<'_>, col: impl StFields) -> Result<Identity, DBError> {
-    read_bytes_from_col(row, col).map(|bytes| Identity::from_slice(&bytes))
+    Ok(Identity::from_u256(row.read_col(col.col_id())?))
 }
 
 /// Read a [`Hash`] directly from the column `col` in `row`.
 ///
 /// The [`Hash`] is assumed to be stored as a flat byte array.
 pub fn read_hash_from_col(row: RowRef<'_>, col: impl StFields) -> Result<Hash, DBError> {
-    read_bytes_from_col(row, col).map(|bytes| Hash::from_slice(&bytes))
+    Ok(Hash::from_u256(row.read_col(col.col_id())?))
 }
 
 impl TryFrom<RowRef<'_>> for StModuleRow {
@@ -908,13 +909,18 @@ impl From<StModuleRow> for ProductValue {
 #[derive(Clone, Debug, Eq, PartialEq, SpacetimeType)]
 #[sats(crate = spacetimedb_lib)]
 pub struct StClientRow {
-    pub(crate) identity: IdentityViaBytes,
-    pub(crate) address: AddressViaBytes,
+    pub(crate) identity: IdentityViaU256,
+    pub(crate) address: AddressViaU128,
 }
 
+impl From<StClientRow> for ProductValue {
+    fn from(var: StClientRow) -> Self {
+        to_product_value(&var)
+    }
+}
 impl From<&StClientRow> for ProductValue {
-    fn from(x: &StClientRow) -> Self {
-        to_product_value(x)
+    fn from(var: &StClientRow) -> Self {
+        to_product_value(var)
     }
 }
 

crates/core/src/host/wasmtime/wasmtime_module.rs

@@ -194,8 +194,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance {
         set_store_fuel(store, budget.into());
 
         // Prepare sender identity and address.
-        let [sender_0, sender_1, sender_2, sender_3] = bytemuck::must_cast(*op.caller_identity.as_bytes());
-        let [address_0, address_1] = bytemuck::must_cast(*op.caller_address.as_slice());
+        let [sender_0, sender_1, sender_2, sender_3] = bytemuck::must_cast(op.caller_identity.to_byte_array());
+        let [address_0, address_1] = bytemuck::must_cast(op.caller_address.as_byte_array());
 
         // Prepare arguments to the reducer + the error sink & start timings.
         let (args_source, errors_sink) = store.data_mut().start_reducer(op.name, op.arg_bytes);

crates/core/src/sql/compiler.rs

@@ -237,7 +237,7 @@ mod tests {
     use spacetimedb_lib::{Address, Identity};
     use spacetimedb_primitives::{col_list, ColList, TableId};
     use spacetimedb_sats::{
-        product, satn, AlgebraicType, AlgebraicValue, ProductType, ProductTypeElement, Typespace, ValueWithType,
+        product, satn, AlgebraicType, AlgebraicValue, GroundSpacetimeType as _, ProductType, Typespace, ValueWithType,
     };
     use spacetimedb_vm::expr::{ColumnOp, IndexJoin, IndexScan, JoinExpr, Query};
     use std::convert::From;
@@ -402,56 +402,50 @@ mod tests {
     #[test]
     fn output_identity_address() -> ResultTest<()> {
         let row = product![AlgebraicValue::from(Identity::__dummy())];
-        let kind = ProductType::new(Box::new([ProductTypeElement::new(
-            Identity::get_type(),
-            Some("i".into()),
-        )]));
+        let kind: ProductType = [("i", Identity::get_type())].into();
         let ty = Typespace::EMPTY.with_type(&kind);
         let out = ty
             .with_values(&row)
             .map(|value| satn::PsqlWrapper { ty: &kind, value }.to_string())
             .collect::<Vec<_>>()
             .join(", ");
-        assert_eq!(
-            out,
-            "0x0000000000000000000000000000000000000000000000000000000000000000"
-        );
+        assert_eq!(out, "0");
 
         // Check tuples
         let kind = [
             ("a", AlgebraicType::String),
-            ("b", AlgebraicType::bytes()),
+            ("b", AlgebraicType::U256),
             ("o", Identity::get_type()),
             ("p", Address::get_type()),
         ]
         .into();
 
         let value = AlgebraicValue::product([
             AlgebraicValue::String("a".into()),
-            AlgebraicValue::Bytes((*Identity::ZERO.as_bytes()).into()),
-            AlgebraicValue::Bytes((*Identity::ZERO.as_bytes()).into()),
-            AlgebraicValue::Bytes((*Address::__DUMMY.as_slice()).into()),
+            Identity::ZERO.to_u256().into(),
+            Identity::ZERO.to_u256().into(),
+            Address::__DUMMY.to_u128().into(),
         ]);
 
         assert_eq!(
             satn::PsqlWrapper { ty: &kind, value }.to_string().as_str(),
-            "(0 = \"a\", 1 = 0x0000000000000000000000000000000000000000000000000000000000000000, 2 = 0x0000000000000000000000000000000000000000000000000000000000000000, 3 = 0x00000000000000000000000000000000)"
+            "(0 = \"a\", 1 = 0, 2 = 0, 3 = 0)"
         );
 
         let ty = Typespace::EMPTY.with_type(&kind);
 
         // Check struct
         let value = product![
-            AlgebraicValue::String("a".into()),
-            AlgebraicValue::Bytes((*Identity::ZERO.as_bytes()).into()),
-            AlgebraicValue::product([AlgebraicValue::Bytes((*Identity::ZERO.as_bytes()).into())]),
-            AlgebraicValue::product([AlgebraicValue::Bytes((*Address::__DUMMY.as_slice()).into())]),
+            "a",
+            Identity::ZERO.to_u256(),
+            AlgebraicValue::product([Identity::ZERO.to_u256().into()]),
+            AlgebraicValue::product([Address::__DUMMY.to_u128().into()]),
         ];
 
         let value = ValueWithType::new(ty, &value);
         assert_eq!(
             satn::PsqlWrapper { ty: ty.ty(), value }.to_string().as_str(),
-            "(a = \"a\", b = 0x0000000000000000000000000000000000000000000000000000000000000000, o = 0x0000000000000000000000000000000000000000000000000000000000000000, p = 0x00000000000000000000000000000000)"
+            "(a = \"a\", b = 0, o = 0, p = 0)"
         );
 
         Ok(())