Update TableSchema & system tables to resemble ABI V9 (#1697)

crates/bench/Cargo.toml

@@ -37,7 +37,7 @@ spacetimedb-client-api = { path = "../client-api" }
 spacetimedb-testing = { path = "../testing" }
 spacetimedb-primitives = { path = "../primitives" }
 spacetimedb-table = { path = "../table" }
-spacetimedb-schema.workspace = true
+spacetimedb-schema = { workspace = true, features = ["test"] }
 
 anyhow.workspace = true
 anymap.workspace = true

crates/bench/benches/special.rs

@@ -6,12 +6,11 @@ use spacetimedb_bench::{
     schemas::{create_sequential, u32_u64_str, u32_u64_u64, u64_u64_u32, BenchTable, RandomTable},
     spacetime_module::BENCHMARKS_MODULE,
 };
-use spacetimedb_lib::db::raw_def::RawTableDefV8;
 use spacetimedb_lib::{sats, ProductValue};
 use spacetimedb_schema::schema::TableSchema;
 use spacetimedb_testing::modules::start_runtime;
-use std::hint::black_box;
 use std::time::{Duration, Instant};
+use std::{hint::black_box, sync::Arc};
 
 #[global_allocator]
 static GLOBAL: MiMalloc = MiMalloc;
@@ -155,9 +154,10 @@ fn serialize_benchmarks<
         );
     });
 
-    #[allow(deprecated)]
+    let mut table_schema = TableSchema::from_product_type(T::product_type());
+    table_schema.table_name = name.into();
     let mut table = spacetimedb_table::table::Table::new(
-        TableSchema::from_def(0.into(), RawTableDefV8::from_product(name, T::product_type().clone())).into(),
+        Arc::new(table_schema),
         spacetimedb_table::indexes::SquashedOffset::COMMITTED_STATE,
     );
     let mut blob_store = spacetimedb_table::blob_store::HashMapBlobStore::default();

crates/bench/src/spacetime_module.rs

@@ -82,8 +82,11 @@ impl BenchDatabase for SpacetimeModule {
                 .await
         });
 
-        for thing in module.client.module.catalog().iter() {
-            log::trace!("SPACETIME_MODULE: LOADED: {} {:?}", thing.0, thing.1.ty());
+        for table in module.client.module.info.module_def.tables() {
+            log::trace!("SPACETIME_MODULE: LOADED TABLE: {:?}", table);
+        }
+        for reducer in module.client.module.info.module_def.reducers() {
+            log::trace!("SPACETIME_MODULE: LOADED REDUCER: {:?}", reducer);
         }
         Ok(SpacetimeModule {
             runtime,

crates/bench/src/spacetime_raw.rs

@@ -5,9 +5,12 @@ use crate::{
 };
 use spacetimedb::db::relational_db::{tests_utils::TestDB, RelationalDB};
 use spacetimedb::execution_context::ExecutionContext;
-use spacetimedb_lib::db::raw_def::{RawIndexDefV8, RawTableDefV8};
 use spacetimedb_lib::sats::AlgebraicValue;
-use spacetimedb_primitives::{ColId, TableId};
+use spacetimedb_primitives::{ColId, IndexId, TableId};
+use spacetimedb_schema::{
+    def::{BTreeAlgorithm, IndexAlgorithm},
+    schema::{IndexSchema, TableSchema},
+};
 use std::hint::black_box;
 use tempdir::TempDir;
 
@@ -38,21 +41,39 @@ impl BenchDatabase for SpacetimeRaw {
     fn create_table<T: BenchTable>(&mut self, index_strategy: IndexStrategy) -> ResultBench<Self::TableId> {
         let name = table_name::<T>(index_strategy);
         self.db.with_auto_commit(&ExecutionContext::default(), |tx| {
-            let table_def = RawTableDefV8::from_product(&name, T::product_type());
-            let table_id = self.db.create_table(tx, table_def)?;
+            let mut table_schema = TableSchema::from_product_type(T::product_type());
+            table_schema.table_name = name.clone().into();
+            let table_id = self.db.create_table(tx, table_schema)?;
             self.db.rename_table(tx, table_id, &name)?;
             match index_strategy {
                 IndexStrategy::Unique0 => {
-                    self.db
-                        .create_index(tx, table_id, RawIndexDefV8::btree("id".into(), ColId(0), true))?;
+                    self.db.create_index(
+                        tx,
+                        IndexSchema {
+                            index_id: IndexId::SENTINEL,
+                            table_id,
+                            index_name: "id".into(),
+                            index_algorithm: IndexAlgorithm::BTree(BTreeAlgorithm {
+                                columns: ColId(0).into(),
+                            }),
+                        },
+                        true,
+                    )?;
                 }
                 IndexStrategy::NoIndex => (),
                 IndexStrategy::BTreeEachColumn => {
                     for (i, column) in T::product_type().elements.iter().enumerate() {
                         self.db.create_index(
                             tx,
-                            table_id,
-                            RawIndexDefV8::btree(column.name.clone().unwrap(), i, false),
+                            IndexSchema {
+                                index_id: IndexId::SENTINEL,
+                                table_id,
+                                index_name: column.name.clone().unwrap(),
+                                index_algorithm: IndexAlgorithm::BTree(BTreeAlgorithm {
+                                    columns: ColId(i as _).into(),
+                                }),
+                            },
+                            false,
                         )?;
                     }
                 }

crates/cli/src/api.rs

@@ -2,9 +2,10 @@ use reqwest::{header, Client, RequestBuilder};
 use serde::Deserialize;
 use serde_json::value::RawValue;
 
+use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9;
 use spacetimedb_lib::de::serde::DeserializeWrapper;
 use spacetimedb_lib::sats::ProductType;
-use spacetimedb_lib::{Address, RawModuleDefV8};
+use spacetimedb_lib::Address;
 
 static APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),);
 
@@ -50,7 +51,7 @@ impl ClientApi {
     }
 
     /// Reads the `ModuleDef` from the `schema` endpoint.
-    pub async fn module_def(&self) -> anyhow::Result<RawModuleDefV8> {
+    pub async fn module_def(&self) -> anyhow::Result<RawModuleDefV9> {
         let res = self
             .client
             .get(self.con.db_uri("schema"))

crates/cli/src/subcommands/generate/csharp.rs

@@ -6,12 +6,12 @@ use std::ops::Deref;
 
 use convert_case::{Case, Casing};
 use spacetimedb_lib::sats::{AlgebraicType, AlgebraicTypeRef, ArrayType, ProductType, SumType};
-use spacetimedb_lib::{ReducerDef, TableDesc};
+use spacetimedb_lib::ReducerDef;
 use spacetimedb_primitives::ColList;
 use spacetimedb_schema::schema::TableSchema;
 
 use super::code_indenter::CodeIndenter;
-use super::{GenCtx, GenItem};
+use super::{GenCtx, GenItem, TableDescHack};
 
 fn scalar_or_string_name(b: &AlgebraicType) -> Option<&str> {
     Some(match b {
@@ -279,17 +279,13 @@ pub fn autogen_csharp_tuple(ctx: &GenCtx, name: &str, tuple: &ProductType, names
 }
 
 #[allow(deprecated)]
-pub fn autogen_csharp_table(ctx: &GenCtx, table: &TableDesc, namespace: &str) -> String {
+pub fn autogen_csharp_table(ctx: &GenCtx, table: &TableDescHack, namespace: &str) -> String {
     let tuple = ctx.typespace[table.data].as_product().unwrap();
     autogen_csharp_product_table_common(
         ctx,
         csharp_typename(ctx, table.data),
         tuple,
-        Some(
-            TableSchema::from_def(0.into(), table.schema.clone())
-                .validated()
-                .expect("Failed to generate table due to validation errors"),
-        ),
+        Some(table.schema.clone()),
         namespace,
     )
 }
@@ -317,7 +313,7 @@ fn autogen_csharp_product_table_common(
         write!(
             output,
             " : SpacetimeDB.{parent}<{name}, {namespace}.ReducerEvent>",
-            parent = if schema.pk().is_some() {
+            parent = if schema.primary_key.is_some() {
                 "DatabaseTableWithPrimaryKey"
             } else {
                 "DatabaseTable"
@@ -383,7 +379,7 @@ fn autogen_csharp_product_table_common(
 
         // If this is a table, we want to generate event accessor and indexes
         if let Some(schema) = &schema {
-            let constraints = schema.column_constraints();
+            let constraints = schema.backcompat_column_constraints();
             let mut unique_indexes = Vec::new();
             // Declare custom index dictionaries
             for col in schema.columns() {
@@ -450,7 +446,7 @@ fn autogen_csharp_access_funcs_for_struct(
     _table_name: &str,
     schema: &TableSchema,
 ) {
-    let constraints = schema.column_constraints();
+    let constraints = schema.backcompat_column_constraints();
     for col in schema.columns() {
         let is_unique = constraints[&ColList::new(col.col_pos)].has_unique();
 

crates/cli/src/subcommands/generate/mod.rs

@@ -10,13 +10,13 @@ use spacetimedb::host::wasmtime::{Mem, MemView, WasmPointee as _};
 use spacetimedb_data_structures::map::HashSet;
 use spacetimedb_lib::de::serde::DeserializeWrapper;
 use spacetimedb_lib::sats::{AlgebraicType, AlgebraicTypeRef, Typespace};
-use spacetimedb_lib::{bsatn, RawModuleDefV8, TableDesc, TypeAlias};
+use spacetimedb_lib::{bsatn, RawModuleDefV8, TypeAlias};
 use spacetimedb_lib::{RawModuleDef, MODULE_ABI_MAJOR_VERSION};
 use spacetimedb_primitives::errno;
 use spacetimedb_schema;
 use spacetimedb_schema::def::{ModuleDef, ReducerDef, ScopedTypeName, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
-use spacetimedb_schema::schema::TableSchema;
+use spacetimedb_schema::schema::{Schema, TableSchema};
 use std::fs;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
@@ -254,8 +254,8 @@ pub fn generate(module: RawModuleDef, lang: Language, namespace: &str) -> anyhow
             let tableset = module.tables().map(|t| t.product_type_ref).collect::<HashSet<_>>();
             let tables = module
                 .tables()
-                .map(|table| TableDesc {
-                    schema: TableSchema::from_module_def(table, 0.into()).into(),
+                .map(|table| TableDescHack {
+                    schema: TableSchema::from_module_def(&module, table, (), 0.into()),
                     data: table.product_type_ref,
                 })
                 .sorted_by(|a, b| a.schema.table_name.cmp(&b.schema.table_name));
@@ -337,8 +337,14 @@ trait Lang {
     fn generate_globals(&self, module: &ModuleDef, namespace: &str) -> Vec<(String, String)>;
 }
 
+/// Backwards-compatibible imitation of `TableDesc` that should be removed once the generators are updated to rely on `ModuleDef`.
+pub struct TableDescHack {
+    schema: TableSchema,
+    data: AlgebraicTypeRef,
+}
+
 pub enum GenItem {
-    Table(TableDesc),
+    Table(TableDescHack),
     TypeAlias(TypeAlias),
     Reducer(spacetimedb_lib::ReducerDef),
 }

crates/cli/src/subcommands/generate/rust.rs

@@ -5,7 +5,7 @@ use spacetimedb_lib::sats::AlgebraicTypeRef;
 use spacetimedb_primitives::ColList;
 use spacetimedb_schema::def::{ModuleDef, ReducerDef, ScopedTypeName, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
-use spacetimedb_schema::schema::TableSchema;
+use spacetimedb_schema::schema::{Schema, TableSchema};
 use spacetimedb_schema::type_for_generate::{
     AlgebraicTypeDef, AlgebraicTypeUse, PlainEnumTypeDef, PrimitiveType, ProductTypeDef, SumTypeDef,
 };
@@ -304,7 +304,7 @@ pub fn autogen_rust_table(module: &ModuleDef, table: &TableDef) -> String {
 
     out.newline();
 
-    let table = TableSchema::from_module_def(table, 0.into())
+    let table = TableSchema::from_module_def(module, table, (), 0.into())
         .validated()
         .expect("Failed to generate table due to validation errors");
     print_impl_tabletype(module, out, &type_name, product_def, &table);
@@ -430,7 +430,7 @@ fn print_table_filter_methods(
     table: &TableSchema,
 ) {
     write!(out, "impl {table_type_name} ");
-    let constraints = table.column_constraints();
+    let constraints = table.backcompat_column_constraints();
     out.delimited_block(
         "{",
         |out| {
@@ -796,8 +796,8 @@ fn print_handle_table_update_defn(module: &ModuleDef, out: &mut Indenter) {
             out.delimited_block(
                 "match table_name {",
                 |out| {
-                    for table_desc in iter_tables(module) {
-                        let table = TableSchema::from_module_def(table_desc, 0.into()).validated().unwrap();
+                    for table_def in iter_tables(module) {
+                        let table = TableSchema::from_module_def(module, table_def, (), 0.into()).validated().unwrap();
                         writeln!(
                             out,
                             "{:?} => client_cache.{}::<{}::{}>(callbacks, table_update),",
@@ -807,8 +807,8 @@ fn print_handle_table_update_defn(module: &ModuleDef, out: &mut Indenter) {
                             } else {
                                 "handle_table_update_no_primary_key"
                             },
-                            type_name(module, table_desc.product_type_ref).to_case(Case::Snake),
-                            type_name(module, table_desc.product_type_ref).to_case(Case::Pascal),
+                            type_name(module, table_def.product_type_ref).to_case(Case::Snake),
+                            type_name(module, table_def.product_type_ref).to_case(Case::Pascal),
                         );
                     }
                     writeln!(

crates/cli/src/subcommands/generate/typescript.rs

@@ -8,12 +8,12 @@ use spacetimedb_lib::sats::product_type::IDENTITY_TAG;
 use spacetimedb_lib::sats::{
     AlgebraicType, AlgebraicTypeRef, ArrayType, ProductType, ProductTypeElement, SumType, SumTypeVariant,
 };
-use spacetimedb_lib::{ReducerDef, TableDesc};
+use spacetimedb_lib::ReducerDef;
 use spacetimedb_primitives::ColList;
 use spacetimedb_schema::schema::TableSchema;
 
 use super::code_indenter::CodeIndenter;
-use super::{GenCtx, GenItem, INDENT};
+use super::{GenCtx, GenItem, TableDescHack, INDENT};
 
 fn scalar_or_string_to_ts(ty: &AlgebraicType) -> Option<(&str, &str)> {
     Some(match ty {
@@ -499,13 +499,13 @@ pub fn autogen_typescript_tuple(ctx: &GenCtx, name: &str, tuple: &ProductType) -
     autogen_typescript_product_table_common(ctx, name, tuple, None)
 }
 #[allow(deprecated)]
-pub fn autogen_typescript_table(ctx: &GenCtx, table: &TableDesc) -> String {
+pub fn autogen_typescript_table(ctx: &GenCtx, table: &TableDescHack) -> String {
     let tuple = ctx.typespace[table.data].as_product().unwrap();
     autogen_typescript_product_table_common(
         ctx,
         typescript_typename(ctx, table.data),
         tuple,
-        Some(TableSchema::from_def(0.into(), table.schema.clone())),
+        Some(table.schema.clone()),
     )
 }
 
@@ -767,7 +767,7 @@ fn autogen_typescript_access_funcs_for_struct(
     table_name: &str,
     table: &TableSchema,
 ) {
-    let constraints = table.column_constraints();
+    let constraints = table.backcompat_column_constraints();
     for col in table.columns() {
         let is_unique = constraints[&ColList::new(col.col_pos)].has_unique();
         let field = &product_type.elements[col.col_pos.idx()];

crates/cli/src/subcommands/subscribe.rs

@@ -6,9 +6,9 @@ use http::uri::Scheme;
 use serde_json::Value;
 use spacetimedb_client_api_messages::websocket::{self as ws, EncodedValue};
 use spacetimedb_data_structures::map::HashMap;
+use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9;
 use spacetimedb_lib::de::serde::{DeserializeWrapper, SeedWrapper};
 use spacetimedb_lib::ser::serde::SerializeWrapper;
-use spacetimedb_lib::RawModuleDefV8;
 use spacetimedb_standalone::TEXT_PROTOCOL;
 use std::time::Duration;
 use tokio::io::AsyncWriteExt;
@@ -92,17 +92,17 @@ fn parse_msg_json(msg: &WsMessage) -> Option<ws::ServerMessage> {
 
 fn reformat_update(
     msg: ws::DatabaseUpdate,
-    schema: &RawModuleDefV8,
+    schema: &RawModuleDefV9,
 ) -> anyhow::Result<HashMap<String, SubscriptionTable>> {
     msg.tables
         .into_iter()
         .map(|upd| {
             let table_schema = schema
                 .tables
                 .iter()
-                .find(|tbl| tbl.schema.table_name.as_ref() == upd.table_name)
+                .find(|tbl| tbl.name.as_ref() == upd.table_name)
                 .context("table not found in schema")?;
-            let table_ty = schema.typespace.resolve(table_schema.data);
+            let table_ty = schema.typespace.resolve(table_schema.product_type_ref);
 
             let reformat_row = |row: EncodedValue| {
                 let EncodedValue::Text(row) = row else {
@@ -208,7 +208,7 @@ where
 
 /// Await the initial [`ServerMessage::SubscriptionUpdate`].
 /// If `module_def` is `Some`, print a JSON representation to stdout.
-async fn await_initial_update<S>(ws: &mut S, module_def: Option<&RawModuleDefV8>) -> anyhow::Result<()>
+async fn await_initial_update<S>(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> anyhow::Result<()>
 where
     S: TryStream<Ok = WsMessage> + Unpin,
     S::Error: std::error::Error + Send + Sync + 'static,
@@ -243,7 +243,7 @@ where
 async fn consume_transaction_updates<S>(
     ws: &mut S,
     num: Option<u32>,
-    module_def: &RawModuleDefV8,
+    module_def: &RawModuleDefV9,
 ) -> anyhow::Result<bool>
 where
     S: TryStream<Ok = WsMessage> + Unpin,