From a371770d24b62e52f91fb7a4cd5c146efbb46c54 Mon Sep 17 00:00:00 2001 From: James Gilles Date: Tue, 3 Sep 2024 17:51:36 -0400 Subject: [PATCH 1/2] WIP: updating Schema and system tables --- crates/core/src/db/datastore/system_tables.rs | 35 +++++++------- crates/lib/src/db/raw_def/v9.rs | 17 +++++-- crates/schema/src/def.rs | 12 +++-- crates/schema/src/def/validate/v8.rs | 2 +- crates/schema/src/def/validate/v9.rs | 2 +- crates/schema/src/schema.rs | 48 +++++++++---------- 6 files changed, 61 insertions(+), 55 deletions(-) diff --git a/crates/core/src/db/datastore/system_tables.rs b/crates/core/src/db/datastore/system_tables.rs index 25daca6503..bd031d5d56 100644 --- a/crates/core/src/db/datastore/system_tables.rs +++ b/crates/core/src/db/datastore/system_tables.rs @@ -173,7 +173,7 @@ macro_rules! st_fields_enum { // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StTableFields { "table_id", TableId = 0, - "table_name", TableName = 1, + "name", TableName = 1, "table_type", TableType = 2, "table_access", TablesAccess = 3, }); @@ -181,18 +181,16 @@ st_fields_enum!(enum StTableFields { st_fields_enum!(enum StColumnFields { "table_id", TableId = 0, "col_pos", ColPos = 1, - "col_name", ColName = 2, - "col_type", ColType = 3, + "name", ColName = 2, + "type", ColType = 3, }); // WARNING: For a stable schema, don't change the field names and discriminants. -st_fields_enum!(enum StIndexFields { - "index_id", IndexId = 0, - "table_id", TableId = 1, - "index_name", IndexName = 2, - "columns", Columns = 3, - "is_unique", IsUnique = 4, - "index_type", IndexType = 5, -}); +st_fields_enum!( + enum StIndexFields { + // TODO(jgilles): how do we add a system table that stores only an enum? + // What if we need to add new variants to the enum? + } +); // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!( /// The fields that define the internal table [crate::db::relational_db::ST_SEQUENCES_NAME]. @@ -208,13 +206,12 @@ st_fields_enum!( "allocated", Allocated = 8, }); // WARNING: For a stable schema, don't change the field names and discriminants. -st_fields_enum!(enum StConstraintFields { - "constraint_id", ConstraintId = 0, - "constraint_name", ConstraintName = 1, - "constraints", Constraints = 2, - "table_id", TableId = 3, - "columns", Columns = 4, -}); +st_fields_enum!( + enum StConstraintFields { + // TODO(jgilles): how do we add a system table that stores only an enum? + // What if we need to add new variants to the enum? + } +); // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StModuleFields { "database_address", DatabaseAddress = 0, @@ -233,10 +230,10 @@ st_fields_enum!(enum StVarFields { "name", Name = 0, "value", Value = 1, }); - st_fields_enum!(enum StScheduledFields { "table_id", TableId = 0, "reducer_name", ReducerName = 1, + "name", Name = 2, }); /// System Table [ST_TABLE_NAME] diff --git a/crates/lib/src/db/raw_def/v9.rs b/crates/lib/src/db/raw_def/v9.rs index e60581c889..b18157b214 100644 --- a/crates/lib/src/db/raw_def/v9.rs +++ b/crates/lib/src/db/raw_def/v9.rs @@ -117,8 +117,8 @@ pub struct RawTableDefV9 { /// The indices of the table. pub indexes: Vec, - /// Any unique constraints on the table. - pub unique_constraints: Vec, + /// Any constraints on the table. + pub constraints: Vec, /// The sequences for the table. pub sequences: Vec, @@ -272,6 +272,13 @@ pub struct RawUniqueConstraintDefV9 { pub columns: ColList, } +#[derive(Debug, Clone, ser::Serialize, de::Deserialize)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +#[non_exhaustive] +pub enum RawConstraintDefV9 { + Unique(RawUniqueConstraintDefV9), +} + /// Marks a table as a timer table for a scheduled reducer. /// /// The table must have columns: @@ -397,7 +404,7 @@ impl RawModuleDefV9Builder { name, product_type_ref, indexes: vec![], - unique_constraints: vec![], + constraints: vec![], sequences: vec![], schedule: None, primary_key: None, @@ -570,8 +577,8 @@ impl<'a> RawTableDefBuilder<'a> { pub fn with_unique_constraint(mut self, columns: ColList, name: Option) -> Self { let name = name.unwrap_or_else(|| self.generate_unique_constraint_name(&columns)); self.table - .unique_constraints - .push(RawUniqueConstraintDefV9 { name, columns }); + .constraints + .push(RawConstraintDefV9::Unique(RawUniqueConstraintDefV9 { name, columns })); self } diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 6bcd3626de..02a6a38403 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -329,8 +329,8 @@ pub struct TableDef { /// The indices on the table, indexed by name. pub indexes: IdentifierMap, - /// The unique constraints on the table, indexed by name. - pub unique_constraints: IdentifierMap, + /// The constraints on the table, indexed by name. + pub constraints: IdentifierMap, /// The sequences for the table, indexed by name. pub sequences: IdentifierMap, @@ -376,7 +376,7 @@ impl From for RawTableDefV9 { product_type_ref, primary_key, indexes: to_raw(indexes, |index: &RawIndexDefV9| &index.name), - unique_constraints: to_raw(unique_constraints, |constraint: &RawUniqueConstraintDefV9| { + constraints: to_raw(unique_constraints, |constraint: &RawUniqueConstraintDefV9| { &constraint.name }), sequences: to_raw(sequences, |sequence: &RawSequenceDefV9| &sequence.name), @@ -518,6 +518,12 @@ pub struct ColumnDef { pub table_name: Identifier, } +/// A constraint on a table. +#[non_exhaustive] +pub enum ConstraintDef { + Unique(UniqueConstraintDef), +} + /// Requires that the projection of the table onto these columns is an bijection. /// /// That is, there must be a one-to-one relationship between a row and the `columns` of that row. diff --git a/crates/schema/src/def/validate/v8.rs b/crates/schema/src/def/validate/v8.rs index 2ad9d061ec..12ce91f3fe 100644 --- a/crates/schema/src/def/validate/v8.rs +++ b/crates/schema/src/def/validate/v8.rs @@ -124,7 +124,7 @@ fn upgrade_table( product_type_ref, primary_key, indexes, - unique_constraints, + constraints: unique_constraints, sequences, schedule, table_type, diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 7c22e0aaf9..926b3a241c 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -129,7 +129,7 @@ impl ModuleValidator<'_> { product_type_ref, primary_key, indexes, - unique_constraints, + constraints: unique_constraints, sequences, schedule, table_type, diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index faab6abc38..00a32a37d0 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -46,7 +46,7 @@ pub struct TableSchema { /// The unique identifier of the table within the database. pub table_id: TableId, /// The name of the table. - pub table_name: Box, + pub name: Box, /// The columns of the table. /// Inaccessible to prevent mutation. /// The ordering of the columns is significant. Columns are frequently identified by `ColId`, that is, position in this list. @@ -714,6 +714,7 @@ pub struct ColumnSchema { /// The name of the column. Unique within the table. pub col_name: Box, /// The type of the column. + /// May refer to the `Typespace` of the module. pub col_type: AlgebraicType, } @@ -893,23 +894,20 @@ impl From for RawSequenceDefV8 { /// A struct representing the schema of a database index. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct IndexSchema { - /// The unique ID of the index within the schema. +pub enum IndexSchema { + BTree(BTreeIndexSchema), +} + +/// A BTree index. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BTreeIndexSchema { + /// The ID of the index. pub index_id: IndexId, - /// The ID of the table associated with the index. + /// The ID of the table to which the index is attached. pub table_id: TableId, - /// The type of the index. - pub index_type: IndexType, - /// The name of the index. This should not be assumed to follow any particular format. - /// Unique within the table. - // TODO(jgilles): check and/or specify if this is currently unique within the database. - pub index_name: Box, - /// If the index is unique. - pub is_unique: bool, - /// The list of columns associated with the index. - /// This is truly a list: the order of the columns is significant. - /// The columns are projected and serialized to bitstrings in this order, - /// which affects the order of elements within a BTreeIndex. + /// The name of the index. + pub name: Box, + /// The columns of the index. pub columns: ColList, } @@ -964,17 +962,15 @@ impl From for RawIndexDefV8 { /// This struct holds information about a database constraint, including its unique identifier, /// name, the table it belongs to, and the columns it is associated with. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct ConstraintSchema { - /// The unique ID of the constraint within the database. +pub enum ConstraintSchema { + Unique(UniqueConstraintSchema), +} + +/// A unique constraint. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UniqueConstraintSchema { + pub name: Identifier, pub constraint_id: ConstraintId, - /// The name of the constraint. - /// Deprecated, in the future constraints will be identified by the columns they refer to and their constraint type. - pub constraint_name: Box, - /// The constraints applied to the columns - pub constraints: Constraints, - /// The ID of the table the constraint applies to. - pub table_id: TableId, - /// The columns the constraint applies to. pub columns: ColList, } From 57f8341779fdd17ab356a20b603e20fcd8c0685a Mon Sep 17 00:00:00 2001 From: James Gilles Date: Tue, 3 Sep 2024 18:02:47 -0400 Subject: [PATCH 2/2] Continue sketching --- crates/core/src/db/datastore/system_tables.rs | 18 ++++++++++--- crates/schema/src/def.rs | 15 +++++++---- crates/schema/src/schema.rs | 27 ++++++++++++++----- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/crates/core/src/db/datastore/system_tables.rs b/crates/core/src/db/datastore/system_tables.rs index bd031d5d56..1f8031308b 100644 --- a/crates/core/src/db/datastore/system_tables.rs +++ b/crates/core/src/db/datastore/system_tables.rs @@ -187,8 +187,14 @@ st_fields_enum!(enum StColumnFields { // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!( enum StIndexFields { - // TODO(jgilles): how do we add a system table that stores only an enum? - // What if we need to add new variants to the enum? + "index_id", IndexId = 0, + "table_id", TableId = 1, + "index_name", IndexName = 2, + + // TODO(jgilles): can a system table store an enum? + // What if we need to add new variants to the enum? Will that break + // the schema? Does this need to store a byte array instead? + "index_algorithm", IndexAlgorithm = 3, } ); // WARNING: For a stable schema, don't change the field names and discriminants. @@ -208,8 +214,12 @@ st_fields_enum!( // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!( enum StConstraintFields { - // TODO(jgilles): how do we add a system table that stores only an enum? - // What if we need to add new variants to the enum? + "constraint_id", ConstraintId = 0, + "name", ConstraintName = 1, + // TODO(jgilles): can a system table store an enum? + // What if we need to add new variants to the enum? Will that break + // the schema? Does this need to store a byte array instead? + "data", ConstraintData = 3, } ); // WARNING: For a stable schema, don't change the field names and discriminants. diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 02a6a38403..e112cdefd6 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -518,9 +518,17 @@ pub struct ColumnDef { pub table_name: Identifier, } -/// A constraint on a table. +/// A constraint definition. +pub struct ConstraintDef { + /// The name of the constraint. + pub name: Identifier, + /// The data for the constraint. + pub data: ConstraintData, +} + +/// Data for a constraint. #[non_exhaustive] -pub enum ConstraintDef { +pub enum ConstraintData { Unique(UniqueConstraintDef), } @@ -530,9 +538,6 @@ pub enum ConstraintDef { #[derive(Debug, Clone, Eq, PartialEq)] #[non_exhaustive] pub struct UniqueConstraintDef { - /// The name of the unique constraint. Must be unique within the containing `RawDatabaseDef`. - pub name: Identifier, - /// The columns on the containing `TableDef` pub columns: ColList, } diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index 00a32a37d0..6967c35f7f 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -61,8 +61,10 @@ pub struct TableSchema { pub table_type: StTableType, /// The visibility of the table. pub table_access: StAccess, - pub scheduled: Option>, + pub scheduled: Option, /// Cache for `row_type_for_table` in the data store. + /// Not stored in resolved form, may contain references to the module's + /// `Typespace`. row_type: ProductType, } @@ -805,8 +807,7 @@ pub struct SequenceSchema { /// The unique identifier for the sequence within a database. pub sequence_id: SequenceId, /// The name of the sequence. - /// Deprecated. In the future, sequences will be identified by col_pos. - pub sequence_name: Box, + pub name: Box, /// The ID of the table associated with the sequence. pub table_id: TableId, /// The position of the column associated with this sequence. @@ -957,20 +958,34 @@ impl From for RawIndexDefV8 { } } +/// A struct representing the schema of a database constraint. +pub struct ConstraintSchema { + /// The unique identifier for the constraint within a database. + pub constraint_id: ConstraintId, + /// The name of the constraint. + pub constraint_name: Box, + /// The data attached to the constraint. + pub constraint_data: ConstraintData, +} + /// A struct representing the schema of a database constraint. /// /// This struct holds information about a database constraint, including its unique identifier, /// name, the table it belongs to, and the columns it is associated with. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum ConstraintSchema { +pub enum ConstraintData { Unique(UniqueConstraintSchema), } /// A unique constraint. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UniqueConstraintSchema { - pub name: Identifier, - pub constraint_id: ConstraintId, + /// A unique constraint applies to columns of this table. + pub table_id: TableId, + /// The columns the constraint applies to. + /// Projecting the table onto these columns must result in a table with the same cardinality + /// (set-theoretically). + /// I.e. if you project onto these columns and deduplicate, you shouldn't need to remove any rows. pub columns: ColList, }